Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Security.Cryptography;
using DurableTask.Core;
using DurableTask.Core.Exceptions;
using DurableTask.Core.History;
Expand All @@ -27,6 +28,16 @@
/// <param name="options">The client options.</param>
class ShimDurableTaskClient(string name, ShimDurableTaskClientOptions options) : DurableTaskClient(name)
{
// Polling parameters for WaitForInstanceStartAsync. PollingInterval matches the historical fixed
// 1-second polling cadence and is used, unjittered, for every steady-state delay -- so long-run
// polling volume never exceeds the historical rate. To desynchronize concurrent callers (avoiding
// synchronized polling bursts against the backend) without inflating that steady-state volume, a
// randomized *initial phase offset* -- uniformly distributed in [0, PollingInterval) -- is applied
// exactly once, before the first delay of a given WaitForInstanceStartAsync call. This is a one-time
// cost per call (not repeated per iteration), so it does not change the long-run polling rate, and
// it still never exceeds the historical 1-second worst-case detection latency.
static readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(1);

readonly ShimDurableTaskClientOptions options = Check.NotNull(options);
ShimDurableEntityClient? entities;

Expand Down Expand Up @@ -220,7 +231,7 @@
{
throw new ArgumentException(
$"Invalid orchestration runtime status: '{s}' for deduplication.");
}

Check warning on line 234 in src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs

View workflow job for this annotation

GitHub Actions / build

return status.ConvertToCore();
})
.ToArray();
Expand Down Expand Up @@ -270,6 +281,10 @@
{
Check.NotNullOrEmpty(instanceId);

// A one-time randomized phase offset (see ComputeNextPollingDelay) is applied only to the first
// delay of this call so concurrent waiters desynchronize without increasing steady-state
// polling volume beyond the historical fixed 1-second cadence.
bool isInitialDelay = true;
while (true)
{
OrchestrationMetadata? metadata = await this.GetInstancesAsync(
Expand All @@ -285,7 +300,14 @@
return metadata;
}

await Task.Delay(TimeSpan.FromSeconds(1), cancellation);
// The first delay is a randomized phase offset (bounded by the historical 1-second
// cadence) that desynchronizes concurrent waiters; every delay after that is the fixed
// historical 1-second interval, unjittered, so steady-state polling volume never exceeds
// the historical rate. Either way, the delay never exceeds 1 second, preserving prompt-start
// observation.
TimeSpan delay = ComputeNextPollingDelay(isInitialDelay);
isInitialDelay = false;
await Task.Delay(delay, cancellation);
}
}

Expand Down Expand Up @@ -344,6 +366,32 @@
return newInstanceId;
}

/// <summary>
/// Computes the delay to wait before the next <see cref="WaitForInstanceStartAsync"/> polling attempt.
/// </summary>
/// <param name="isInitialDelay">
/// <see langword="true"/> if this is the first delay computed for a given <see
/// cref="WaitForInstanceStartAsync"/> call; <see langword="false"/> for every subsequent delay in
/// that call.
/// </param>
/// <returns>
/// When <paramref name="isInitialDelay"/> is <see langword="true"/>, a one-time randomized phase
/// offset uniformly distributed in [<see cref="TimeSpan.Zero"/>, <see cref="PollingInterval"/>) that
/// desynchronizes concurrent callers. Otherwise, the fixed <see cref="PollingInterval"/> (1 second),
/// unjittered, so steady-state polling volume never exceeds the historical rate. In both cases the
/// returned delay never exceeds <see cref="PollingInterval"/>, preserving the historical worst-case
/// detection latency for <see cref="WaitForInstanceStartAsync"/>.
/// </returns>
internal static TimeSpan ComputeNextPollingDelay(bool isInitialDelay)
{
if (isInitialDelay)
{
return TimeSpan.FromMilliseconds(PollingInterval.TotalMilliseconds * PollingJitter.NextDouble());
}
Comment thread
berndverst marked this conversation as resolved.
Resolved

return PollingInterval;
}

[return: NotNullIfNotNull("state")]
OrchestrationMetadata? ToMetadata(Core.OrchestrationState? state, bool getInputsAndOutputs)
{
Expand Down Expand Up @@ -449,4 +497,37 @@
}
}
}

/// <summary>
/// A minimal thread-safe random source used to jitter <see cref="WaitForInstanceStartAsync"/> polling
/// delays across concurrent callers. <see cref="System.Random"/> is not thread-safe, and its
/// parameterless constructor can produce correlated sequences when many instances are created around
/// the same tick -- which is exactly the kind of synchronized behavior this jitter is meant to avoid.
/// A single, securely-seeded instance guarded by a lock avoids both issues.
/// </summary>
static class PollingJitter
{
static readonly object SyncRoot = new();
static readonly Random Shared = CreateSeededRandom();

/// <summary>
/// Returns a thread-safe random double in the range [0.0, 1.0).
/// </summary>
/// <returns>A random double in the range [0.0, 1.0).</returns>
public static double NextDouble()
{
lock (SyncRoot)
{
return Shared.NextDouble();
}
}

static Random CreateSeededRandom()
{
byte[] seedBytes = new byte[sizeof(int)];
using RandomNumberGenerator rng = RandomNumberGenerator.Create();
rng.GetBytes(seedBytes);
return new Random(BitConverter.ToInt32(seedBytes, 0));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,118 @@ public async Task WaitForInstanceStart()
Validate(metadata, state2, false);
}

[Fact]
public async Task WaitForInstanceStart_MultiplePendingPolls_EventuallyReturnsTerminalMetadata()
{
// arrange
DateTimeOffset start = DateTimeOffset.UtcNow;
OrchestrationInstance instance = new()
{
InstanceId = Guid.NewGuid().ToString(),
ExecutionId = Guid.NewGuid().ToString(),
};

Core.OrchestrationState pending1 = CreateState("input", start: start);
pending1.OrchestrationInstance = instance;
pending1.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState pending2 = CreateState("input", start: start);
pending2.OrchestrationInstance = instance;
pending2.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState terminal = CreateState("input", start: start);
terminal.OrchestrationInstance = instance;

this.orchestrationClient.SetupSequence(m => m.GetOrchestrationStateAsync(instance.InstanceId, false))
.ReturnsAsync([pending1])
.ReturnsAsync([pending2])
.ReturnsAsync([terminal]);

// act
OrchestrationMetadata metadata = await this.client.WaitForInstanceStartAsync(
instance.InstanceId, false, default);

Comment thread
berndverst marked this conversation as resolved.
Resolved
// assert -- multiple polling iterations (exercising the polling loop, including its one-time
// initial phase offset followed by fixed-interval delays) still converge on the terminal state
// once observed.
this.orchestrationClient.Verify(
m => m.GetOrchestrationStateAsync(instance.InstanceId, false), Times.Exactly(3));
Validate(metadata, terminal, false);
}

[Fact]
public void ComputeNextPollingDelay_InitialDelay_NeverExceedsHistoricalOneSecondCadence()
{
// assert -- across many samples, the randomized initial-phase-offset delay must always be
// non-negative and never exceed the historical 1-second polling cadence, enforcing the max
// detection-latency contract for the very first delay of a WaitForInstanceStartAsync call.
for (int i = 0; i < 1000; i++)
{
TimeSpan delay = ShimDurableTaskClient.ComputeNextPollingDelay(isInitialDelay: true);
delay.Should().BeGreaterThanOrEqualTo(TimeSpan.Zero);
delay.Should().BeLessThan(TimeSpan.FromSeconds(1));
}
}

[Fact]
public void ComputeNextPollingDelay_SteadyState_ReturnsFixedHistoricalIntervalWithNoJitter()
{
// assert -- every delay after the initial phase offset must be exactly the fixed historical
// 1-second cadence with no jitter and no growth, so steady-state polling volume never exceeds
// (or falls below) the historical rate. This guards against a regression to either unconstrained
// backoff growth or a per-iteration jitter reduction that would increase polling frequency.
for (int i = 0; i < 100; i++)
{
TimeSpan delay = ShimDurableTaskClient.ComputeNextPollingDelay(isInitialDelay: false);
delay.Should().Be(TimeSpan.FromSeconds(1));
}
}

[Fact]
public async Task WaitForInstanceStart_InstanceNotFound_ThrowsImmediatelyWithoutPolling()
{
// arrange
string instanceId = Guid.NewGuid().ToString();
this.orchestrationClient.Setup(m => m.GetOrchestrationStateAsync(instanceId, false))
.ReturnsAsync([]);

// act
Func<Task> act = () => this.client.WaitForInstanceStartAsync(instanceId, false, default);

// assert -- not-found behavior is preserved: no retry/polling delay before throwing.
await act.Should().ThrowExactlyAsync<InvalidOperationException>()
.WithMessage($"Orchestration with instanceId '{instanceId}' does not exist");
this.orchestrationClient.Verify(
m => m.GetOrchestrationStateAsync(instanceId, false), Times.Once);
}

[Fact]
public async Task WaitForInstanceStart_CancelledDuringPollingDelay_ThrowsTaskCanceledException()
Comment thread
berndverst marked this conversation as resolved.
Resolved
{
// arrange
DateTimeOffset start = DateTimeOffset.UtcNow;
OrchestrationInstance instance = new()
{
InstanceId = Guid.NewGuid().ToString(),
ExecutionId = Guid.NewGuid().ToString(),
};

Core.OrchestrationState pending = CreateState("input", start: start);
pending.OrchestrationInstance = instance;
pending.OrchestrationStatus = Core.OrchestrationStatus.Pending;

this.orchestrationClient.Setup(m => m.GetOrchestrationStateAsync(instance.InstanceId, false))
.ReturnsAsync([pending]);

using CancellationTokenSource cts = new();

// act -- cancel shortly after the first (immediate) poll so cancellation fires while the loop is
// awaiting the polling delay rather than before any polling occurs.
cts.CancelAfter(TimeSpan.FromMilliseconds(50));
Func<Task> act = () => this.client.WaitForInstanceStartAsync(instance.InstanceId, false, cts.Token);

// assert
await act.Should().ThrowAsync<OperationCanceledException>();
}

[Fact]
public Task ScheduleNewOrchestrationInstance_IdGenerated_NoInput()
=> this.RunScheduleNewOrchestrationInstanceAsync("test", null, null);
Expand Down
Loading