Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Security.Cryptography;
using DurableTask.Core;
using DurableTask.Core.Exceptions;
using DurableTask.Core.History;
Expand All @@ -27,6 +28,14 @@ namespace Microsoft.DurableTask.Client.OrchestrationServiceClientShim;
/// <param name="options">The client options.</param>
class ShimDurableTaskClient(string name, ShimDurableTaskClientOptions options) : DurableTaskClient(name)
{
// Polling parameters for WaitForInstanceStartAsync. PollingInterval matches the historical fixed
// 1-second polling cadence. Callers rely on prompt (<= 1 second) observation of a status
// transition, so jitter is only ever applied *downward* from this value -- never grown beyond it
// -- to desynchronize concurrent callers (avoiding synchronized polling bursts against the
// backend) without regressing the historical worst-case detection latency.
const double PollingJitterFactor = 0.2;
static readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(1);

readonly ShimDurableTaskClientOptions options = Check.NotNull(options);
ShimDurableEntityClient? entities;

Expand Down Expand Up @@ -285,7 +294,12 @@ public override async Task<OrchestrationMetadata> WaitForInstanceStartAsync(
return metadata;
}

await Task.Delay(TimeSpan.FromSeconds(1), cancellation);
// Poll with a delay that is jittered *downward* from the historical 1-second cadence.
// This desynchronizes concurrent waiters (avoiding synchronized polling bursts against the
// backend) while guaranteeing the worst-case detection latency for a status transition
// never exceeds the historical 1-second cadence -- preserving prompt-start observation.
TimeSpan delay = ComputeNextPollingDelay();
await Task.Delay(delay, cancellation);
Comment thread
berndverst marked this conversation as resolved.
Resolved
}
}

Expand Down Expand Up @@ -344,6 +358,22 @@ await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync(
return newInstanceId;
}

/// <summary>
/// Computes the delay to wait before the next <see cref="WaitForInstanceStartAsync"/> polling attempt.
/// </summary>
/// <returns>
/// A delay in the range [<see cref="PollingInterval"/> * (1 - <see cref="PollingJitterFactor"/>),
/// <see cref="PollingInterval"/>]. Jitter only ever reduces the delay, so the returned value never
/// exceeds <see cref="PollingInterval"/> -- preserving the historical worst-case detection latency
/// for <see cref="WaitForInstanceStartAsync"/> -- while still desynchronizing concurrent callers.
Comment thread
berndverst marked this conversation as resolved.
Resolved
/// </returns>
internal static TimeSpan ComputeNextPollingDelay()
{
double reduction = PollingJitterFactor * PollingJitter.NextDouble();
double delayMs = PollingInterval.TotalMilliseconds * (1.0 - reduction);
return TimeSpan.FromMilliseconds(delayMs);
}

[return: NotNullIfNotNull("state")]
OrchestrationMetadata? ToMetadata(Core.OrchestrationState? state, bool getInputsAndOutputs)
{
Expand Down Expand Up @@ -449,4 +479,37 @@ async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync(
}
}
}

/// <summary>
/// A minimal thread-safe random source used to jitter <see cref="WaitForInstanceStartAsync"/> polling
/// delays across concurrent callers. <see cref="System.Random"/> is not thread-safe, and its
/// parameterless constructor can produce correlated sequences when many instances are created around
/// the same tick -- which is exactly the kind of synchronized behavior this jitter is meant to avoid.
/// A single, securely-seeded instance guarded by a lock avoids both issues.
/// </summary>
static class PollingJitter
{
static readonly object SyncRoot = new();
static readonly Random Shared = CreateSeededRandom();

/// <summary>
/// Returns a thread-safe random double in the range [0.0, 1.0).
/// </summary>
/// <returns>A random double in the range [0.0, 1.0).</returns>
public static double NextDouble()
{
lock (SyncRoot)
{
return Shared.NextDouble();
}
}

static Random CreateSeededRandom()
{
byte[] seedBytes = new byte[sizeof(int)];
using RandomNumberGenerator rng = RandomNumberGenerator.Create();
rng.GetBytes(seedBytes);
return new Random(BitConverter.ToInt32(seedBytes, 0));
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Diagnostics;
using DurableTask.Core;
using DurableTask.Core.Entities;
using DurableTask.Core.Exceptions;
Expand Down Expand Up @@ -300,6 +301,145 @@ public async Task WaitForInstanceStart()
Validate(metadata, state2, false);
}

[Fact]
public async Task WaitForInstanceStart_MultiplePendingPolls_EventuallyReturnsTerminalMetadata()
{
// arrange
DateTimeOffset start = DateTimeOffset.UtcNow;
OrchestrationInstance instance = new()
{
InstanceId = Guid.NewGuid().ToString(),
ExecutionId = Guid.NewGuid().ToString(),
};

Core.OrchestrationState pending1 = CreateState("input", start: start);
pending1.OrchestrationInstance = instance;
pending1.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState pending2 = CreateState("input", start: start);
pending2.OrchestrationInstance = instance;
pending2.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState terminal = CreateState("input", start: start);
terminal.OrchestrationInstance = instance;

this.orchestrationClient.SetupSequence(m => m.GetOrchestrationStateAsync(instance.InstanceId, false))
.ReturnsAsync([pending1])
.ReturnsAsync([pending2])
.ReturnsAsync([terminal]);

// act
OrchestrationMetadata metadata = await this.client.WaitForInstanceStartAsync(
instance.InstanceId, false, default);

Comment thread
berndverst marked this conversation as resolved.
Resolved
// assert -- multiple polling iterations (exercising the jittered polling loop) still converge
// on the terminal state once observed.
this.orchestrationClient.Verify(
m => m.GetOrchestrationStateAsync(instance.InstanceId, false), Times.Exactly(3));
Validate(metadata, terminal, false);
}

[Fact]
public async Task WaitForInstanceStart_RepeatedPendingObservations_HonorsMaxDetectionLatencyContract()
{
// arrange
DateTimeOffset start = DateTimeOffset.UtcNow;
OrchestrationInstance instance = new()
{
InstanceId = Guid.NewGuid().ToString(),
ExecutionId = Guid.NewGuid().ToString(),
};

Core.OrchestrationState pending1 = CreateState("input", start: start);
pending1.OrchestrationInstance = instance;
pending1.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState pending2 = CreateState("input", start: start);
pending2.OrchestrationInstance = instance;
pending2.OrchestrationStatus = Core.OrchestrationStatus.Pending;
Core.OrchestrationState terminal = CreateState("input", start: start);
terminal.OrchestrationInstance = instance;

this.orchestrationClient.SetupSequence(m => m.GetOrchestrationStateAsync(instance.InstanceId, false))
.ReturnsAsync([pending1])
.ReturnsAsync([pending2])
.ReturnsAsync([terminal]);

Stopwatch stopwatch = Stopwatch.StartNew();

// act
OrchestrationMetadata metadata = await this.client.WaitForInstanceStartAsync(
instance.InstanceId, false, default);

stopwatch.Stop();

// assert -- two "still pending" observations occur before the terminal state is returned, so
// exactly two polling delays are incurred. Each delay is bounded by the historical 1-second
// detection cadence (jitter only ever reduces it), so the worst-case total wait here is ~2
// seconds. This guards against a regression -- like unconstrained backoff growth -- that would
// push per-poll delays past the historical cadence and violate prompt-start observation.
stopwatch.Elapsed.Should().BeLessThan(TimeSpan.FromSeconds(2.5));
Validate(metadata, terminal, false);
}

[Fact]
public void ComputeNextPollingDelay_NeverExceedsHistoricalOneSecondCadence()
{
// assert -- across many samples, the jittered delay must never exceed the historical 1-second
// polling cadence (jitter only ever reduces the delay), enforcing the max detection-latency
// contract for WaitForInstanceStartAsync, and must always be a non-negative, finite delay.
for (int i = 0; i < 1000; i++)
{
TimeSpan delay = ShimDurableTaskClient.ComputeNextPollingDelay();
delay.Should().BeGreaterThan(TimeSpan.Zero);
delay.Should().BeLessThanOrEqualTo(TimeSpan.FromSeconds(1));
}
}

[Fact]
public async Task WaitForInstanceStart_InstanceNotFound_ThrowsImmediatelyWithoutPolling()
{
// arrange
string instanceId = Guid.NewGuid().ToString();
this.orchestrationClient.Setup(m => m.GetOrchestrationStateAsync(instanceId, false))
.ReturnsAsync([]);

// act
Func<Task> act = () => this.client.WaitForInstanceStartAsync(instanceId, false, default);

// assert -- not-found behavior is preserved: no retry/polling delay before throwing.
await act.Should().ThrowExactlyAsync<InvalidOperationException>()
.WithMessage($"Orchestration with instanceId '{instanceId}' does not exist");
this.orchestrationClient.Verify(
m => m.GetOrchestrationStateAsync(instanceId, false), Times.Once);
}

[Fact]
public async Task WaitForInstanceStart_CancelledDuringPollingDelay_ThrowsTaskCanceledException()
Comment thread
berndverst marked this conversation as resolved.
Resolved
{
// arrange
DateTimeOffset start = DateTimeOffset.UtcNow;
OrchestrationInstance instance = new()
{
InstanceId = Guid.NewGuid().ToString(),
ExecutionId = Guid.NewGuid().ToString(),
};

Core.OrchestrationState pending = CreateState("input", start: start);
pending.OrchestrationInstance = instance;
pending.OrchestrationStatus = Core.OrchestrationStatus.Pending;

this.orchestrationClient.Setup(m => m.GetOrchestrationStateAsync(instance.InstanceId, false))
.ReturnsAsync([pending]);

using CancellationTokenSource cts = new();

// act -- cancel shortly after the first (immediate) poll so cancellation fires while the loop is
// awaiting the jittered polling delay rather than before any polling occurs.
cts.CancelAfter(TimeSpan.FromMilliseconds(50));
Func<Task> act = () => this.client.WaitForInstanceStartAsync(instance.InstanceId, false, cts.Token);

// assert
await act.Should().ThrowAsync<OperationCanceledException>();
}

[Fact]
public Task ScheduleNewOrchestrationInstance_IdGenerated_NoInput()
=> this.RunScheduleNewOrchestrationInstanceAsync("test", null, null);
Expand Down
Loading