Repository navigation
Reduce shim instance-start polling load with bounded jitter #779
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Bernd Verst (berndverst)
merged 7 commits into
main
from
berndverst-reduce-shim-polling-load
Jul 27, 2026
+307
−33
Merged
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
a74caed
Reduce shim instance-start polling load with jitter and backoff (#776)
e2d659f
Fix: cap WaitForInstanceStartAsync polling delay at historical 1s cad…
500481c
Fix: use one-time initial phase offset instead of per-poll jitter red…
78d527f
Fix: make cancellation test deterministic via internal delay seam
3a74e2d
Make cancellation test fully prove token propagation into DelayAsync
c8b66aa
Stabilize PerItem_HeartbeatReset_KeepsTimerAlive against CI schedulin…
8779f98
Make WorkItemStreamConsumer per-item timer reset test fully determini…
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
| using System.Diagnostics; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Globalization; | ||
| using System.Security.Cryptography; | ||
| using DurableTask.Core; | ||
| using DurableTask.Core.Exceptions; | ||
| using DurableTask.Core.History; | ||
|
|
@@ -27,6 +28,16 @@ | |
| /// <param name="options">The client options.</param> | ||
| class ShimDurableTaskClient(string name, ShimDurableTaskClientOptions options) : DurableTaskClient(name) | ||
| { | ||
| // Polling parameters for WaitForInstanceStartAsync. PollingInterval matches the historical fixed | ||
| // 1-second polling cadence and is used, unjittered, for every steady-state delay -- so long-run | ||
| // polling volume never exceeds the historical rate. To desynchronize concurrent callers (avoiding | ||
| // synchronized polling bursts against the backend) without inflating that steady-state volume, a | ||
| // randomized *initial phase offset* -- uniformly distributed in [0, PollingInterval) -- is applied | ||
| // exactly once, before the first delay of a given WaitForInstanceStartAsync call. This is a one-time | ||
| // cost per call (not repeated per iteration), so it does not change the long-run polling rate, and | ||
| // it still never exceeds the historical 1-second worst-case detection latency. | ||
| static readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(1); | ||
|
|
||
| readonly ShimDurableTaskClientOptions options = Check.NotNull(options); | ||
| ShimDurableEntityClient? entities; | ||
|
|
||
|
|
@@ -220,7 +231,7 @@ | |
| { | ||
| throw new ArgumentException( | ||
| $"Invalid orchestration runtime status: '{s}' for deduplication."); | ||
| } | ||
|
Check warning on line 234 in src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs
|
||
| return status.ConvertToCore(); | ||
| }) | ||
| .ToArray(); | ||
|
|
@@ -270,6 +281,10 @@ | |
| { | ||
| Check.NotNullOrEmpty(instanceId); | ||
|
|
||
| // A one-time randomized phase offset (see ComputeNextPollingDelay) is applied only to the first | ||
| // delay of this call so concurrent waiters desynchronize without increasing steady-state | ||
| // polling volume beyond the historical fixed 1-second cadence. | ||
| bool isInitialDelay = true; | ||
| while (true) | ||
| { | ||
| OrchestrationMetadata? metadata = await this.GetInstancesAsync( | ||
|
|
@@ -285,7 +300,14 @@ | |
| return metadata; | ||
| } | ||
|
|
||
| await Task.Delay(TimeSpan.FromSeconds(1), cancellation); | ||
| // The first delay is a randomized phase offset (bounded by the historical 1-second | ||
| // cadence) that desynchronizes concurrent waiters; every delay after that is the fixed | ||
| // historical 1-second interval, unjittered, so steady-state polling volume never exceeds | ||
| // the historical rate. Either way, the delay never exceeds 1 second, preserving prompt-start | ||
| // observation. | ||
| TimeSpan delay = ComputeNextPollingDelay(isInitialDelay); | ||
| isInitialDelay = false; | ||
| await Task.Delay(delay, cancellation); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -344,6 +366,32 @@ | |
| return newInstanceId; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Computes the delay to wait before the next <see cref="WaitForInstanceStartAsync"/> polling attempt. | ||
| /// </summary> | ||
| /// <param name="isInitialDelay"> | ||
| /// <see langword="true"/> if this is the first delay computed for a given <see | ||
| /// cref="WaitForInstanceStartAsync"/> call; <see langword="false"/> for every subsequent delay in | ||
| /// that call. | ||
| /// </param> | ||
| /// <returns> | ||
| /// When <paramref name="isInitialDelay"/> is <see langword="true"/>, a one-time randomized phase | ||
| /// offset uniformly distributed in [<see cref="TimeSpan.Zero"/>, <see cref="PollingInterval"/>) that | ||
| /// desynchronizes concurrent callers. Otherwise, the fixed <see cref="PollingInterval"/> (1 second), | ||
| /// unjittered, so steady-state polling volume never exceeds the historical rate. In both cases the | ||
| /// returned delay never exceeds <see cref="PollingInterval"/>, preserving the historical worst-case | ||
| /// detection latency for <see cref="WaitForInstanceStartAsync"/>. | ||
| /// </returns> | ||
| internal static TimeSpan ComputeNextPollingDelay(bool isInitialDelay) | ||
| { | ||
| if (isInitialDelay) | ||
| { | ||
| return TimeSpan.FromMilliseconds(PollingInterval.TotalMilliseconds * PollingJitter.NextDouble()); | ||
| } | ||
|
|
||
| return PollingInterval; | ||
| } | ||
|
|
||
| [return: NotNullIfNotNull("state")] | ||
| OrchestrationMetadata? ToMetadata(Core.OrchestrationState? state, bool getInputsAndOutputs) | ||
| { | ||
|
|
@@ -449,4 +497,37 @@ | |
| } | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// A minimal thread-safe random source used to jitter <see cref="WaitForInstanceStartAsync"/> polling | ||
| /// delays across concurrent callers. <see cref="System.Random"/> is not thread-safe, and its | ||
| /// parameterless constructor can produce correlated sequences when many instances are created around | ||
| /// the same tick -- which is exactly the kind of synchronized behavior this jitter is meant to avoid. | ||
| /// A single, securely-seeded instance guarded by a lock avoids both issues. | ||
| /// </summary> | ||
| static class PollingJitter | ||
| { | ||
| static readonly object SyncRoot = new(); | ||
| static readonly Random Shared = CreateSeededRandom(); | ||
|
|
||
| /// <summary> | ||
| /// Returns a thread-safe random double in the range [0.0, 1.0). | ||
| /// </summary> | ||
| /// <returns>A random double in the range [0.0, 1.0).</returns> | ||
| public static double NextDouble() | ||
| { | ||
| lock (SyncRoot) | ||
| { | ||
| return Shared.NextDouble(); | ||
| } | ||
| } | ||
|
|
||
| static Random CreateSeededRandom() | ||
| { | ||
| byte[] seedBytes = new byte[sizeof(int)]; | ||
| using RandomNumberGenerator rng = RandomNumberGenerator.Create(); | ||
| rng.GetBytes(seedBytes); | ||
| return new Random(BitConverter.ToInt32(seedBytes, 0)); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.