From 5b36e2bd0efea5c350c96bf040332d71114bef55 Mon Sep 17 00:00:00 2001 From: Oisin G Date: Mon, 8 Nov 2021 19:32:17 -0500 Subject: [PATCH 01/30] add Reset() method to IStreamQueueCheckpointer --- .../DynamoDBStreamQueueCheckpointer.cs | 7 ++ .../NoOpCheckpointer.cs | 13 ++- .../AzureTableStreamQueueCheckpointer.cs | 54 +++++++++++ .../EventHub/EventHubAdapterReceiver.cs | 31 ++++++ .../Streams/EventHub/EventHubCheckpointer.cs | 7 ++ .../GrainStreamQueueCheckpointer.cs | 72 ++++++++++++++ .../IStreamQueueCheckpointer.cs | 16 ++++ .../Orleans.Streaming.EventHubs.cs | 15 +++ .../Orleans.Streaming/Orleans.Streaming.cs | 8 ++ .../EventHubCheckpointerTests.cs | 95 ++++++++++++++++++- .../StreamQueueCheckpointerTests.cs | 14 +++ 11 files changed, 328 insertions(+), 4 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 56ba751fb24..93319477a39 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -51,6 +51,13 @@ internal static async Task> Create( /// public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index be76b2e5df1..9d51d6f4452 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -1,5 +1,6 @@ using Orleans.Streams; using System; +using System.Threading; using System.Threading.Tasks; namespace Orleans.Streaming.EventHubs.Testing @@ -22,7 +23,7 @@ public Task> Create(string partition) } /// /// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, - /// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. + /// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. /// So a checkpointer with no ops should be enough. /// public class NoOpCheckpointer : IStreamQueueCheckpointer @@ -40,6 +41,16 @@ public Task Load() { return Task.FromResult(EventHubConstants.StartOfStream); } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Task.CompletedTask; + + /// + public Task Reset(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + /// /// Ignores a checkpoint update because generated-event streams do not persist checkpoints. /// diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 7c69c12b365..20c767c7b60 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -127,6 +127,53 @@ public async Task Load() return checkpoint; } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public async Task Reset(CancellationToken cancellationToken) + { + Task resetTask; + lock (_lock) + { + var inProgressSave = _inProgressSave; + _latestCheckpoint = string.Empty; + _throttleSavesUntilUtc = DateTime.MaxValue; + resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); + } + + try + { + await resetTask; + } + catch + { + lock (_lock) + { + if (ReferenceEquals(resetTask, _inProgressSave)) + { + _latestCheckpoint = _persistedCheckpoint; + _throttleSavesUntilUtc = null; + _inProgressSave = Task.CompletedTask; + } + } + + throw; + } + + lock (_lock) + { + if (ReferenceEquals(resetTask, _inProgressSave)) + { + _latestCheckpoint = string.Empty; + _persistedCheckpoint = string.Empty; + _throttleSavesUntilUtc = null; + _inProgressSave = Task.CompletedTask; + } + } + } + /// public void Update(string offset, DateTime utcNow) { @@ -213,6 +260,13 @@ private async Task Save(string checkpoint) } } + private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) + { + await inProgressSave.WaitAsync(cancellationToken); + _entity.Offset = string.Empty; + await _dataManager.UpsertTableEntryAsync(_entity); + } + [LoggerMessage( Level = LogLevel.Information, Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 3489a9a8d12..d56eba71841 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -208,6 +208,11 @@ public async Task> GetQueueMessagesAsync( watch.Stop(); this.monitor?.TrackRead(false, watch.Elapsed, ex); LogWarningFailedToReadFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition, ex); + + if (ex is ArgumentException && this.checkpointer?.CheckpointExists == true) + { + await ResetReceiver(cancellationToken); + } throw; } @@ -234,6 +239,32 @@ public async Task> GetQueueMessagesAsync( return batches; } + private async Task ResetReceiver(CancellationToken cancellationToken) + { + var checkpointer = Interlocked.Exchange(ref this.checkpointer, null); + var receiver = Interlocked.Exchange(ref this.receiver, null); + var cache = Interlocked.Exchange(ref this.cache, null); + + try + { + await checkpointer!.Reset(cancellationToken); + } + finally + { + try + { + cache?.Dispose(); + } + finally + { + if (receiver is not null) + { + await receiver.CloseAsync(cancellationToken); + } + } + } + } + public void AddToCache(IList messages) { // do nothing, we add data directly into cache. No need for agent involvement diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 1c13fac1227..e6e8a60c52d 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -110,6 +110,13 @@ public async Task Load(CancellationToken cancellationToken) return CheckpointExists ? checkpoint : EventHubConstants.StartOfStream; } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); + /// /// Updates the checkpoint. This is a best effort. It does not always update the checkpoint. /// The latest offset is always tracked in memory so that can persist it on shutdown. diff --git a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs index e1ae927ae3e..229a3eba02a 100644 --- a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs @@ -173,6 +173,53 @@ internal static string GetConfiguredStorageProviderName(ReadOnlySpan grain /// public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public async Task Reset(CancellationToken cancellationToken) + { + Task resetTask; + lock (_lock) + { + var inProgressSave = _inProgressSave; + _latestCheckpoint = string.Empty; + _throttleSavesUntilUtc = DateTime.MaxValue; + resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); + } + + try + { + await resetTask; + } + catch + { + lock (_lock) + { + if (ReferenceEquals(resetTask, _inProgressSave)) + { + _latestCheckpoint = _persistedCheckpoint; + _throttleSavesUntilUtc = null; + _inProgressSave = Task.CompletedTask; + } + } + + throw; + } + + lock (_lock) + { + if (ReferenceEquals(resetTask, _inProgressSave)) + { + _latestCheckpoint = string.Empty; + _persistedCheckpoint = string.Empty; + _throttleSavesUntilUtc = null; + _inProgressSave = Task.CompletedTask; + } + } + } + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) @@ -206,5 +253,30 @@ public async ValueTask Update( return new(persistedCheckpoint, persistedCheckpoint); } } + + private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) + { + await inProgressSave.WaitAsync(cancellationToken); + + string expectedCheckpoint; + lock (_lock) + { + expectedCheckpoint = _persistedCheckpoint; + } + + while (true) + { + var persistedCheckpoint = await _grain.Update( + string.Empty, + expectedCheckpoint, + cancellationToken); + if (string.IsNullOrEmpty(persistedCheckpoint)) + { + return; + } + + expectedCheckpoint = persistedCheckpoint; + } + } } } diff --git a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs index be0c7c013c6..84b0f894de0 100644 --- a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs @@ -57,6 +57,22 @@ public interface IStreamQueueCheckpointer Task Load(CancellationToken cancellationToken) => Load(); #pragma warning restore CS0618 + /// + /// Clears the persisted checkpoint. + /// + /// A representing the operation. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task Reset() => throw new NotSupportedException("This checkpointer does not support resetting checkpoints."); + + /// + /// Clears the persisted checkpoint. + /// + /// The cancellation token. + /// A representing the operation. +#pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. + Task Reset(CancellationToken cancellationToken) => Reset(); +#pragma warning restore CS0618 + /// /// Updates the checkpoint. /// diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index ab34b90eaa2..7bd23fad022 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -377,6 +377,11 @@ internal EventHubCheckpointer() { } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] @@ -745,6 +750,11 @@ public partial class NoOpCheckpointer : Streams.IStreamQueueCheckpointer public System.Threading.Tasks.Task Load() { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow) { } } @@ -784,6 +794,11 @@ internal AzureTableStreamQueueCheckpointer() { } public System.Threading.Tasks.Task Load() { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow) { } } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 609eca4357d..90c0d2d7d9c 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -1557,6 +1557,11 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain) { } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] @@ -1854,6 +1859,9 @@ public partial interface IStreamQueueCheckpointer [System.Obsolete("Use the overload which accepts a CancellationToken.")] System.Threading.Tasks.Task Load(); System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken); + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + System.Threading.Tasks.Task Reset(); + System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken); void Update(TCheckpoint offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken); [System.Obsolete("Use the overload which accepts a CancellationToken.")] void Update(TCheckpoint offset, System.DateTime utcNow); diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 97846df0868..f8c7d078cb3 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -25,15 +25,24 @@ public class EventHubCheckpointerTests /// private class TestCheckpointer : IStreamQueueCheckpointer { - public bool CheckpointExists { get; init; } = true; + public bool CheckpointExists { get; set; } = true; public string LoadedOffset { get; init; } = EventHubConstants.StartOfStream; public string? LastOffset { get; private set; } public int UpdateCount { get; private set; } public string? FlushedOffset { get; private set; } public int FlushCount { get; private set; } + public int ResetCount { get; private set; } public Task Load() => Task.FromResult(LoadedOffset); + public virtual Task Reset(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + CheckpointExists = false; + ResetCount++; + return Task.CompletedTask; + } + public void Update(string offset, DateTime utcNow) { if (LastOffset is not null @@ -296,6 +305,20 @@ public Task> ReceiveAsync(int maxCount, TimeSpan waitTime public Task CloseAsync() => Task.CompletedTask; } + private sealed class InvalidOffsetEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => throw new ArgumentException("The supplied offset is invalid."); + + public Task CloseAsync() + { + CloseCount++; + return Task.CompletedTask; + } + } + private sealed class CancellableEventHubReceiver : IEventHubReceiver { public TaskCompletionSource ReceiveStarted { get; } = new( @@ -354,7 +377,8 @@ private static async Task CreateReceiver( TestCheckpointer checkpointer, TestEventHubQueueCache? cache = null, IEventHubReceiver? eventHubReceiver = null, - Action? onReceiverCreated = null) + Action? onReceiverCreated = null, + Func? receiverFactory = null) { var settings = new EventHubPartitionSettings { @@ -385,7 +409,7 @@ private static async Task CreateReceiver( eventHubReceiverFactory: (_, offset, _) => { onReceiverCreated?.Invoke(offset); - return eventHubReceiver ?? new TestEventHubReceiver(); + return receiverFactory?.Invoke(offset) ?? eventHubReceiver ?? new TestEventHubReceiver(); }); await receiver.Initialize(TimeSpan.FromSeconds(5)); @@ -426,6 +450,52 @@ public async Task GetQueueMessagesAsync_TreatsNullReceiverResultAsEmpty() Assert.Equal(0, cache.AddCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenCheckpointOffsetIsInvalid_ResetsAndRecreatesReceiver() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var cache = new TestEventHubQueueCache(); + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + cache, + onReceiverCreated: offsets.Add, + receiverFactory: offset => offset == "123" ? invalidReceiver : new TestEventHubReceiver()); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(1, checkpointer.ResetCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(1, cache.DisposeCount); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(["123", EventHubConstants.StartOfStream], offsets); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenNoCheckpointExists_DoesNotResetArgumentFailure() + { + var checkpointer = new TestCheckpointer + { + CheckpointExists = false, + LoadedOffset = string.Empty, + }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, eventHubReceiver: invalidReceiver); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal(0, checkpointer.ResetCount); + Assert.Equal(0, invalidReceiver.CloseCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task GetQueueMessagesAsync_ForwardsCancellationToken() @@ -658,6 +728,25 @@ public async Task EventHubCompatibilityWrapper_PreservesLoadSemantics( Assert.Equal(expected, await checkpointer.Load(TestContext.Current.CancellationToken)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task EventHubCompatibilityWrapper_DelegatesReset() + { + var inner = new TestCheckpointer(); + var constructor = typeof(EventHubCheckpointer).GetConstructor( + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [typeof(IStreamQueueCheckpointer)], + modifiers: null); + Assert.NotNull(constructor); + var checkpointer = (EventHubCheckpointer)constructor.Invoke([inner]); + + await checkpointer.Reset(CancellationToken.None); + + Assert.Equal(1, inner.ResetCount); + Assert.False(checkpointer.CheckpointExists); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task SingleSubscription_CheckpointsProcessedOffset() diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index 032005ed111..28fc9833e80 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -60,6 +60,20 @@ public async Task Load_WhenCheckpointExists_ReturnsPersistedCheckpoint() Assert.Empty(store.WriteAttempts); } + [Fact] + public async Task Reset_ClearsPersistedCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + + await checkpointer.Reset(CancellationToken.None); + + Assert.False(checkpointer.CheckpointExists); + Assert.Equal(NoCheckpoint, await checkpointer.Load(CancellationToken.None)); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.Equal([NoCheckpoint], store.WriteAttempts); + Assert.Equal([NoCheckpoint], store.CompletedWrites); + } + [Fact] public async Task Update_PersistsCheckpoint() { From 039c4e8fa0cd8c9ff21d9140b41d23f95c7cda93 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 00:29:40 -0700 Subject: [PATCH 02/30] fix(streaming): harden stale checkpoint recovery --- .../DynamoDBStreamQueueCheckpointer.cs | 123 +++--- .../NoOpCheckpointer.cs | 81 ++-- .../AzureTableStreamQueueCheckpointer.cs | 416 +++++++++--------- .../Streams/EventHub/EventHubCheckpointer.cs | 241 +++++----- .../Shared/Storage/AzureTableDataManager.cs | 13 +- .../IStreamQueueCheckpointer.cs | 151 ++++--- .../StreamQueueCheckpointerTests.cs | 41 ++ 7 files changed, 544 insertions(+), 522 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 93319477a39..4297dd0a647 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -4,81 +4,80 @@ using Orleans.Configuration; using Orleans.Streams; -namespace Orleans.Streaming.Kinesis +namespace Orleans.Streaming.Kinesis; + +/// +/// Persists stream queue checkpoints in DynamoDB. +/// +internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer { - /// - /// Persists stream queue checkpoints in DynamoDB. - /// - internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer - { - private readonly GrainStreamQueueCheckpointer _inner; + private readonly GrainStreamQueueCheckpointer _inner; - internal DynamoDBStreamQueueCheckpointer( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options) - { - _inner = new GrainStreamQueueCheckpointer( - new StreamCheckpointStoreAdapter(store), - new GrainStreamQueueCheckpointerOptions - { - CheckpointComparer = StreamCheckpointComparers.Numeric, - PersistInterval = options.PersistInterval, - }); - } + internal DynamoDBStreamQueueCheckpointer( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options) + { + _inner = new GrainStreamQueueCheckpointer( + new StreamCheckpointStoreAdapter(store), + new GrainStreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = options.PersistInterval, + }); + } - /// - public bool CheckpointExists => _inner.CheckpointExists; + /// + public bool CheckpointExists => _inner.CheckpointExists; - internal static async Task> Create( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options) - => await Create(store, options, CancellationToken.None); + internal static async Task> Create( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options) + => await Create(store, options, CancellationToken.None); - internal static async Task> Create( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options, - CancellationToken cancellationToken) - { - var result = new DynamoDBStreamQueueCheckpointer(store, options); - _ = await result.Load(cancellationToken); - return result; - } + internal static async Task> Create( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options, + CancellationToken cancellationToken) + { + var result = new DynamoDBStreamQueueCheckpointer(store, options); + _ = await result.Load(cancellationToken); + return result; + } - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Load() => Load(CancellationToken.None); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); - /// - public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); + /// + public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Reset() => Reset(CancellationToken.None); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); - /// - public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public void Update(string offset, DateTime utcNow) - => Update(offset, utcNow, CancellationToken.None); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); - /// - public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) - => _inner.Update(offset, utcNow, cancellationToken); + /// + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); - /// - public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + /// + public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); - private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain - { - public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); + private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain + { + public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); - public ValueTask Update( - string checkpoint, - string expectedCheckpoint, - CancellationToken cancellationToken) - => store.Update(checkpoint, expectedCheckpoint, cancellationToken); - } + public ValueTask Update( + string checkpoint, + string expectedCheckpoint, + CancellationToken cancellationToken) + => store.Update(checkpoint, expectedCheckpoint, cancellationToken); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index 9d51d6f4452..2f422cb6101 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -1,63 +1,42 @@ -using Orleans.Streams; -using System; +using System; using System.Threading; using System.Threading.Tasks; +using Orleans.Streams; + +namespace Orleans.Streaming.EventHubs.Testing; -namespace Orleans.Streaming.EventHubs.Testing +public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory { - /// - /// Creates checkpointers which keep no persistent checkpoint state. - /// - public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory + public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); + public Task> Create(string partition) { - /// - /// Gets the shared factory instance. - /// - public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); - - /// - public Task> Create(string partition) - { - return Task.FromResult>(NoOpCheckpointer.Instance); - } + return Task.FromResult>(NoOpCheckpointer.Instance); } - /// - /// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, - /// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. - /// So a checkpointer with no ops should be enough. - /// - public class NoOpCheckpointer : IStreamQueueCheckpointer - { - /// - /// Gets the shared checkpointer instance. - /// - public static NoOpCheckpointer Instance = new NoOpCheckpointer(); +} +/// +/// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, +/// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. +/// So a checkpointer with no ops should be enough. +/// +public class NoOpCheckpointer : IStreamQueueCheckpointer +{ + public static NoOpCheckpointer Instance = new NoOpCheckpointer(); - /// - public bool CheckpointExists => true; + public bool CheckpointExists => true; + public Task Load() + { + return Task.FromResult(EventHubConstants.StartOfStream); + } - /// - public Task Load() - { - return Task.FromResult(EventHubConstants.StartOfStream); - } - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Reset() => Task.CompletedTask; + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Task.CompletedTask; - /// - public Task Reset(CancellationToken cancellationToken) - => cancellationToken.IsCancellationRequested - ? Task.FromCanceled(cancellationToken) - : Task.CompletedTask; + public Task Reset(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; - /// - /// Ignores a checkpoint update because generated-event streams do not persist checkpoints. - /// - /// The ignored offset. - /// The ignored update time. - public void Update(string offset, DateTime utcNow) - { - } + public void Update(string offset, DateTime utcNow) + { } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 20c767c7b60..0f87be78966 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -6,274 +6,286 @@ using Orleans.Configuration; using Orleans.Streaming.EventHubs; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Persists stream queue checkpoints using Azure Table Storage. +/// +public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer { - /// - /// Persists stream queue checkpoints using Azure Table Storage. - /// - public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer + private readonly AzureTableDataManager _dataManager; + private readonly TimeSpan _persistInterval; + private readonly IComparer? _checkpointComparer; + private readonly object _lock = new(); + + private StreamQueueCheckpointEntity _entity; + private Task _inProgressSave = Task.CompletedTask; + private DateTime? _throttleSavesUntilUtc; + private string _latestCheckpoint = string.Empty; + private string _persistedCheckpoint = string.Empty; + + private AzureTableStreamQueueCheckpointer( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + IComparer? defaultComparer = null, + string? partitionKeyPrefix = null) { - private readonly AzureTableDataManager _dataManager; - private readonly TimeSpan _persistInterval; - private readonly IComparer? _checkpointComparer; - private readonly object _lock = new(); + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName); + ArgumentException.ThrowIfNullOrWhiteSpace(partition); + ArgumentNullException.ThrowIfNull(loggerFactory); + if (options.PersistInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(options), + options.PersistInterval, + $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); + } - private StreamQueueCheckpointEntity _entity; - private Task _inProgressSave = Task.CompletedTask; - private DateTime? _throttleSavesUntilUtc; - private string _latestCheckpoint = string.Empty; - private string _persistedCheckpoint = string.Empty; + _persistInterval = options.PersistInterval; + _checkpointComparer = options.CheckpointComparer ?? defaultComparer; + _dataManager = new AzureTableDataManager( + options, + loggerFactory.CreateLogger()); + _entity = StreamQueueCheckpointEntity.Create( + partitionKeyPrefix ?? options.PartitionKeyPrefix, + streamProviderName, + serviceId, + partition); + LogCreatingCheckpointer( + loggerFactory.CreateLogger(), + partition, + streamProviderName, + serviceId); + } - private AzureTableStreamQueueCheckpointer( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory, - IComparer? defaultComparer = null, - string? partitionKeyPrefix = null) + /// + public bool CheckpointExists + { + get { - ArgumentNullException.ThrowIfNull(options); - ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName); - ArgumentException.ThrowIfNullOrWhiteSpace(partition); - ArgumentNullException.ThrowIfNull(loggerFactory); - if (options.PersistInterval <= TimeSpan.Zero) + lock (_lock) { - throw new ArgumentOutOfRangeException( - nameof(options), - options.PersistInterval, - $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); + return !string.IsNullOrEmpty(_latestCheckpoint); } - - _persistInterval = options.PersistInterval; - _checkpointComparer = options.CheckpointComparer ?? defaultComparer; - _dataManager = new AzureTableDataManager( - options, - loggerFactory.CreateLogger()); - _entity = StreamQueueCheckpointEntity.Create( - partitionKeyPrefix ?? options.PartitionKeyPrefix, - streamProviderName, - serviceId, - partition); - LogCreatingCheckpointer( - loggerFactory.CreateLogger(), - partition, - streamProviderName, - serviceId); } + } + + /// + /// Creates and initializes an Azure Table stream queue checkpointer. + /// + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory) + { + return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + } - /// - public bool CheckpointExists + internal static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + IComparer? defaultComparer, + string? partitionKeyPrefix = null) + { + var checkpointer = new AzureTableStreamQueueCheckpointer( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer, + partitionKeyPrefix); + await checkpointer._dataManager.InitTableAsync(); + return checkpointer; + } + + /// + public async Task Load() + { + var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); + var checkpoint = result.Entity?.Offset ?? string.Empty; + lock (_lock) { - get + if (result.Entity is not null) { - lock (_lock) - { - return !string.IsNullOrEmpty(_latestCheckpoint); - } + _entity = result.Entity; } + + _latestCheckpoint = checkpoint; + _persistedCheckpoint = checkpoint; } - /// - /// Creates and initializes an Azure Table stream queue checkpointer. - /// - public static Task> Create( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory) + return checkpoint; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public async Task Reset(CancellationToken cancellationToken) + { + Task resetTask; + Task inProgressSave; + string latestCheckpoint; + DateTime? throttleSavesUntilUtc; + lock (_lock) { - return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + inProgressSave = _inProgressSave; + latestCheckpoint = _latestCheckpoint; + throttleSavesUntilUtc = _throttleSavesUntilUtc; + _latestCheckpoint = string.Empty; + _throttleSavesUntilUtc = DateTime.MaxValue; + resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); } - internal static async Task> Create( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory, - IComparer? defaultComparer, - string? partitionKeyPrefix = null) + try { - var checkpointer = new AzureTableStreamQueueCheckpointer( - options, - streamProviderName, - partition, - serviceId, - loggerFactory, - defaultComparer, - partitionKeyPrefix); - await checkpointer._dataManager.InitTableAsync(); - return checkpointer; + await resetTask; } - - /// - public async Task Load() + catch { - var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); - var checkpoint = result.Entity?.Offset ?? string.Empty; lock (_lock) { - if (result.Entity is not null) + if (ReferenceEquals(resetTask, _inProgressSave)) { - _entity = result.Entity; + _latestCheckpoint = latestCheckpoint; + _throttleSavesUntilUtc = throttleSavesUntilUtc; + _inProgressSave = inProgressSave; } - - _latestCheckpoint = checkpoint; - _persistedCheckpoint = checkpoint; } - return checkpoint; + throw; } - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Reset() => Reset(CancellationToken.None); - - /// - public async Task Reset(CancellationToken cancellationToken) + lock (_lock) { - Task resetTask; - lock (_lock) + if (ReferenceEquals(resetTask, _inProgressSave)) { - var inProgressSave = _inProgressSave; _latestCheckpoint = string.Empty; - _throttleSavesUntilUtc = DateTime.MaxValue; - resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); + _persistedCheckpoint = string.Empty; + _throttleSavesUntilUtc = null; + _inProgressSave = Task.CompletedTask; } + } + } - try - { - await resetTask; - } - catch - { - lock (_lock) - { - if (ReferenceEquals(resetTask, _inProgressSave)) - { - _latestCheckpoint = _persistedCheckpoint; - _throttleSavesUntilUtc = null; - _inProgressSave = Task.CompletedTask; - } - } + /// + public void Update(string offset, DateTime utcNow) + { + ArgumentNullException.ThrowIfNull(offset); - throw; + lock (_lock) + { + if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) + || (_checkpointComparer is { } comparer + && !string.IsNullOrEmpty(_latestCheckpoint) + && comparer.Compare(offset, _latestCheckpoint) <= 0)) + { + return; } - lock (_lock) + _latestCheckpoint = offset; + if (!_inProgressSave.IsCompleted + || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) { - if (ReferenceEquals(resetTask, _inProgressSave)) - { - _latestCheckpoint = string.Empty; - _persistedCheckpoint = string.Empty; - _throttleSavesUntilUtc = null; - _inProgressSave = Task.CompletedTask; - } + return; } + + _throttleSavesUntilUtc = utcNow + _persistInterval; + _inProgressSave = Save(offset); + _inProgressSave.Ignore(); } + } - /// - public void Update(string offset, DateTime utcNow) + /// + public async Task FlushAsync(CancellationToken cancellationToken) + { + var retryingSave = false; + while (true) { - ArgumentNullException.ThrowIfNull(offset); - + Task inProgressSave; lock (_lock) { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_checkpointComparer is { } comparer - && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) + inProgressSave = _inProgressSave; + } + + if (retryingSave) + { + await inProgressSave.WaitAsync(cancellationToken); + } + else + { + try { - return; + await inProgressSave.WaitAsync(cancellationToken); } - - _latestCheckpoint = offset; - if (_throttleSavesUntilUtc.HasValue - && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) + catch (Exception) when (!cancellationToken.IsCancellationRequested) { - return; } - _throttleSavesUntilUtc = utcNow + _persistInterval; - _inProgressSave = Save(offset); - _inProgressSave.Ignore(); + cancellationToken.ThrowIfCancellationRequested(); } - } - /// - public async Task FlushAsync(CancellationToken cancellationToken) - { - var retryingSave = false; - while (true) + lock (_lock) { - Task inProgressSave; - lock (_lock) + if (!ReferenceEquals(inProgressSave, _inProgressSave)) { - inProgressSave = _inProgressSave; + retryingSave = false; + continue; } - if (retryingSave) + if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) { - await inProgressSave.WaitAsync(cancellationToken); - } - else - { - try - { - await inProgressSave.WaitAsync(cancellationToken); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) - { - } - - cancellationToken.ThrowIfCancellationRequested(); + return; } - lock (_lock) - { - if (!ReferenceEquals(inProgressSave, _inProgressSave)) - { - retryingSave = false; - continue; - } - - if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) - { - return; - } - - _inProgressSave = Save(_latestCheckpoint); - retryingSave = true; - } + _inProgressSave = Save(_latestCheckpoint); + retryingSave = true; } } + } - private async Task Save(string checkpoint) + private async Task Save(string checkpoint) + { + _entity.Offset = checkpoint; + await _dataManager.UpsertTableEntryAsync(_entity); + lock (_lock) { - _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity); - lock (_lock) - { - _persistedCheckpoint = checkpoint; - } + _persistedCheckpoint = checkpoint; } + } - private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) + private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) + { + try { await inProgressSave.WaitAsync(cancellationToken); - _entity.Offset = string.Empty; - await _dataManager.UpsertTableEntryAsync(_entity); + } + catch (Exception) when (!cancellationToken.IsCancellationRequested) + { } - [LoggerMessage( - Level = LogLevel.Information, - Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] - private static partial void LogCreatingCheckpointer( - ILogger logger, - string partition, - string streamProviderName, - string serviceId); + cancellationToken.ThrowIfCancellationRequested(); + _entity.Offset = string.Empty; + await _dataManager.UpsertTableEntryAsync(_entity, cancellationToken); } + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] + private static partial void LogCreatingCheckpointer( + ILogger logger, + string partition, + string streamProviderName, + string serviceId); } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index e6e8a60c52d..6d804067340 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -1,147 +1,130 @@ using System; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Streams; -using Orleans.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Orleans.Configuration; using Orleans.Configuration.Overrides; +using Orleans.Streams; + +namespace Orleans.Streaming.EventHubs; + +public class EventHubCheckpointerFactory : IStreamQueueCheckpointerFactory +{ + private readonly ILoggerFactory loggerFactory; + private readonly string providerName; + private readonly AzureTableStreamCheckpointerOptions options; + private readonly ClusterOptions clusterOptions; + + public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpointerOptions options, IOptions clusterOptions, ILoggerFactory loggerFactory) + { + this.options = options; + this.clusterOptions = clusterOptions.Value; + this.loggerFactory = loggerFactory; + this.providerName = providerName; + } + + public Task> Create(string partition) + { + return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); + } -namespace Orleans.Streaming.EventHubs + public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) + { + var options = services.GetOptionsByName(providerName); + IOptions clusterOptions = services.GetProviderClusterOptions(providerName); + return ActivatorUtilities.CreateInstance(services, providerName, options, clusterOptions); + } +} + +/// +/// This class stores EventHub partition checkpoint information in Azure Table Storage. +/// +public class EventHubCheckpointer : IStreamQueueCheckpointer { + private readonly IStreamQueueCheckpointer _inner; + /// - /// Creates Event Hub partition checkpointers backed by Azure Table Storage. + /// Indicates if a checkpoint exists /// - public class EventHubCheckpointerFactory : IStreamQueueCheckpointerFactory + public bool CheckpointExists => _inner.CheckpointExists; + + /// + /// Factory function that creates and initializes the checkpointer + /// + /// + /// + /// + /// + /// + /// + public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) + { + var inner = await AzureTableStreamQueueCheckpointer.Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + StreamCheckpointComparers.Numeric, + StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); + return new EventHubCheckpointer(inner); + } + + private EventHubCheckpointer(IStreamQueueCheckpointer inner) { - private readonly ILoggerFactory loggerFactory; - private readonly string providerName; - private readonly AzureTableStreamCheckpointerOptions options; - private readonly ClusterOptions clusterOptions; - - /// - /// Initializes a new instance of the class. - /// - /// The stream provider name. - /// The Azure Table Storage checkpointer options. - /// The cluster options. - /// The logger factory. - public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpointerOptions options, IOptions clusterOptions, ILoggerFactory loggerFactory) - { - this.options = options; - this.clusterOptions = clusterOptions.Value; - this.loggerFactory = loggerFactory; - this.providerName = providerName; - } - - /// - public Task> Create(string partition) - { - return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); - } - - /// - /// Creates an Event Hub partition checkpointer factory. - /// - /// The service provider. - /// The stream provider name. - /// The checkpointer factory. - public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) - { - var options = services.GetOptionsByName(providerName); - IOptions clusterOptions = services.GetProviderClusterOptions(providerName); - return ActivatorUtilities.CreateInstance(services, providerName, options, clusterOptions); - } + _inner = inner; } /// - /// This class stores EventHub partition checkpoint information in Azure Table Storage. + /// Loads a checkpoint /// - public class EventHubCheckpointer : IStreamQueueCheckpointer + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + + /// + /// Loads a checkpoint. + /// + /// The cancellation token. + /// The checkpoint. + public async Task Load(CancellationToken cancellationToken) { - private readonly IStreamQueueCheckpointer _inner; - - /// - public bool CheckpointExists => _inner.CheckpointExists; - - /// - /// Creates and initializes an Event Hub partition checkpointer. - /// - /// The Azure Table Storage checkpointer options. - /// The stream provider name. - /// The Event Hub partition identifier. - /// The Orleans service identifier. - /// The logger factory. - /// A task which resolves to the initialized checkpointer. - public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) - { - var inner = await AzureTableStreamQueueCheckpointer.Create( - options, - streamProviderName, - partition, - serviceId, - loggerFactory, - StreamCheckpointComparers.Numeric, - StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); - return new EventHubCheckpointer(inner); - } - - private EventHubCheckpointer(IStreamQueueCheckpointer inner) - { - _inner = inner; - } - - /// - /// Loads the checkpoint. - /// - /// A task which resolves to the checkpoint offset. - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Load() => Load(CancellationToken.None); - - /// - /// Loads a checkpoint. - /// - /// The cancellation token. - /// The checkpoint. - public async Task Load(CancellationToken cancellationToken) - { - var checkpoint = await _inner.Load(cancellationToken); - return CheckpointExists ? checkpoint : EventHubConstants.StartOfStream; - } - - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Reset() => Reset(CancellationToken.None); - - /// - public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); - - /// - /// Updates the checkpoint. This is a best effort. It does not always update the checkpoint. - /// The latest offset is always tracked in memory so that can persist it on shutdown. - /// - /// The checkpoint offset. - /// The current UTC time. - [Obsolete("Use the overload which accepts a CancellationToken.")] - public void Update(string offset, DateTime utcNow) - => Update(offset, utcNow, CancellationToken.None); - - /// - /// Updates the checkpoint. - /// - /// The checkpoint offset. - /// The current UTC time. - /// The cancellation token. - public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) - => _inner.Update(offset, utcNow, cancellationToken); - - /// - /// Flushes any pending checkpoint to persistent storage. - /// Awaits any in-progress save, then persists the latest offset if it has advanced beyond the last saved value. - /// - /// The cancellation token. - /// A task which represents the flush operation. - public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + var checkpoint = await _inner.Load(cancellationToken); + return CheckpointExists ? checkpoint : EventHubConstants.StartOfStream; } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); + + /// + /// Updates the checkpoint. This is a best effort. It does not always update the checkpoint. + /// The latest offset is always tracked in memory so that can persist it on shutdown. + /// + /// + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); + + /// + /// Updates the checkpoint. + /// + /// The checkpoint offset. + /// The current UTC time. + /// The cancellation token. + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); + + /// + /// Flushes any pending checkpoint to persistent storage. + /// Awaits any in-progress save, then persists the latest offset if it has advanced beyond the last saved value. + /// + /// The cancellation token. + public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); } diff --git a/src/Azure/Shared/Storage/AzureTableDataManager.cs b/src/Azure/Shared/Storage/AzureTableDataManager.cs index da64f019586..1c6cd3b8722 100644 --- a/src/Azure/Shared/Storage/AzureTableDataManager.cs +++ b/src/Azure/Shared/Storage/AzureTableDataManager.cs @@ -195,7 +195,16 @@ public async Task CreateTableEntryAsync(T data, CancellationToken cancel /// /// Data to be inserted or replaced in the table. /// Value promise with new Etag for this data entry after completing this storage operation. - public async Task UpsertTableEntryAsync(T data) + public Task UpsertTableEntryAsync(T data) + => UpsertTableEntryAsync(data, CancellationToken.None); + + /// + /// Inserts a data entry in the Azure table: creates a new one if does not exist or overwrites an existing version. + /// + /// Data to be inserted or replaced in the table. + /// The cancellation token. + /// Value promise with new Etag for this data entry after completing this storage operation. + public async Task UpsertTableEntryAsync(T data, CancellationToken cancellationToken) { const string operation = "UpsertTableEntry"; var startTime = DateTime.UtcNow; @@ -204,7 +213,7 @@ public async Task UpsertTableEntryAsync(T data) { try { - var opResult = await Table.UpsertEntityAsync(data); + var opResult = await Table.UpsertEntityAsync(data, cancellationToken: cancellationToken); return opResult.Headers.ETag.GetValueOrDefault().ToString(); } catch (Exception exc) diff --git a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs index 84b0f894de0..0dbb620aae5 100644 --- a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs @@ -2,102 +2,101 @@ using System.Threading; using System.Threading.Tasks; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Factory for creating instances. +/// +public interface IStreamQueueCheckpointerFactory { /// - /// Factory for creating instances. + /// Creates a stream checkpointer for the specified partition. /// - public interface IStreamQueueCheckpointerFactory - { - /// - /// Creates a stream checkpointer for the specified partition. - /// - /// The partition. - /// The stream checkpointer. - [Obsolete("Use the overload which accepts a CancellationToken.")] - Task> Create(string partition); + /// The partition. + /// The stream checkpointer. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task> Create(string partition); - /// - /// Creates a stream checkpointer for the specified partition. - /// - /// The partition. - /// The cancellation token. - /// The stream checkpointer. + /// + /// Creates a stream checkpointer for the specified partition. + /// + /// The partition. + /// The cancellation token. + /// The stream checkpointer. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task> Create(string partition, CancellationToken cancellationToken) - => Create(partition); + Task> Create(string partition, CancellationToken cancellationToken) + => Create(partition); #pragma warning restore CS0618 - } +} +/// +/// Functionality for checkpointing a stream. +/// +/// The checkpoint type. +public interface IStreamQueueCheckpointer +{ /// - /// Functionality for checkpointing a stream. + /// Gets a value indicating whether a checkpoint exists. /// - /// The checkpoint type. - public interface IStreamQueueCheckpointer - { - /// - /// Gets a value indicating whether a checkpoint exists. - /// - /// if checkpoint exists; otherwise, . - bool CheckpointExists { get; } + /// if checkpoint exists; otherwise, . + bool CheckpointExists { get; } - /// - /// Loads the checkpoint. - /// - /// The checkpoint. - [Obsolete("Use the overload which accepts a CancellationToken.")] - Task Load(); + /// + /// Loads the checkpoint. + /// + /// The checkpoint. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task Load(); - /// - /// Loads the checkpoint. - /// - /// The cancellation token. - /// The checkpoint. + /// + /// Loads the checkpoint. + /// + /// The cancellation token. + /// The checkpoint. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task Load(CancellationToken cancellationToken) => Load(); + Task Load(CancellationToken cancellationToken) => Load(); #pragma warning restore CS0618 - /// - /// Clears the persisted checkpoint. - /// - /// A representing the operation. - [Obsolete("Use the overload which accepts a CancellationToken.")] - Task Reset() => throw new NotSupportedException("This checkpointer does not support resetting checkpoints."); + /// + /// Clears the persisted checkpoint. + /// + /// A representing the operation. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task Reset() => throw new NotSupportedException("This checkpointer does not support resetting checkpoints."); - /// - /// Clears the persisted checkpoint. - /// - /// The cancellation token. - /// A representing the operation. + /// + /// Clears the persisted checkpoint. + /// + /// The cancellation token. + /// A representing the operation. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task Reset(CancellationToken cancellationToken) => Reset(); + Task Reset(CancellationToken cancellationToken) => Reset(); #pragma warning restore CS0618 - /// - /// Updates the checkpoint. - /// - /// The offset. - /// The current UTC time. - [Obsolete("Use the overload which accepts a CancellationToken.")] - void Update(TCheckpoint offset, DateTime utcNow); + /// + /// Updates the checkpoint. + /// + /// The offset. + /// The current UTC time. + [Obsolete("Use the overload which accepts a CancellationToken.")] + void Update(TCheckpoint offset, DateTime utcNow); - /// - /// Updates the checkpoint. - /// - /// The offset. - /// The current UTC time. - /// The cancellation token. + /// + /// Updates the checkpoint. + /// + /// The offset. + /// The current UTC time. + /// The cancellation token. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationToken) - => Update(offset, utcNow); + void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationToken) + => Update(offset, utcNow); #pragma warning restore CS0618 - /// - /// Flushes any pending checkpoint to persistent storage, ensuring the latest offset is durably saved. - /// Called during shutdown or rebalancing to prevent message replay on restart. - /// - /// The cancellation token. - /// A representing the flush operation. - Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - } + /// + /// Flushes any pending checkpoint to persistent storage, ensuring the latest offset is durably saved. + /// Called during shutdown or rebalancing to prevent message replay on restart. + /// + /// The cancellation token. + /// A representing the flush operation. + Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; } diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index 28fc9833e80..e83129129de 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -74,6 +74,47 @@ public async Task Reset_ClearsPersistedCheckpoint() Assert.Equal([NoCheckpoint], store.CompletedWrites); } + [Fact] + public async Task Reset_WhenCanceledBehindPendingWrite_RetryRemainsSerialized() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedWrite = store.BlockNextWrite(); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForWriteAttempts(1); + using var cancellation = new CancellationTokenSource(); + + var canceledReset = checkpointer.Reset(cancellation.Token); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledReset); + + var retry = checkpointer.Reset(CancellationToken.None); + Assert.False(retry.IsCompleted); + Assert.Equal(["20"], store.WriteAttempts); + + blockedWrite.SetResult(); + await retry; + + Assert.Equal(["20", NoCheckpoint], store.WriteAttempts); + Assert.Equal(["20", NoCheckpoint], store.CompletedWrites); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.False(checkpointer.CheckpointExists); + } + + [Fact] + public async Task Reset_AfterPendingWriteFails_ClearsCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + store.FailNextWrite(new InvalidOperationException("checkpoint write failed")); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + + await checkpointer.Reset(CancellationToken.None); + + Assert.Equal(["20", NoCheckpoint], store.WriteAttempts); + Assert.Equal([NoCheckpoint], store.CompletedWrites); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.False(checkpointer.CheckpointExists); + } + [Fact] public async Task Update_PersistsCheckpoint() { From c1157973af0dc51ecc69a8aeb5ad23b73bd64c58 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 00:59:07 -0700 Subject: [PATCH 03/30] fix(streaming): propagate checkpoint cancellation --- .../AzureTableStreamQueueCheckpointer.cs | 43 ++++++++- ...zureTableStreamQueueCheckpointerFactory.cs | 95 ++++++++++--------- .../Streams/EventHub/EventHubCheckpointer.cs | 34 ++++++- .../Orleans.Streaming.EventHubs.cs | 10 ++ 4 files changed, 131 insertions(+), 51 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 0f87be78966..b50b403d12d 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -83,8 +83,33 @@ public static Task> Create( string partition, string serviceId, ILoggerFactory loggerFactory) + => Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + + /// + /// Creates and initializes an Azure Table stream queue checkpointer. + /// + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) { - return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: cancellationToken); } internal static async Task> Create( @@ -94,7 +119,8 @@ internal static async Task> Create( string serviceId, ILoggerFactory loggerFactory, IComparer? defaultComparer, - string? partitionKeyPrefix = null) + string? partitionKeyPrefix = null, + CancellationToken cancellationToken = default) { var checkpointer = new AzureTableStreamQueueCheckpointer( options, @@ -104,14 +130,21 @@ internal static async Task> Create( loggerFactory, defaultComparer, partitionKeyPrefix); - await checkpointer._dataManager.InitTableAsync(); + await checkpointer._dataManager.InitTableAsync(cancellationToken); return checkpointer; } /// - public async Task Load() + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + + /// + public async Task Load(CancellationToken cancellationToken) { - var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); + var result = await _dataManager.ReadSingleTableEntryAsync( + _entity.PartitionKey, + _entity.RowKey, + cancellationToken); var checkpoint = result.Entity?.Offset ?? string.Empty; lock (_lock) { diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs index 6aae03ee714..3e0f4cd1912 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -6,56 +7,62 @@ using Orleans.Configuration; using Orleans.Configuration.Overrides; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Creates Azure Table stream queue checkpointers. +/// +public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory { + private readonly string _providerName; + private readonly AzureTableStreamCheckpointerOptions _options; + private readonly ClusterOptions _clusterOptions; + private readonly ILoggerFactory _loggerFactory; + /// - /// Creates Azure Table stream queue checkpointers. + /// Initializes a new instance. /// - public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory + public AzureTableStreamQueueCheckpointerFactory( + string providerName, + AzureTableStreamCheckpointerOptions options, + IOptions clusterOptions, + ILoggerFactory loggerFactory) { - private readonly string _providerName; - private readonly AzureTableStreamCheckpointerOptions _options; - private readonly ClusterOptions _clusterOptions; - private readonly ILoggerFactory _loggerFactory; + _providerName = providerName; + _options = options; + _clusterOptions = clusterOptions.Value; + _loggerFactory = loggerFactory; + } - /// - /// Initializes a new instance. - /// - public AzureTableStreamQueueCheckpointerFactory( - string providerName, - AzureTableStreamCheckpointerOptions options, - IOptions clusterOptions, - ILoggerFactory loggerFactory) - { - _providerName = providerName; - _options = options; - _clusterOptions = clusterOptions.Value; - _loggerFactory = loggerFactory; - } + /// + /// Creates a factory from a service provider. + /// + public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) + { + var options = services.GetOptionsByName(providerName); + var clusterOptions = services.GetProviderClusterOptions(providerName); + return ActivatorUtilities.CreateInstance( + services, + providerName, + options, + clusterOptions); + } - /// - /// Creates a factory from a service provider. - /// - public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) - { - var options = services.GetOptionsByName(providerName); - var clusterOptions = services.GetProviderClusterOptions(providerName); - return ActivatorUtilities.CreateInstance( - services, - providerName, - options, - clusterOptions); - } + /// + public Task> Create(string partition) + => Create(partition, CancellationToken.None); - /// - public Task> Create(string partition) - { - return AzureTableStreamQueueCheckpointer.Create( - _options, - _providerName, - partition, - _clusterOptions.ServiceId.ToString(), - _loggerFactory); - } + /// + public Task> Create( + string partition, + CancellationToken cancellationToken) + { + return AzureTableStreamQueueCheckpointer.Create( + _options, + _providerName, + partition, + _clusterOptions.ServiceId.ToString(), + _loggerFactory, + cancellationToken); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 6d804067340..a0be92a5588 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -26,8 +26,19 @@ public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpoi } public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) { - return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); + return EventHubCheckpointer.Create( + options, + providerName, + partition, + this.clusterOptions.ServiceId.ToString(), + loggerFactory, + cancellationToken); } public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) @@ -60,6 +71,24 @@ public class EventHubCheckpointer : IStreamQueueCheckpointer /// /// public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) + => await Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + + /// + /// Factory function that creates and initializes the checkpointer. + /// + public static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) { var inner = await AzureTableStreamQueueCheckpointer.Create( options, @@ -68,7 +97,8 @@ public static async Task> Create(AzureTableStre serviceId, loggerFactory, StreamCheckpointComparers.Numeric, - StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); + StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix, + cancellationToken); return new EventHubCheckpointer(inner); } diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index 7bd23fad022..91311b5a37a 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -368,6 +368,8 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -392,6 +394,8 @@ public partial class EventHubCheckpointerFactory : Streams.IStreamQueueCheckpoin { public EventHubCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } @@ -788,12 +792,16 @@ internal AzureTableStreamQueueCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } public System.Threading.Tasks.Task Load() { throw null; } + public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Reset() { throw null; } @@ -806,6 +814,8 @@ public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueChec { public AzureTableStreamQueueCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } From 03ef0ebe03be3ff798b7ebc8797e47a562ee123f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 01:16:37 -0700 Subject: [PATCH 04/30] docs(streaming): explain Event Hubs checkpoint recovery --- .../Providers/Streams/EventHub/EventHubStreamOptions.cs | 3 ++- src/Azure/Orleans.Streaming.EventHubs/README.md | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs index bc12faf9568..e8006b583af 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs @@ -255,8 +255,9 @@ public class EventHubReceiverOptions /// public int? PrefetchCount { get; set; } /// - /// In cases where no checkpoint is found, this indicates if service should read from the most recent data, or from the beginning of a partition. + /// Selects the receiver position when a checkpoint is unavailable, including after recovery from an invalid persisted offset. /// + /// starts at the partition tail and reads newly enqueued events; starts at the earliest retained event. public bool StartFromNow { get; set; } = DEFAULT_START_FROM_NOW; private const bool DEFAULT_START_FROM_NOW = true; } diff --git a/src/Azure/Orleans.Streaming.EventHubs/README.md b/src/Azure/Orleans.Streaming.EventHubs/README.md index 4f2c4ab0578..df8309cfa3f 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/README.md +++ b/src/Azure/Orleans.Streaming.EventHubs/README.md @@ -74,6 +74,14 @@ Set `GrainStreamQueueCheckpointerOptions.StorageProviderName` to use another reg Both `UseGrainCheckpointer` and `UseAzureTableCheckpointer` extend `ISiloPersistentStreamConfigurator`, so they can also be used with other persistent stream providers. Event Hubs configures numeric checkpoint ordering by default; other providers can set the corresponding `CheckpointComparer` option for their checkpoint format. +### Recovering from an expired checkpoint + +When Event Hubs rejects a persisted partition offset as invalid, the provider clears that partition's checkpoint, closes and recreates its receiver, and resumes on the next pull using the configured no-checkpoint position. `EventHubReceiverOptions.StartFromNow` selects that position. Its default value, `true`, starts at the partition tail and reads newly enqueued events, skipping the retained backlog. Set it to `false` to replay from the earliest retained event, which can duplicate delivery. Choose the policy according to the application's recovery requirements and keep consumers idempotent. + +The built-in Azure Table and grain-backed checkpointers support this reset contract. A custom `IStreamQueueCheckpointer` participates in recovery by implementing `Reset(CancellationToken)`; the default implementation surfaces unsupported recovery with `NotSupportedException`. + +Existing Orleans subscription cursors rebind to the replacement partition cache, so subscriptions continue without application re-registration. Checkpoint reset, receiver shutdown, and receiver initialization observe the pulling agent's cancellation token. + ## Example - Using Event Hub Streams in a Grain ```csharp From a08d63479adf27411032e6c7d51f27d6e1e334ac Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 02:49:37 -0700 Subject: [PATCH 05/30] fix(streaming): honor checkpointer semantics --- .../NoOpCheckpointer.cs | 2 +- .../AzureTableStreamQueueCheckpointer.cs | 8 +++---- .../Streams/EventHub/EventHubCheckpointer.cs | 24 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index 2f422cb6101..3607b2dcf44 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -22,7 +22,7 @@ public class NoOpCheckpointer : IStreamQueueCheckpointer { public static NoOpCheckpointer Instance = new NoOpCheckpointer(); - public bool CheckpointExists => true; + public bool CheckpointExists => false; public Task Load() { return Task.FromResult(EventHubConstants.StartOfStream); diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index b50b403d12d..8925d3d353a 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -235,7 +235,7 @@ public void Update(string offset, DateTime utcNow) } _throttleSavesUntilUtc = utcNow + _persistInterval; - _inProgressSave = Save(offset); + _inProgressSave = Save(offset, CancellationToken.None); _inProgressSave.Ignore(); } } @@ -282,16 +282,16 @@ public async Task FlushAsync(CancellationToken cancellationToken) return; } - _inProgressSave = Save(_latestCheckpoint); + _inProgressSave = Save(_latestCheckpoint, cancellationToken); retryingSave = true; } } } - private async Task Save(string checkpoint) + private async Task Save(string checkpoint, CancellationToken cancellationToken) { _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity); + await _dataManager.UpsertTableEntryAsync(_entity, cancellationToken); lock (_lock) { _persistedCheckpoint = checkpoint; diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index a0be92a5588..b7e710e8533 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -12,17 +12,17 @@ namespace Orleans.Streaming.EventHubs; public class EventHubCheckpointerFactory : IStreamQueueCheckpointerFactory { - private readonly ILoggerFactory loggerFactory; - private readonly string providerName; - private readonly AzureTableStreamCheckpointerOptions options; - private readonly ClusterOptions clusterOptions; + private readonly ILoggerFactory _loggerFactory; + private readonly string _providerName; + private readonly AzureTableStreamCheckpointerOptions _options; + private readonly ClusterOptions _clusterOptions; public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpointerOptions options, IOptions clusterOptions, ILoggerFactory loggerFactory) { - this.options = options; - this.clusterOptions = clusterOptions.Value; - this.loggerFactory = loggerFactory; - this.providerName = providerName; + _options = options; + _clusterOptions = clusterOptions.Value; + _loggerFactory = loggerFactory; + _providerName = providerName; } public Task> Create(string partition) @@ -33,11 +33,11 @@ public Task> Create( CancellationToken cancellationToken) { return EventHubCheckpointer.Create( - options, - providerName, + _options, + _providerName, partition, - this.clusterOptions.ServiceId.ToString(), - loggerFactory, + _clusterOptions.ServiceId.ToString(), + _loggerFactory, cancellationToken); } From c49870573d6117572df78ea5a90f7f1850660b56 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 25 Aug 2026 07:29:39 -0700 Subject: [PATCH 06/30] fix(streaming): complete cache recovery contracts --- .../NoOpCheckpointer.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index 3607b2dcf44..4cdf3bfc39a 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -9,9 +9,14 @@ public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory { public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); public Task> Create(string partition) - { - return Task.FromResult>(NoOpCheckpointer.Instance); - } + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled>(cancellationToken) + : Task.FromResult>(NoOpCheckpointer.Instance); } /// /// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, From fa6a4c7a178b1243bc0c8a18ab7d88bf40a0d7b7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Aug 2026 02:19:27 -0700 Subject: [PATCH 07/30] fix(streaming): honor canceled checkpointer operations --- .../IStreamQueueCheckpointer.cs | 7 +++-- ...entStreamCancellationCompatibilityTests.cs | 29 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs index 0dbb620aae5..837cde0650f 100644 --- a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs @@ -25,7 +25,9 @@ public interface IStreamQueueCheckpointerFactory /// The stream checkpointer. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. Task> Create(string partition, CancellationToken cancellationToken) - => Create(partition); + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled>(cancellationToken) + : Create(partition); #pragma warning restore CS0618 } @@ -70,7 +72,8 @@ public interface IStreamQueueCheckpointer /// The cancellation token. /// A representing the operation. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task Reset(CancellationToken cancellationToken) => Reset(); + Task Reset(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Reset(); #pragma warning restore CS0618 /// diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs index dea5034d7c2..18267e205d2 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs @@ -32,12 +32,13 @@ public async Task CancellationOverloads_FallBackToLegacyImplementations() Assert.True(((LegacyQueueAdapterReceiver)receiver).MessagesDelivered); Assert.True(((LegacyQueueAdapterReceiver)receiver).ShutdownCalled); + var cancellationToken = CancellationToken.None; IStreamQueueCheckpointer checkpointer = new LegacyStreamQueueCheckpointer(); IStreamQueueCheckpointerFactory checkpointerFactory = new LegacyStreamQueueCheckpointerFactory(checkpointer); - Assert.Same(checkpointer, await checkpointerFactory.Create("partition", cancellation.Token)); - Assert.Equal("10", await checkpointer.Load(cancellation.Token)); + Assert.Same(checkpointer, await checkpointerFactory.Create("partition", cancellationToken)); + Assert.Equal("10", await checkpointer.Load(cancellationToken)); - checkpointer.Update("20", DateTime.UtcNow, cancellation.Token); + checkpointer.Update("20", DateTime.UtcNow, cancellationToken); Assert.Equal("20", ((LegacyStreamQueueCheckpointer)checkpointer).Checkpoint); } @@ -96,6 +97,28 @@ await Assert.ThrowsAnyAsync( () => receiver.Shutdown(TimeSpan.FromSeconds(1), cancellation.Token)); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task CheckpointerCompatibilityOverloads_HonorPreCanceledOperations() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + IStreamQueueCheckpointer checkpointer = new LegacyStreamQueueCheckpointer(); + IStreamQueueCheckpointerFactory checkpointerFactory = new LegacyStreamQueueCheckpointerFactory(checkpointer); + + await Assert.ThrowsAnyAsync( + () => checkpointerFactory.Create("partition", cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.Load(cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.Reset(cancellation.Token)); + Assert.ThrowsAny( + () => checkpointer.Update("20", DateTime.UtcNow, cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.FlushAsync(cancellation.Token)); + Assert.Equal("10", ((LegacyStreamQueueCheckpointer)checkpointer).Checkpoint); + } + private sealed class LegacyQueueAdapterFactory(IQueueAdapter adapter) : IQueueAdapterFactory { public Task CreateAdapter() => Task.FromResult(adapter); From 02d88033850ffef44bc0ef259a81d19864861f7a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Aug 2026 03:51:49 -0700 Subject: [PATCH 08/30] fix(streaming): honor all checkpointer cancellations --- .../PersistentStreams/IStreamQueueCheckpointer.cs | 11 ++++++++--- .../PersistentStreamCancellationCompatibilityTests.cs | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs index 837cde0650f..f8c686cbb90 100644 --- a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs @@ -56,7 +56,8 @@ public interface IStreamQueueCheckpointer /// The cancellation token. /// The checkpoint. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task Load(CancellationToken cancellationToken) => Load(); + Task Load(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Load(); #pragma warning restore CS0618 /// @@ -92,7 +93,10 @@ Task Reset(CancellationToken cancellationToken) /// The cancellation token. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationToken) - => Update(offset, utcNow); + { + cancellationToken.ThrowIfCancellationRequested(); + Update(offset, utcNow); + } #pragma warning restore CS0618 /// @@ -101,5 +105,6 @@ void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationT /// /// The cancellation token. /// A representing the flush operation. - Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + Task FlushAsync(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs index 18267e205d2..0b1e76ddb9b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs @@ -176,6 +176,5 @@ private sealed class LegacyStreamQueueCheckpointer : IStreamQueueCheckpointer Checkpoint = offset; - public Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; } } From 1990c7aec14629d11eae3030ab12c0e5c4df4b0a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 3 Sep 2026 05:38:33 -0700 Subject: [PATCH 09/30] fix(streaming): preserve current stream contracts --- .../Orleans.Streaming.EventHubs.cs | 2 ++ .../Orleans.Azure.Tests/AzureTableDataManagerTests.cs | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index 91311b5a37a..04ea2b688b9 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -765,6 +765,7 @@ public void Update(string offset, System.DateTime utcNow) { } public partial class NoOpCheckpointerFactory : Streams.IStreamQueueCheckpointerFactory { public static NoOpCheckpointerFactory Instance; + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } public System.Threading.Tasks.Task> Create(string partition) { throw null; } } @@ -798,6 +799,7 @@ internal AzureTableStreamQueueCheckpointer() { } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs index 2521be419d1..65e3f72b2b2 100644 --- a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs +++ b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs @@ -64,7 +64,7 @@ public async Task AzureTableDataManager_CreateTableEntryAsync() public async Task AzureTableDataManager_UpsertTableEntryAsync() { var data = GenerateNewData(); - await manager.UpsertTableEntryAsync(data); + await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var tuple = await manager.ReadSingleTableEntryAsync( data.PartitionKey, data.RowKey, @@ -74,7 +74,7 @@ public async Task AzureTableDataManager_UpsertTableEntryAsync() var data2 = data.Clone(); data2.StringData = "NewData"; - await manager.UpsertTableEntryAsync(data2); + await manager.UpsertTableEntryAsync(data2, TestContext.Current.CancellationToken); tuple = await manager.ReadSingleTableEntryAsync( data2.PartitionKey, data2.RowKey, @@ -103,7 +103,7 @@ public async Task AzureTableDataManager_UpdateTableEntryAsync() Assert.Equal(TableErrorCode.ResourceNotFound.ToString(), restStatus); } - await manager.UpsertTableEntryAsync(data); + await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var tuple = await manager.ReadSingleTableEntryAsync( data.PartitionKey, data.RowKey, @@ -164,7 +164,7 @@ public async Task AzureTableDataManager_DeleteTableAsync() Assert.Equal(HttpStatusCode.NotFound, httpStatusCode); } - string eTag1 = await manager.UpsertTableEntryAsync(data); + string eTag1 = await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); await manager.DeleteTableEntryAsync(data, eTag1); try @@ -207,7 +207,7 @@ public async Task AzureTableDataManager_MergeTableAsync() Assert.Equal(TableErrorCode.ResourceNotFound.ToString(), restStatus); } - string eTag1 = await manager.UpsertTableEntryAsync(data); + string eTag1 = await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var data2 = data.Clone(); data2.StringData = "NewData"; await manager.MergeTableEntryAsync(data2, eTag1, TestContext.Current.CancellationToken); From 249c8b96e7f8fcbf2724a1ae329a6b66a39b1ea6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 3 Sep 2026 15:29:15 -0700 Subject: [PATCH 10/30] fix(azure): propagate storage cancellation --- .../Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs index 31382d25b66..d110b6df080 100644 --- a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs +++ b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs @@ -149,7 +149,7 @@ private async Task WriteAlot_Async( dataObject.PartitionKey = partitionKey; dataObject.RowKey = rowKey; dataObject.StringData = rowKey; - var promise = manager.UpsertTableEntryAsync(dataObject); + var promise = manager.UpsertTableEntryAsync(dataObject, cancellationToken); promises.Add(promise); if ((i % batchSize) == 0 && i > 0) { From 881531c5f8e0ca40063645a8fd163cca09bfdaff Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 02:37:00 -0700 Subject: [PATCH 11/30] fix(streaming): align Event Hubs diagnostics --- .../EventDataGeneratorStreamProvider/NoOpCheckpointer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index 4cdf3bfc39a..ab5e43accd7 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using Orleans.Streams; From 938a3a6001fc35fb1bc802932554b1c5e95229e2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 23:36:46 -0700 Subject: [PATCH 12/30] fix(streaming): obsolete legacy checkpointer factories --- .../Streams/DynamoDBStreamQueueCheckpointerFactory.cs | 1 + .../EventDataGeneratorStreamProvider/NoOpCheckpointer.cs | 2 ++ .../EventHub/AzureTableStreamQueueCheckpointerFactory.cs | 1 + .../Providers/Streams/EventHub/EventHubCheckpointer.cs | 1 + .../Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs | 1 + .../Orleans.Streaming.EventHubs.cs | 4 ++++ 6 files changed, 10 insertions(+) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs index ff76c59de58..04dc841f450 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs @@ -77,6 +77,7 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public async Task> Create(string partition) => await Create(partition, CancellationToken.None); diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index ab5e43accd7..183645aa2a2 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -8,6 +8,8 @@ namespace Orleans.Streaming.EventHubs.Testing; public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory { public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); + + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) => Create(partition, CancellationToken.None); diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs index 3e0f4cd1912..69f24caf8fe 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs @@ -49,6 +49,7 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) => Create(partition, CancellationToken.None); diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index b7e710e8533..b3560074f29 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -25,6 +25,7 @@ public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpoi _providerName = providerName; } + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) => Create(partition, CancellationToken.None); diff --git a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs index 9ef8844adb3..ecf77c70234 100644 --- a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs +++ b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs @@ -89,6 +89,7 @@ public DynamoDBStreamQueueCheckpointerFactory(string providerName, Configuration public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index 04ea2b688b9..a5d8ec5ef59 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -396,6 +396,7 @@ public EventHubCheckpointerFactory(string providerName, Configuration.AzureTable public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } @@ -766,6 +767,8 @@ public partial class NoOpCheckpointerFactory : Streams.IStreamQueueCheckpointerF { public static NoOpCheckpointerFactory Instance; public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } } @@ -818,6 +821,7 @@ public AzureTableStreamQueueCheckpointerFactory(string providerName, Configurati public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } From ff70b14ec73cd059b4eb5c5011fffc97516aefa4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 5 Sep 2026 00:00:29 -0700 Subject: [PATCH 13/30] fix(streaming): isolate Azure checkpoint writes --- .../AzureTableStreamQueueCheckpointer.cs | 27 ++++++++++++++--- .../EventHubCheckpointerTests.cs | 30 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 8925d3d353a..fb6842b0d8b 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -228,6 +228,7 @@ public void Update(string offset, DateTime utcNow) } _latestCheckpoint = offset; + _entity.Offset = offset; if (!_inProgressSave.IsCompleted || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) { @@ -290,10 +291,11 @@ public async Task FlushAsync(CancellationToken cancellationToken) private async Task Save(string checkpoint, CancellationToken cancellationToken) { - _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity, cancellationToken); + var entity = CreateWriteEntity(checkpoint); + await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); lock (_lock) { + _entity.Offset = checkpoint; _persistedCheckpoint = checkpoint; } } @@ -309,8 +311,25 @@ private async Task ResetCore(Task inProgressSave, CancellationToken cancellation } cancellationToken.ThrowIfCancellationRequested(); - _entity.Offset = string.Empty; - await _dataManager.UpsertTableEntryAsync(_entity, cancellationToken); + var entity = CreateWriteEntity(string.Empty); + await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); + lock (_lock) + { + _entity.Offset = string.Empty; + } + } + + private StreamQueueCheckpointEntity CreateWriteEntity(string checkpoint) + { + lock (_lock) + { + return new StreamQueueCheckpointEntity + { + PartitionKey = _entity.PartitionKey, + RowKey = _entity.RowKey, + Offset = checkpoint, + }; + } } [LoggerMessage( diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index f8c7d078cb3..a96182e1b5e 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -682,6 +682,22 @@ public void PartitionKeyPrefix_IsAppliedPerCheckpointer(string partitionKeyPrefi Assert.Equal(expected, GetEntityPartitionKey(checkpointer)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void AzureTableCheckpointer_CreatesIndependentWriteEntity() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetPersistedOffset(checkpointer, "10"); + + var writeEntity = InvokeCreateWriteEntity(checkpointer, "20"); + + Assert.NotSame(GetField(checkpointer, "_entity"), writeEntity); + Assert.Equal(GetEntityPartitionKey(checkpointer), GetEntityProperty(writeEntity, "PartitionKey")); + Assert.Equal(GetEntityRowKey(checkpointer), GetEntityProperty(writeEntity, "RowKey")); + Assert.Equal("20", GetEntityProperty(writeEntity, "Offset")); + Assert.Equal("10", GetEntityOffset(checkpointer)); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public void EventHubCheckpointEntity_PreservesLegacyAzureTableSchema() @@ -965,6 +981,20 @@ private static void SetEntityOffset(AzureTableStreamQueueCheckpointer checkpoint entity.GetType().GetProperty("Offset")!.SetValue(entity, offset); } + private static object InvokeCreateWriteEntity( + AzureTableStreamQueueCheckpointer checkpointer, + string offset) + { + var method = typeof(AzureTableStreamQueueCheckpointer).GetMethod( + "CreateWriteEntity", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(method); + return method.Invoke(checkpointer, [offset])!; + } + + private static string GetEntityProperty(object entity, string propertyName) + => (string)entity.GetType().GetProperty(propertyName)!.GetValue(entity)!; + private static object GetField(AzureTableStreamQueueCheckpointer checkpointer, string name) { var field = typeof(AzureTableStreamQueueCheckpointer).GetField( From 933cc94c859d832dbbde13c8acd87bb5f1cc1bb6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 5 Sep 2026 15:37:09 -0700 Subject: [PATCH 14/30] fix(streaming): adapt checkpoint recovery to typed cursors --- .../EventHub/EventHubAdapterReceiver.cs | 483 +++++++++++++++--- .../EventHubCheckpointerTests.cs | 296 ++++++++++- 2 files changed, 708 insertions(+), 71 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index d56eba71841..2c50021000f 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -52,6 +52,7 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly IQueueAdapterReceiverMonitor monitor; private readonly LoadSheddingOptions loadSheddingOptions; private readonly IEnvironmentStatisticsProvider environmentStatisticsProvider; + private readonly object cacheLock = new(); private IEventHubQueueCache? cache; private IEventHubReceiver? receiver; @@ -59,7 +60,12 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly Func eventHubReceiverFactory; private IStreamQueueCheckpointer? checkpointer; - private AggregatedQueueFlowController flowController = null!; + private AggregatedQueueFlowController flowController = new(MaxMessagesPerRead); + private bool receiverUsesCheckpoint; + private IEventHubQueueCache? recoveryCache; + private Dictionary? recoveredCursorProgress; + private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance); + private HashSet? recoveryPendingCursors; // Receiver life cycle private int receiverState = ReceiverShutdown; @@ -69,7 +75,10 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa public int GetMaxAddCount() { - return this.flowController.GetMaxAddCount(); + lock (this.cacheLock) + { + return this.flowController.GetMaxAddCount(); + } } public EventHubAdapterReceiver(EventHubPartitionSettings settings, @@ -136,22 +145,36 @@ private async Task Initialize(CancellationToken cancellationToken) var watch = Stopwatch.StartNew(); try { - this.checkpointer = await this.checkpointerFactory( + var checkpointer = await this.checkpointerFactory( this.settings.Partition, cancellationToken); - if (this.cache != null) + string offset = await checkpointer.Load(cancellationToken); + var receiverUsesCheckpoint = checkpointer.CheckpointExists; + if (!receiverUsesCheckpoint) { - this.cache.Dispose(); - this.cache = null; + offset = EventHubConstants.StartOfStream; } - this.cache = this.cacheFactory(this.settings.Partition, this.checkpointer, this.loggerFactory); - this.flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) { this.cache, LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) }; - string offset = await this.checkpointer.Load(cancellationToken); - if (!this.checkpointer.CheckpointExists) + + var cache = this.cacheFactory(this.settings.Partition, checkpointer, this.loggerFactory); + var flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) { - offset = EventHubConstants.StartOfStream; + cache, + LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) + }; + + lock (this.cacheLock) + { + this.cache?.Dispose(); + this.checkpointer = checkpointer; + this.cache = cache; + this.flowController = flowController; + if (this.recoveredCursorProgress is not null) + { + this.recoveryCache = cache; + } } + this.receiverUsesCheckpoint = receiverUsesCheckpoint; this.receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger); watch.Stop(); this.monitor?.TrackInitialization(true, watch.Elapsed, null); @@ -209,9 +232,23 @@ public async Task> GetQueueMessagesAsync( this.monitor?.TrackRead(false, watch.Elapsed, ex); LogWarningFailedToReadFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition, ex); - if (ex is ArgumentException && this.checkpointer?.CheckpointExists == true) + if (this.receiverUsesCheckpoint && IsInvalidOffsetException(ex)) { - await ResetReceiver(cancellationToken); + try + { + await ResetReceiver(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception recoveryException) + { + LogWarningFailedToRecoverFromInvalidCheckpoint( + this.settings.Hub.EventHubName, + this.settings.Partition, + recoveryException); + } } throw; } @@ -231,7 +268,17 @@ public async Task> GetQueueMessagesAsync( this.monitor?.TrackMessagesReceived(messages.Count, oldestMessageEnqueueTime, newestMessageEnqueueTime); - List messageStreamPositions = this.cache!.Add(messages, dequeueTimeUtc); + List messageStreamPositions; + lock (this.cacheLock) + { + if (this.cache is null) + { + return batches; + } + + messageStreamPositions = this.cache.Add(messages, dequeueTimeUtc); + } + foreach (var streamPosition in messageStreamPositions) { batches.Add(new StreamActivityNotificationBatch(streamPosition)); @@ -239,30 +286,79 @@ public async Task> GetQueueMessagesAsync( return batches; } + private static bool IsInvalidOffsetException(Exception exception) + => exception is ArgumentException + && exception.Message.StartsWith("The supplied offset", StringComparison.OrdinalIgnoreCase) + && exception.Message.Contains(" is invalid.", StringComparison.OrdinalIgnoreCase); + private async Task ResetReceiver(CancellationToken cancellationToken) { - var checkpointer = Interlocked.Exchange(ref this.checkpointer, null); - var receiver = Interlocked.Exchange(ref this.receiver, null); - var cache = Interlocked.Exchange(ref this.cache, null); + IStreamQueueCheckpointer checkpointer; + lock (this.cacheLock) + { + checkpointer = this.checkpointer!; + this.recoveryCache = null; + this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); + this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); + } try { - await checkpointer!.Reset(cancellationToken); + await checkpointer.Reset(cancellationToken); } - finally + catch { - try + lock (this.cacheLock) { - cache?.Dispose(); + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; } - finally + + throw; + } + + this.receiverUsesCheckpoint = false; + var receiver = Interlocked.Exchange(ref this.receiver, null); + var exceptions = new List(); + try + { + if (receiver is not null) { - if (receiver is not null) - { - await receiver.CloseAsync(cancellationToken); - } + await receiver.CloseAsync(cancellationToken); } } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + exceptions.Add(exception); + } + + try + { + await Initialize(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + exceptions.Add(exception); + } + + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } + + if (exceptions.Count > 1) + { + throw new AggregateException(exceptions); + } } public void AddToCache(IList messages) @@ -274,10 +370,21 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList { purgedItems = null; - //if not under pressure, signal the cache to do a time based purge - //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache - if (!this.IsUnderPressure()) - this.cache!.SignalPurge(); + lock (this.cacheLock) + { + var cache = this.cache; + if (cache is null) + { + return false; + } + + //if not under pressure, signal the cache to do a time based purge + //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache + if (!this.IsUnderPressure()) + { + cache.SignalPurge(); + } + } return false; } @@ -285,27 +392,48 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList [Obsolete("Use IQueueCache.TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) { - return new Cursor(this.cache!, streamId, token); + return new Cursor(this, streamId, token); } QueueCacheCursorResult IQueueCache.TryGetCacheCursor( StreamId streamId, StreamSequenceToken? token) { - return WrapCursorResult(this.cache!.TryGetCursor(streamId, token)); + lock (this.cacheLock) + { + var cache = this.cache!; + return WrapCursorResult( + streamId, + cache, + cache.TryGetCursor(streamId, token), + ReferenceEquals(cache, this.recoveryCache) ? token : null); + } } QueueCacheCursorResult IQueueCache.TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) { - return WrapCursorResult(this.cache!.TryGetCursorAtPosition(streamId, startPosition)); + lock (this.cacheLock) + { + var cache = this.cache!; + return WrapCursorResult( + streamId, + cache, + cache.TryGetCursorAtPosition(streamId, startPosition), + resumeToken: null); + } } - private QueueCacheCursorResult WrapCursorResult(QueueCacheCursorResult result) + private QueueCacheCursorResult WrapCursorResult( + StreamId streamId, + IEventHubQueueCache cache, + QueueCacheCursorResult result, + StreamSequenceToken? resumeToken) => result.Kind switch { - QueueCacheCursorResultKind.Success => QueueCacheCursorResult.FromCursor(new Cursor(this.cache!, result.Cursor!)), + QueueCacheCursorResultKind.Success => QueueCacheCursorResult.FromCursor( + new Cursor(this, streamId, cache, result.Cursor!, resumeToken)), QueueCacheCursorResultKind.CacheMiss => QueueCacheCursorResult.FromCacheMiss(result.CacheMiss!.Value), QueueCacheCursorResultKind.NotSupported => QueueCacheCursorResult.NotSupported, _ => throw new InvalidOperationException("The cursor result is not initialized."), @@ -326,7 +454,66 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke if (earliestSubscriptionToken is IEventHubPartitionLocation location && long.TryParse(location.EventHubOffset, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) { - this.checkpointer?.Update(location.EventHubOffset, utcNow, CancellationToken.None); + IStreamQueueCheckpointer? checkpointer; + lock (this.cacheLock) + { + if (this.recoveredCursorProgress is { } recoveredProgress) + { + if (this.recoveryPendingCursors is { Count: > 0 } + || !recoveredProgress.Values.Any( + progress => ReferenceEquals(progress.DeliveredToken, earliestSubscriptionToken) + || ReferenceEquals(progress.ResumeToken, earliestSubscriptionToken))) + { + return; + } + + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + } + + checkpointer = this.checkpointer; + } + + checkpointer?.Update(location.EventHubOffset, utcNow, CancellationToken.None); + } + } + + private void RecordRecoveredDeliveryToken( + Cursor cursor, + IEventHubQueueCache? cursorCache, + StreamSequenceToken deliveredToken, + StreamSequenceToken? resumeToken) + { + if (this.recoveredCursorProgress is not { } cursorProgress + || !ReferenceEquals(cursorCache, this.recoveryCache)) + { + return; + } + + cursorProgress.TryGetValue(cursor, out var previousProgress); + cursorProgress[cursor] = new( + deliveredToken, + previousProgress.ResumeToken ?? resumeToken); + this.recoveryPendingCursors?.Remove(cursor); + } + + private void ClearRecoveredDeliveryToken(Cursor cursor) + { + this.recoveredCursorProgress?.Remove(cursor); + } + + private readonly record struct RecoveredCursorProgress( + StreamSequenceToken DeliveredToken, + StreamSequenceToken? ResumeToken); + + private void UnregisterCursor(Cursor cursor) + { + lock (this.cacheLock) + { + this.ClearRecoveredDeliveryToken(cursor); + this.recoveryPendingCursors?.Remove(cursor); + this.cursors.Remove(cursor); } } @@ -360,9 +547,7 @@ public async Task Shutdown(TimeSpan timeout) shutdownExceptions.Add(ex); } - // clear cache and receiver - IEventHubQueueCache? localCache = Interlocked.Exchange(ref this.cache, null); - + // clear receiver var localReceiver = Interlocked.Exchange(ref this.receiver, null); // start closing receiver @@ -384,7 +569,12 @@ public async Task Shutdown(TimeSpan timeout) // dispose of cache try { - localCache?.Dispose(); + lock (this.cacheLock) + { + var localCache = this.cache; + this.cache = null; + localCache?.Dispose(); + } } catch (Exception ex) { @@ -466,26 +656,51 @@ public StreamActivityNotificationBatch(StreamPosition position) private class Cursor : IQueueCacheCursor { - private readonly IEventHubQueueCache cache; - private readonly object cursor; + private readonly EventHubAdapterReceiver owner; + private readonly StreamId streamId; + private IEventHubQueueCache? cache; + private object? cursor; private IBatchContainer? current; + private StreamSequenceToken? recoveryResumeToken; + private QueueCacheCursorMoveResult? pendingMoveResult; - public Cursor(IEventHubQueueCache cache, StreamId streamId, StreamSequenceToken? token) + public Cursor(EventHubAdapterReceiver owner, StreamId streamId, StreamSequenceToken? token) { - this.cache = cache; + this.owner = owner; + this.streamId = streamId; + lock (owner.cacheLock) + { + this.cache = owner.cache!; #pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. - this.cursor = cache.GetCursor(streamId, token); + this.cursor = this.cache.GetCursor(streamId, token); #pragma warning restore CS0618 + if (ReferenceEquals(this.cache, owner.recoveryCache)) + { + this.recoveryResumeToken = token; + } + + owner.cursors.Add(this); + } } - public Cursor(IEventHubQueueCache cache, object cursor) + public Cursor( + EventHubAdapterReceiver owner, + StreamId streamId, + IEventHubQueueCache cache, + object cursor, + StreamSequenceToken? resumeToken) { + this.owner = owner; + this.streamId = streamId; this.cache = cache; this.cursor = cursor; + this.recoveryResumeToken = resumeToken; + owner.cursors.Add(this); } public void Dispose() { + this.owner.UnregisterCursor(this); } public IBatchContainer? GetCurrent(out Exception? exception) @@ -497,33 +712,176 @@ public void Dispose() [Obsolete("Use MoveNextWithResult instead.")] public bool MoveNext() { + lock (this.owner.cacheLock) + { + if (this.pendingMoveResult is { } pendingResult) + { + this.pendingMoveResult = null; + this.current = null; + if (pendingResult.CacheMiss is { } cacheMiss) + { + Invalidate(); + throw cacheMiss.ToException(); + } + + if (pendingResult.Kind == QueueCacheCursorMoveResultKind.NoData) + { + return false; + } + + throw new InvalidOperationException("The cursor move result is not initialized."); + } + + if (this.cache is null || this.cursor is null || !ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + return false; + } + + try + { #pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. - if (!this.cache.TryGetNextMessage(this.cursor, out var next)) + if (!this.cache.TryGetNextMessage(this.cursor, out var next)) #pragma warning restore CS0618 - { - this.current = null; - return false; - } + { + this.current = null; + return false; + } - this.current = next; - return true; + SetCurrent(next); + return true; + } + catch + { + Invalidate(); + throw; + } + } } public QueueCacheCursorMoveResult MoveNextWithResult() { - var result = this.cache.TryGetNextMessageWithResult(this.cursor, out var next); - this.current = result.Kind == QueueCacheCursorMoveResultKind.Success ? next : null; - return result; + lock (this.owner.cacheLock) + { + if (this.pendingMoveResult is { } pendingResult) + { + this.pendingMoveResult = null; + this.current = null; + if (pendingResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) + { + Invalidate(); + } + + return pendingResult; + } + + if (this.cache is null || this.cursor is null || !ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + return QueueCacheCursorMoveResult.NoData; + } + + QueueCacheCursorMoveResult result; + IBatchContainer? next; + try + { + result = this.cache.TryGetNextMessageWithResult(this.cursor, out next); + } + catch + { + Invalidate(); + throw; + } + + switch (result.Kind) + { + case QueueCacheCursorMoveResultKind.Success: + SetCurrent(next ?? throw new InvalidOperationException( + "A successful cursor move did not produce a current item.")); + break; + case QueueCacheCursorMoveResultKind.CacheMiss: + this.current = null; + Invalidate(); + break; + case QueueCacheCursorMoveResultKind.NoData: + this.current = null; + break; + default: + throw new InvalidOperationException("The cursor move result is not initialized."); + } + + return result; + } } - public void Refresh(StreamSequenceToken? token) + public void Refresh(StreamSequenceToken token) { - this.cache.Refresh(this.cursor, token); + lock (this.owner.cacheLock) + { + var cache = this.owner.cache; + if (cache is null) + { + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = null; + this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; + this.pendingMoveResult = null; + return; + } + + if (!ReferenceEquals(this.cache, cache)) + { + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = cache; + this.current = null; + this.recoveryResumeToken = null; + var result = cache.TryGetCursor(this.streamId, token); + switch (result.Kind) + { + case QueueCacheCursorResultKind.Success: + this.cursor = result.Cursor; + this.pendingMoveResult = null; + break; + case QueueCacheCursorResultKind.CacheMiss: + this.cursor = null; + this.pendingMoveResult = QueueCacheCursorMoveResult.FromCacheMiss(result.CacheMiss!.Value); + break; + default: + throw new InvalidOperationException("The cursor result is not initialized."); + } + + return; + } + + cache.Refresh(this.cursor!, token); + } } public void RecordDeliveryFailure() { } + + private void SetCurrent(IBatchContainer next) + { + this.current = next; + this.owner.RecordRecoveredDeliveryToken( + this, + this.cache, + next.SequenceToken, + this.recoveryResumeToken); + this.recoveryResumeToken = null; + } + + private void Invalidate() + { + this.owner.UnregisterCursor(this); + this.cache = null; + this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; + this.pendingMoveResult = null; + } } [LoggerMessage( @@ -540,7 +898,7 @@ public void RecordDeliveryFailure() [LoggerMessage( Level = LogLevel.Warning, - EventId = (int)OrleansEventHubErrorCode.FailedPartitionRead, + EventId = (int)OrleansEventHubErrorCode.RetryReceiverInit, Message = "Retrying initialization of EventHub partition {EventHubName}-{Partition}." )] private partial void LogWarningRetryingInitializationOfEventHubPartition(string eventHubName, string partition); @@ -551,5 +909,12 @@ public void RecordDeliveryFailure() Message = "Failed to read from EventHub partition {EventHubName}-{Partition}" )] private partial void LogWarningFailedToReadFromEventHubPartition(string eventHubName, string partition, Exception exception); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = (int)OrleansEventHubErrorCode.RetryReceiverInit, + Message = "Failed to recover EventHub partition {EventHubName}-{Partition} from an invalid checkpoint. The original read failure will be rethrown." + )] + private partial void LogWarningFailedToRecoverFromInvalidCheckpoint(string eventHubName, string partition, Exception exception); } } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index a96182e1b5e..fab3c5a8116 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -73,6 +73,15 @@ public override Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class FailingResetCheckpointer : TestCheckpointer + { + public override Task Reset(CancellationToken cancellationToken) + { + _ = base.Reset(cancellationToken); + throw new InvalidOperationException("Reset failed"); + } + } + private class TestEventHubQueueCache : IEventHubQueueCache { private readonly IStreamQueueCheckpointer? checkpointer; @@ -89,19 +98,30 @@ public TestEventHubQueueCache(IStreamQueueCheckpointer? checkpointer = n public object? RefreshedCursor { get; private set; } public StreamSequenceToken? RefreshToken { get; private set; } public Exception? CursorException { get; set; } - public QueueCacheMissException? MoveNextException { get; set; } + public Exception? MoveNextException { get; set; } public IBatchContainer? NoDataMessage { get; set; } + public int GetCursorCount { get; private set; } + public IBatchContainer? NextMessage { get; set; } + public List PositionsToReturn { get; init; } = []; public int GetMaxAddCount() => 1_000; public List Add(List message, DateTime dequeueTimeUtc) { AddCount++; - return []; + return PositionsToReturn; } public object GetCursor(StreamId streamId, StreamSequenceToken? sequenceToken) - => CursorException is { } exception ? throw exception : Cursor; + { + if (CursorException is { } exception) + { + throw exception; + } + + GetCursorCount++; + return Cursor; + } public void Refresh(object cursor, StreamSequenceToken? sequenceToken) { @@ -116,6 +136,13 @@ public bool TryGetNextMessage(object cursorObj, out IBatchContainer message) throw exception; } + if (NextMessage is { } next) + { + NextMessage = null; + message = next; + return true; + } + message = NoDataMessage!; return false; } @@ -141,10 +168,11 @@ public void Dispose() private sealed class TestEventHubReceiver : IEventHubReceiver { public int CloseCount { get; private set; } + public IEnumerable Messages { get; init; } = []; public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) { - return Task.FromResult>([]); + return Task.FromResult(Messages); } public Task CloseAsync() @@ -154,6 +182,16 @@ public Task CloseAsync() } } + private sealed class TestBatchContainer : IBatchContainer + { + public StreamId StreamId { get; } = StreamId.Create("namespace", Guid.NewGuid()); + public StreamSequenceToken SequenceToken { get; init; } = MakeToken(1); + + public IEnumerable> GetEvents() => []; + + public bool ImportRequestContext() => false; + } + [Fact, TestCategory("BVT")] public void GetCursorPreservesLegacyEventHubCacheBehavior() { @@ -319,6 +357,20 @@ public Task CloseAsync() } } + private sealed class InvalidArgumentEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => throw new ArgumentException("The receiver rejected an unrelated argument."); + + public Task CloseAsync() + { + CloseCount++; + return Task.CompletedTask; + } + } + private sealed class CancellableEventHubReceiver : IEventHubReceiver { public TaskCompletionSource ReceiveStarted { get; } = new( @@ -363,6 +415,17 @@ private static EventHubSequenceToken MakeToken(long offset, long sequenceNumber return new EventHubSequenceToken(offset.ToString(), sequenceNumber, 0); } + private static IQueueCacheCursor AssertCursor( + EventHubAdapterReceiver receiver, + StreamId streamId, + StreamSequenceToken token) + { + var result = ((IQueueCache)receiver).TryGetCacheCursor(streamId, token); + Assert.Equal(QueueCacheCursorResultKind.Success, result.Kind); + Assert.NotNull(result.Cursor); + return result.Cursor; + } + private static void UpdateDeliveryProgress(EventHubAdapterReceiver receiver, StreamSequenceToken token) { receiver.UpdateDeliveryProgress(token, DateTime.UtcNow); @@ -378,7 +441,9 @@ private static async Task CreateReceiver( TestEventHubQueueCache? cache = null, IEventHubReceiver? eventHubReceiver = null, Action? onReceiverCreated = null, - Func? receiverFactory = null) + Func? receiverFactory = null, + Func? cacheFactory = null, + bool initialize = true) { var settings = new EventHubPartitionSettings { @@ -394,7 +459,7 @@ private static async Task CreateReceiver( var receiver = new EventHubAdapterReceiver( settings, - cacheFactory: (_, createdCheckpointer, _) => cache ?? new TestEventHubQueueCache(createdCheckpointer), + cacheFactory: (_, createdCheckpointer, _) => cacheFactory?.Invoke() ?? cache ?? new TestEventHubQueueCache(createdCheckpointer), checkpointerFactory: _ => Task.FromResult>(checkpointer), loggerFactory: Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance, monitor: new Orleans.Streaming.EventHubs.DefaultEventHubReceiverMonitor( @@ -412,11 +477,23 @@ private static async Task CreateReceiver( return receiverFactory?.Invoke(offset) ?? eventHubReceiver ?? new TestEventHubReceiver(); }); - await receiver.Initialize(TimeSpan.FromSeconds(5)); + if (initialize) + { + await receiver.Initialize(TimeSpan.FromSeconds(5)); + } return receiver; } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetMaxAddCount_BeforeInitialize_UsesSafeDefault() + { + var receiver = await CreateReceiver(new TestCheckpointer(), initialize: false); + + Assert.Equal(EventHubAdapterReceiver.MaxMessagesPerRead, receiver.GetMaxAddCount()); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Initialize_WhenCheckpointDoesNotExist_UsesStartOfStream() @@ -458,23 +535,217 @@ public async Task GetQueueMessagesAsync_WhenCheckpointOffsetIsInvalid_ResetsAndR { LoadedOffset = "123", }; - var cache = new TestEventHubQueueCache(); + var initialBatch = new TestBatchContainer(); + var initialCache = new TestEventHubQueueCache + { + NextMessage = initialBatch, + }; + var recoveredToken = MakeToken(10); + var recoveredBatch = new TestBatchContainer + { + SequenceToken = recoveredToken, + }; + var replacementCache = new TestEventHubQueueCache + { + NextMessage = recoveredBatch, + PositionsToReturn = + [ + new StreamPosition( + StreamId.Create("namespace", Guid.NewGuid()), + recoveredToken), + ], + }; + var replacementReceiver = new TestEventHubReceiver + { + Messages = [new EventData()], + }; + var caches = new Queue([initialCache, replacementCache]); var invalidReceiver = new InvalidOffsetEventHubReceiver(); var offsets = new List(); var receiver = await CreateReceiver( checkpointer, - cache, onReceiverCreated: offsets.Add, - receiverFactory: offset => offset == "123" ? invalidReceiver : new TestEventHubReceiver()); + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver, + cacheFactory: caches.Dequeue); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var abandonedCursor = AssertCursor(receiver, streamId, MakeToken(1)); + initialCache.MoveNextException = new InvalidOperationException("Cache cursor failed"); + Assert.Throws(() => abandonedCursor.MoveNextWithResult()); + initialCache.MoveNextException = null; + + var cursor = AssertCursor(receiver, streamId, MakeToken(1)); + var pendingCursor = AssertCursor(receiver, StreamId.Create("namespace", Guid.NewGuid()), MakeToken(1)); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(initialBatch, cursor.GetCurrent(out _)); await Assert.ThrowsAsync( () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); Assert.Equal(1, checkpointer.ResetCount); Assert.Equal(1, invalidReceiver.CloseCount); - Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, initialCache.DisposeCount); + + receiver.UpdateDeliveryProgress(MakeToken(123), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); - Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Single(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); Assert.Equal(["123", EventHubConstants.StartOfStream], offsets); + + var refreshToken = MakeToken(2); + cursor.Refresh(refreshToken); + Assert.Equal(1, replacementCache.GetCursorCount); + Assert.Null(cursor.GetCurrent(out _)); + + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(recoveredBatch, cursor.GetCurrent(out _)); + receiver.UpdateDeliveryProgress(recoveredToken, DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + pendingCursor.Dispose(); + receiver.UpdateDeliveryProgress(recoveredToken, DateTime.UtcNow); + Assert.Equal("10", checkpointer.LastOffset); + Assert.Equal(1, checkpointer.UpdateCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenArgumentFailureIsNotAnInvalidOffset_DoesNotReset() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var eventHubReceiver = new InvalidArgumentEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: eventHubReceiver, + onReceiverCreated: offsets.Add); + + var exception = await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal("The receiver rejected an unrelated argument.", exception.Message); + Assert.Equal(0, checkpointer.ResetCount); + Assert.Equal(0, eventHubReceiver.CloseCount); + Assert.Equal(["123"], offsets); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_AcceptsResumeTokenResolvedByReplacementCache() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var recoveredToken = MakeToken(10); + var replacementCache = new TestEventHubQueueCache + { + NextMessage = new TestBatchContainer + { + SequenceToken = recoveredToken, + }, + PositionsToReturn = + [ + new StreamPosition( + StreamId.Create("namespace", Guid.NewGuid()), + recoveredToken), + ], + }; + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver { Messages = [new EventData()] }, + cacheFactory: caches.Dequeue); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Single(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + var resumeToken = MakeToken(9); + using var cursor = AssertCursor( + receiver, + StreamId.Create("namespace", Guid.NewGuid()), + resumeToken); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + + receiver.UpdateDeliveryProgress(MakeToken(9), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + receiver.UpdateDeliveryProgress(resumeToken, DateTime.UtcNow); + Assert.Equal("9", checkpointer.LastOffset); + Assert.Equal(1, checkpointer.UpdateCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorRefresh_WhenReplacementCacheMisses_LegacyMoveNextThrows() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var replacementCache = new TestEventHubQueueCache + { + CursorException = new QueueCacheMissException("requested", "low", "high"), + }; + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor APIs. + var cursor = ((IQueueCache)receiver).GetCacheCursor( + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); +#pragma warning restore CS0618 + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + cursor.Refresh(MakeToken(2)); + +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor APIs. + var exception = Assert.Throws(() => cursor.MoveNext()); +#pragma warning restore CS0618 + Assert.Equal("requested", exception.Requested); + Assert.Equal("low", exception.Low); + Assert.Equal("high", exception.High); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsReadFailure() + { + var checkpointer = new FailingResetCheckpointer + { + LoadedOffset = "123", + }; + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: new InvalidOffsetEventHubReceiver()); + + var exception = await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + + Assert.Equal("The supplied offset is invalid.", exception.Message); + Assert.Equal(1, checkpointer.ResetCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task NoOpCheckpointerFactory_CreateHonorsCancellation() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => NoOpCheckpointerFactory.Instance.Create("partition", cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); } [TestSuite("BVT")] @@ -488,6 +759,7 @@ public async Task GetQueueMessagesAsync_WhenNoCheckpointExists_DoesNotResetArgum }; var invalidReceiver = new InvalidOffsetEventHubReceiver(); var receiver = await CreateReceiver(checkpointer, eventHubReceiver: invalidReceiver); + checkpointer.CheckpointExists = true; await Assert.ThrowsAsync( () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); From 5520a1274f7e7a792047f15721156ce9b4331367 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 7 Sep 2026 02:13:38 -0700 Subject: [PATCH 15/30] fix(streaming): distinguish checkpoint recovery failures --- .../Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs | 1 + .../Providers/Streams/EventHub/EventHubAdapterReceiver.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs index 63a904a0cbc..e62c2461618 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs @@ -12,5 +12,6 @@ internal enum OrleansEventHubErrorCode FailedPartitionRead = ServiceBus + 1, RetryReceiverInit = ServiceBus + 2, + FailedPartitionRecovery = ServiceBus + 3, } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 2c50021000f..ea9b0f7eb6a 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -912,7 +912,7 @@ private void Invalidate() [LoggerMessage( Level = LogLevel.Warning, - EventId = (int)OrleansEventHubErrorCode.RetryReceiverInit, + EventId = (int)OrleansEventHubErrorCode.FailedPartitionRecovery, Message = "Failed to recover EventHub partition {EventHubName}-{Partition} from an invalid checkpoint. The original read failure will be rethrown." )] private partial void LogWarningFailedToRecoverFromInvalidCheckpoint(string eventHubName, string partition, Exception exception); From b98ce8b2084d35fe9f6c8becdde1dcec020f45ff Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 7 Sep 2026 03:01:13 -0700 Subject: [PATCH 16/30] fix(streaming): preserve pending cache misses --- .../Providers/Streams/EventHub/EventHubAdapterReceiver.cs | 6 ++++++ .../CheckpointerTests/EventHubCheckpointerTests.cs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index ea9b0f7eb6a..7a84360d353 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -854,6 +854,12 @@ public void Refresh(StreamSequenceToken token) return; } + if (this.cursor is null + && this.pendingMoveResult is { Kind: QueueCacheCursorMoveResultKind.CacheMiss }) + { + return; + } + cache.Refresh(this.cursor!, token); } } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index fab3c5a8116..6d24753f32a 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -97,6 +97,7 @@ public TestEventHubQueueCache(IStreamQueueCheckpointer? checkpointer = n public object Cursor { get; } = new(); public object? RefreshedCursor { get; private set; } public StreamSequenceToken? RefreshToken { get; private set; } + public int RefreshCount { get; private set; } public Exception? CursorException { get; set; } public Exception? MoveNextException { get; set; } public IBatchContainer? NoDataMessage { get; set; } @@ -125,6 +126,7 @@ public object GetCursor(StreamId streamId, StreamSequenceToken? sequenceToken) public void Refresh(object cursor, StreamSequenceToken? sequenceToken) { + RefreshCount++; RefreshedCursor = cursor; RefreshToken = sequenceToken; } @@ -707,6 +709,8 @@ public async Task CursorRefresh_WhenReplacementCacheMisses_LegacyMoveNextThrows( await Assert.ThrowsAsync( () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); cursor.Refresh(MakeToken(2)); + cursor.Refresh(MakeToken(3)); + Assert.Equal(0, replacementCache.RefreshCount); #pragma warning disable CS0618 // Verify compatibility of the obsolete cursor APIs. var exception = Assert.Throws(() => cursor.MoveNext()); From 065ce002af308ae2baade15055c0e193acc85ba5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 14:36:10 -0700 Subject: [PATCH 17/30] fix(streaming): reconcile checkpoint resets with main --- .../AzureTableStreamQueueCheckpointer.cs | 145 +++++++++++----- .../GrainStreamQueueCheckpointer.cs | 68 +------- .../Checkpointers/StreamQueueCheckpointer.cs | 155 ++++++++++++++++++ .../Orleans.Streaming/Orleans.Streaming.cs | 5 + .../EventHubCheckpointerTests.cs | 37 +++++ .../ReusableStreamQueueCheckpointerTests.cs | 92 +++++++++++ .../StreamQueueCheckpointerTests.cs | 143 ++++++++++++++++ 7 files changed, 542 insertions(+), 103 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index fb6842b0d8b..cbc9c84d38d 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -23,6 +23,8 @@ public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointe private DateTime? _throttleSavesUntilUtc; private string _latestCheckpoint = string.Empty; private string _persistedCheckpoint = string.Empty; + private int _pendingResetCount; + private long _updateGeneration; private AzureTableStreamQueueCheckpointer( AzureTableStreamCheckpointerOptions options, @@ -165,50 +167,48 @@ public async Task Load(CancellationToken cancellationToken) public Task Reset() => Reset(CancellationToken.None); /// - public async Task Reset(CancellationToken cancellationToken) + public Task Reset(CancellationToken cancellationToken) { Task resetTask; - Task inProgressSave; - string latestCheckpoint; - DateTime? throttleSavesUntilUtc; lock (_lock) { - inProgressSave = _inProgressSave; - latestCheckpoint = _latestCheckpoint; - throttleSavesUntilUtc = _throttleSavesUntilUtc; - _latestCheckpoint = string.Empty; + _pendingResetCount++; _throttleSavesUntilUtc = DateTime.MaxValue; - resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); + resetTask = _inProgressSave = RunReset( + _inProgressSave, + _latestCheckpoint, + _updateGeneration, + cancellationToken); + resetTask.Ignore(); } + return resetTask.WaitAsync(cancellationToken); + } + + private async Task RunReset( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { try { - await resetTask; + await ResetCore( + inProgressSave, + enqueuedCheckpoint, + updateGeneration, + cancellationToken); } - catch + finally { lock (_lock) { - if (ReferenceEquals(resetTask, _inProgressSave)) + _pendingResetCount--; + if (_pendingResetCount == 0) { - _latestCheckpoint = latestCheckpoint; - _throttleSavesUntilUtc = throttleSavesUntilUtc; - _inProgressSave = inProgressSave; + _throttleSavesUntilUtc = null; } } - - throw; - } - - lock (_lock) - { - if (ReferenceEquals(resetTask, _inProgressSave)) - { - _latestCheckpoint = string.Empty; - _persistedCheckpoint = string.Empty; - _throttleSavesUntilUtc = null; - _inProgressSave = Task.CompletedTask; - } } } @@ -219,15 +219,30 @@ public void Update(string offset, DateTime utcNow) lock (_lock) { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_checkpointComparer is { } comparer + if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal)) + { + if (string.Equals(_persistedCheckpoint, offset, StringComparison.Ordinal) + || !_inProgressSave.IsCompleted + || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) + { + return; + } + + _throttleSavesUntilUtc = utcNow + _persistInterval; + _inProgressSave = Save(offset, CancellationToken.None); + _inProgressSave.Ignore(); + return; + } + + if (_checkpointComparer is { } comparer && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) + && comparer.Compare(offset, _latestCheckpoint) <= 0) { return; } _latestCheckpoint = offset; + _updateGeneration++; _entity.Offset = offset; if (!_inProgressSave.IsCompleted || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) @@ -284,6 +299,7 @@ public async Task FlushAsync(CancellationToken cancellationToken) } _inProgressSave = Save(_latestCheckpoint, cancellationToken); + _inProgressSave.Ignore(); retryingSave = true; } } @@ -300,22 +316,77 @@ private async Task Save(string checkpoint, CancellationToken cancellationToken) } } - private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) + private async Task ResetCore( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) { try { - await inProgressSave.WaitAsync(cancellationToken); + await inProgressSave; } - catch (Exception) when (!cancellationToken.IsCancellationRequested) + catch { } cancellationToken.ThrowIfCancellationRequested(); - var entity = CreateWriteEntity(string.Empty); - await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); + + string rollbackCheckpoint; lock (_lock) { - _entity.Offset = string.Empty; + if (_updateGeneration <= updateGeneration) + { + rollbackCheckpoint = _latestCheckpoint; + _latestCheckpoint = string.Empty; + } + else + { + rollbackCheckpoint = enqueuedCheckpoint; + } + } + + try + { + var entity = CreateWriteEntity(string.Empty); + await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); + lock (_lock) + { + _persistedCheckpoint = string.Empty; + if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _entity.Offset = string.Empty; + } + } + } + catch + { + lock (_lock) + { + RestoreLatestCheckpoint(rollbackCheckpoint); + } + + throw; + } + } + + private void RestoreLatestCheckpoint(string latestCheckpoint) + { + if (_checkpointComparer is { } comparer + && !string.IsNullOrEmpty(_latestCheckpoint) + && !string.IsNullOrEmpty(latestCheckpoint)) + { + if (comparer.Compare(_latestCheckpoint, latestCheckpoint) < 0) + { + _latestCheckpoint = latestCheckpoint; + } + + return; + } + + if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _latestCheckpoint = latestCheckpoint; } } diff --git a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs index 229a3eba02a..98e4a7ec21d 100644 --- a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs @@ -178,47 +178,8 @@ internal static string GetConfiguredStorageProviderName(ReadOnlySpan grain public Task Reset() => Reset(CancellationToken.None); /// - public async Task Reset(CancellationToken cancellationToken) - { - Task resetTask; - lock (_lock) - { - var inProgressSave = _inProgressSave; - _latestCheckpoint = string.Empty; - _throttleSavesUntilUtc = DateTime.MaxValue; - resetTask = _inProgressSave = ResetCore(inProgressSave, cancellationToken); - } - - try - { - await resetTask; - } - catch - { - lock (_lock) - { - if (ReferenceEquals(resetTask, _inProgressSave)) - { - _latestCheckpoint = _persistedCheckpoint; - _throttleSavesUntilUtc = null; - _inProgressSave = Task.CompletedTask; - } - } - - throw; - } - - lock (_lock) - { - if (ReferenceEquals(resetTask, _inProgressSave)) - { - _latestCheckpoint = string.Empty; - _persistedCheckpoint = string.Empty; - _throttleSavesUntilUtc = null; - _inProgressSave = Task.CompletedTask; - } - } - } + public Task Reset(CancellationToken cancellationToken) + => _inner.Reset(cancellationToken); /// [Obsolete("Use the overload which accepts a CancellationToken.")] @@ -253,30 +214,5 @@ public async ValueTask Update( return new(persistedCheckpoint, persistedCheckpoint); } } - - private async Task ResetCore(Task inProgressSave, CancellationToken cancellationToken) - { - await inProgressSave.WaitAsync(cancellationToken); - - string expectedCheckpoint; - lock (_lock) - { - expectedCheckpoint = _persistedCheckpoint; - } - - while (true) - { - var persistedCheckpoint = await _grain.Update( - string.Empty, - expectedCheckpoint, - cancellationToken); - if (string.IsNullOrEmpty(persistedCheckpoint)) - { - return; - } - - expectedCheckpoint = persistedCheckpoint; - } - } } } diff --git a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs index 2cc967877c2..c027fcc909b 100644 --- a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs @@ -17,6 +17,8 @@ public sealed class StreamQueueCheckpointer : IStreamQueueCheckpointer private StreamCheckpointStoreState _persistedState = new(string.Empty, string.Empty); private Task _inProgressSave = Task.CompletedTask; private DateTime? _throttleSavesUntilUtc; + private int _pendingResetCount; + private long _updateGeneration; /// /// Initializes a new instance of the class. @@ -68,6 +70,56 @@ public async Task Load(CancellationToken cancellationToken) return state.Checkpoint; } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) + { + Task resetTask; + lock (_lock) + { + _pendingResetCount++; + _throttleSavesUntilUtc = DateTime.MaxValue; + resetTask = _inProgressSave = RunReset( + _inProgressSave, + _latestCheckpoint, + _updateGeneration, + cancellationToken); + resetTask.Ignore(); + } + + return resetTask.WaitAsync(cancellationToken); + } + + private async Task RunReset( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await ResetCore( + inProgressSave, + enqueuedCheckpoint, + updateGeneration, + cancellationToken); + } + finally + { + lock (_lock) + { + _pendingResetCount--; + if (_pendingResetCount == 0) + { + _throttleSavesUntilUtc = null; + } + } + } + } + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) @@ -104,6 +156,7 @@ public void Update(string offset, DateTime utcNow, CancellationToken cancellatio } _latestCheckpoint = offset; + _updateGeneration++; if (_throttleSavesUntilUtc.HasValue && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) { @@ -159,6 +212,7 @@ public async Task FlushAsync(CancellationToken cancellationToken) } _inProgressSave = Save(_latestCheckpoint, cancellationToken); + _inProgressSave.Ignore(); retryingSave = true; } } @@ -205,6 +259,107 @@ private async Task Save(string checkpoint, CancellationToken cancellationToken) } } + private async Task ResetCore( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await inProgressSave; + } + catch + { + } + + cancellationToken.ThrowIfCancellationRequested(); + + string rollbackCheckpoint; + string expectedVersion; + string persistedVersion; + lock (_lock) + { + persistedVersion = expectedVersion = _persistedState.Version; + if (_updateGeneration <= updateGeneration) + { + rollbackCheckpoint = _latestCheckpoint; + _latestCheckpoint = string.Empty; + } + else + { + rollbackCheckpoint = enqueuedCheckpoint; + } + } + + try + { + while (true) + { + var persistedState = await _store.Update(string.Empty, expectedVersion, cancellationToken); + lock (_lock) + { + _persistedState = persistedState; + } + + if (string.IsNullOrEmpty(persistedState.Checkpoint)) + { + return; + } + + expectedVersion = persistedState.Version; + } + } + catch + { + lock (_lock) + { + RestoreLatestCheckpoint(rollbackCheckpoint, persistedVersion); + } + + throw; + } + } + + private void RestoreLatestCheckpoint(string latestCheckpoint, string persistedVersion) + { + if (string.Equals(_persistedState.Version, persistedVersion, StringComparison.Ordinal)) + { + if (_options.CheckpointComparer is { } checkpointComparer) + { + if (Compare(checkpointComparer, _latestCheckpoint, latestCheckpoint) < 0) + { + _latestCheckpoint = latestCheckpoint; + } + } + else if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _latestCheckpoint = latestCheckpoint; + } + + return; + } + + if (_options.CheckpointComparer is not { } comparer) + { + _latestCheckpoint = _persistedState.Checkpoint; + return; + } + + var candidate = _latestCheckpoint; + if (Compare(comparer, candidate, latestCheckpoint) < 0) + { + candidate = latestCheckpoint; + } + + if (Compare(comparer, candidate, _persistedState.Checkpoint) < 0) + { + candidate = _persistedState.Checkpoint; + } + + _latestCheckpoint = candidate; + } + private static int Compare(IComparer comparer, string left, string right) { if (string.IsNullOrEmpty(left)) diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 90c0d2d7d9c..b9df1a0cd64 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -2338,6 +2338,11 @@ public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpoi public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 6d24753f32a..da4fc3def54 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -947,6 +947,32 @@ public void Update_WithNoComparer_TracksOpaqueCheckpoint() Assert.Equal("opaque-checkpoint", GetEntityOffset(checkpointer)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void Update_WhenLatestCheckpointIsNotPersisted_RetriesSave() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetField(checkpointer, "_latestCheckpoint", "20"); + SetField(checkpointer, "_persistedCheckpoint", string.Empty); + var previousSave = GetField(checkpointer, "_inProgressSave"); + + checkpointer.Update("20", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + + Assert.NotSame(previousSave, GetField(checkpointer, "_inProgressSave")); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void AzureTableCheckpointer_FailedResetRejectsRegressedConcurrentUpdate() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetField(checkpointer, "_latestCheckpoint", "50"); + + InvokeRestoreLatestCheckpoint(checkpointer, "100"); + + Assert.Equal("100", GetLatestOffset(checkpointer)); + } + [TestSuite("BVT")] [Theory, TestCategory("BVT")] [InlineData("", "provider_service")] @@ -1268,6 +1294,17 @@ private static object InvokeCreateWriteEntity( return method.Invoke(checkpointer, [offset])!; } + private static void InvokeRestoreLatestCheckpoint( + AzureTableStreamQueueCheckpointer checkpointer, + string offset) + { + var method = typeof(AzureTableStreamQueueCheckpointer).GetMethod( + "RestoreLatestCheckpoint", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(method); + method.Invoke(checkpointer, [offset]); + } + private static string GetEntityProperty(object entity, string propertyName) => (string)entity.GetType().GetProperty(propertyName)!.GetValue(entity)!; diff --git a/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs index a1a3d54ecea..9861382e55c 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs @@ -112,6 +112,49 @@ public async Task ConditionalConflict_WithoutComparerAdoptsAuthoritativeCheckpoi Assert.Equal("40", (await store.Load(CancellationToken.None)).Checkpoint); } + [Fact] + public async Task Reset_WhenConflictAdvancesCheckpointThenRetryFails_PreservesAuthoritativeCheckpoint() + { + var store = new ResetConflictThenFailureStore(); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal("10", await checkpointer.Load(CancellationToken.None)); + + await Assert.ThrowsAsync( + () => checkpointer.Reset(CancellationToken.None)); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal([("version-1", string.Empty), ("version-2", string.Empty)], store.Attempts); + Assert.Equal("20", (await store.Load(CancellationToken.None)).Checkpoint); + } + + [Fact] + public async Task Reset_WhenConflictAndConcurrentUpdateRegress_PreservesLargestCheckpoint() + { + var store = new ResetConflictThenFailureStore("100", "80"); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal("100", await checkpointer.Load(CancellationToken.None)); + store.OnConflict = () => checkpointer.Update("50", DateTime.UtcNow, CancellationToken.None); + + await Assert.ThrowsAsync( + () => checkpointer.Reset(CancellationToken.None)); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal( + [("version-1", string.Empty), ("version-2", string.Empty), ("version-2", "100")], + store.Attempts); + Assert.Equal("100", (await store.Load(CancellationToken.None)).Checkpoint); + } + private sealed class TestCheckpointStore(ControllableCheckpointStore store) : IStreamCheckpointStore { public async ValueTask Load(CancellationToken cancellationToken) @@ -167,6 +210,55 @@ public ValueTask Update( } } + private sealed class ResetConflictThenFailureStore : IStreamCheckpointStore + { + private StreamCheckpointStoreState state; + private readonly string conflictCheckpoint; + private int updateCount; + + public List<(string ExpectedVersion, string Checkpoint)> Attempts { get; } = []; + + public Action? OnConflict { get; set; } + + public ResetConflictThenFailureStore( + string initialCheckpoint = "10", + string conflictCheckpoint = "20") + { + state = new(initialCheckpoint, "version-1"); + this.conflictCheckpoint = conflictCheckpoint; + } + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(state); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Attempts.Add((expectedVersion, checkpoint)); + updateCount++; + if (updateCount == 1) + { + state = new(conflictCheckpoint, "version-2"); + OnConflict?.Invoke(); + return ValueTask.FromResult(state); + } + + if (updateCount == 2) + { + throw new InvalidOperationException("checkpoint reset failed"); + } + + state = new(checkpoint, "version-3"); + return ValueTask.FromResult(state); + } + } + private sealed class EmptyCheckpointConflictStore : IStreamCheckpointStore { private StreamCheckpointStoreState state = new(string.Empty, "version-1"); diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index e83129129de..8039615559f 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -115,6 +115,149 @@ public async Task Reset_AfterPendingWriteFails_ClearsCheckpoint() Assert.False(checkpointer.CheckpointExists); } + [Fact] + public async Task Reset_WhenWriteFails_PreservesConcurrentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("checkpoint reset failed")); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => reset); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal([NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal(["20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenWriteFails_ReconcilesConcurrentRegressedUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("100"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("checkpoint reset failed")); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("50", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => reset); + await checkpointer.FlushAsync(CancellationToken.None); + + var expected = RegressionPolicy is OffsetRegressionPolicy.Ignore ? "100" : "50"; + Assert.Equal(expected, store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenNestedResetFails_ClearsThrottleMarker() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + store.FailNextWrite(new InvalidOperationException("nested checkpoint reset failed")); + var secondReset = checkpointer.Reset(CancellationToken.None); + + blockedReset.SetResult(); + await firstReset; + await Assert.ThrowsAsync(() => secondReset); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForCompletedWrites(2).WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal([NoCheckpoint, NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenUpdatePrecedesFailedNestedReset_RestoresUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + store.FailNextWrite(new InvalidOperationException("nested checkpoint reset failed")); + var secondReset = checkpointer.Reset(CancellationToken.None); + + blockedReset.SetResult(); + await firstReset; + await Assert.ThrowsAsync(() => secondReset); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.True(checkpointer.CheckpointExists); + Assert.Equal([NoCheckpoint, NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenAllNestedResetsFail_RestoresCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("first checkpoint reset failed")); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + var secondReset = checkpointer.Reset(CancellationToken.None); + store.FailNextWrite(new InvalidOperationException("second checkpoint reset failed")); + + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => firstReset); + await Assert.ThrowsAsync(() => secondReset); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.True(checkpointer.CheckpointExists); + Assert.Equal([NoCheckpoint, NoCheckpoint], store.WriteAttempts); + Assert.Empty(store.CompletedWrites); + Assert.Equal("10", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenUpdateAndFlushOverlap_PersistsSubsequentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var resetWrite = store.BlockNextWrite(); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + var flush = checkpointer.FlushAsync(CancellationToken.None); + resetWrite.SetResult(); + + await Task.WhenAll(reset, flush); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + + checkpointer.Update("30", TestTimeUtc + PersistInterval, CancellationToken.None); + await store.WaitForCompletedWrites(3).WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal([NoCheckpoint, "20", "30"], store.CompletedWrites); + Assert.Equal("30", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenQueuedBehindSave_PreservesSubsequentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedWrite = store.BlockNextWrite(); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForWriteAttempts(1); + var reset = checkpointer.Reset(CancellationToken.None); + + checkpointer.Update("30", TestTimeUtc + PersistInterval, CancellationToken.None); + blockedWrite.SetResult(); + await reset; + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(["20", NoCheckpoint, "30"], store.WriteAttempts); + Assert.Equal(["20", NoCheckpoint, "30"], store.CompletedWrites); + Assert.Equal("30", store.PersistedCheckpoint); + } + [Fact] public async Task Update_PersistsCheckpoint() { From 3d81a0db6867789b5f55fb86a0fbbd03f7c56756 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 08:00:48 -0700 Subject: [PATCH 18/30] fix(streaming): close checkpoint recovery races --- .../AzureTableStreamQueueCheckpointer.cs | 107 ++++++++- .../EventHub/EventHubAdapterReceiver.cs | 126 ++++++++--- .../Checkpointers/StreamQueueCheckpointer.cs | 107 ++++++++- .../EventHubCheckpointerTests.cs | 204 +++++++++++++++++- .../StreamQueueCheckpointerTests.cs | 18 +- 5 files changed, 511 insertions(+), 51 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index cbc9c84d38d..5bf056faaf8 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -169,24 +169,31 @@ public async Task Load(CancellationToken cancellationToken) /// public Task Reset(CancellationToken cancellationToken) { - Task resetTask; + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); lock (_lock) { _pendingResetCount++; _throttleSavesUntilUtc = DateTime.MaxValue; - resetTask = _inProgressSave = RunReset( + var inProgressSave = _inProgressSave; + _inProgressSave = completion.Task; + RunReset( + completion, + inProgressSave, _inProgressSave, _latestCheckpoint, _updateGeneration, - cancellationToken); - resetTask.Ignore(); + cancellationToken).Ignore(); + completion.Task.Ignore(); } - return resetTask.WaitAsync(cancellationToken); + return completion.Task.WaitAsync(cancellationToken); } private async Task RunReset( + TaskCompletionSource completion, Task inProgressSave, + Task resetTask, string enqueuedCheckpoint, long updateGeneration, CancellationToken cancellationToken) @@ -198,20 +205,100 @@ await ResetCore( enqueuedCheckpoint, updateGeneration, cancellationToken); + + while (true) + { + string checkpoint; + lock (_lock) + { + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + completion.TrySetResult(); + return; + } + + checkpoint = _latestCheckpoint; + } + + await Save(checkpoint, cancellationToken); + } } - finally + catch (OperationCanceledException exception) { - lock (_lock) + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: true); + } + catch (Exception exception) + { + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: false); + } + } + + private async Task CompleteFailedReset( + TaskCompletionSource completion, + Task resetTask, + Exception resetFailure, + bool canceled) + { + try + { + while (true) { - _pendingResetCount--; - if (_pendingResetCount == 0) + string checkpoint; + lock (_lock) { - _throttleSavesUntilUtc = null; + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + if (canceled) + { + completion.TrySetCanceled( + ((OperationCanceledException)resetFailure).CancellationToken); + } + else + { + completion.TrySetException(resetFailure); + } + + return; + } + + checkpoint = _latestCheckpoint; } + + await Save(checkpoint, CancellationToken.None); + } + } + catch (Exception persistenceFailure) + { + lock (_lock) + { + CompleteReset(); + completion.TrySetException( + new AggregateException(resetFailure, persistenceFailure)); } } } + private void CompleteReset() + { + _pendingResetCount--; + if (_pendingResetCount == 0) + { + _throttleSavesUntilUtc = null; + } + } + /// public void Update(string offset, DateTime utcNow) { diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 7a84360d353..bc28e0d68b0 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -143,6 +143,10 @@ public async Task Initialize(TimeSpan timeout) private async Task Initialize(CancellationToken cancellationToken) { var watch = Stopwatch.StartNew(); + IEventHubQueueCache? cache = null; + IEventHubReceiver? receiver = null; + var installed = false; + var failures = new List(); try { var checkpointer = await this.checkpointerFactory( @@ -155,36 +159,78 @@ private async Task Initialize(CancellationToken cancellationToken) offset = EventHubConstants.StartOfStream; } - var cache = this.cacheFactory(this.settings.Partition, checkpointer, this.loggerFactory); + cache = this.cacheFactory(this.settings.Partition, checkpointer, this.loggerFactory); var flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) { cache, LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) }; + receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger); + cancellationToken.ThrowIfCancellationRequested(); lock (this.cacheLock) { - this.cache?.Dispose(); - this.checkpointer = checkpointer; - this.cache = cache; - this.flowController = flowController; - if (this.recoveredCursorProgress is not null) + if (this.receiverState == ReceiverRunning) { - this.recoveryCache = cache; + this.cache?.Dispose(); + this.checkpointer = checkpointer; + this.cache = cache; + this.flowController = flowController; + this.receiverUsesCheckpoint = receiverUsesCheckpoint; + this.receiver = receiver; + if (this.recoveredCursorProgress is not null) + { + this.recoveryCache = cache; + } + + installed = true; } } - - this.receiverUsesCheckpoint = receiverUsesCheckpoint; - this.receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger); - watch.Stop(); - this.monitor?.TrackInitialization(true, watch.Elapsed, null); } catch (Exception ex) { - watch.Stop(); - this.monitor?.TrackInitialization(false, watch.Elapsed, ex); - throw; + failures.Add(ex); + } + + if (!installed) + { + try + { + cache?.Dispose(); + } + catch (Exception exception) + { + failures.Add(exception); + } + + try + { + if (receiver is not null) + { + using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); + await receiver.CloseAsync(cleanupCancellation.Token) + .WaitAsync(cleanupCancellation.Token); + } + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + watch.Stop(); + if (failures.Count == 0) + { + this.monitor?.TrackInitialization(true, watch.Elapsed, null); + return; } + + var failure = failures.Count == 1 + ? failures[0] + : new AggregateException(failures); + this.monitor?.TrackInitialization(false, watch.Elapsed, failure); + ExceptionDispatchInfo.Capture(failure).Throw(); + throw new UnreachableException(); } [Obsolete("Use the overload which accepts a CancellationToken.")] @@ -318,25 +364,41 @@ private async Task ResetReceiver(CancellationToken cancellationToken) throw; } - this.receiverUsesCheckpoint = false; - var receiver = Interlocked.Exchange(ref this.receiver, null); + IEventHubReceiver? receiver; + lock (this.cacheLock) + { + this.receiverUsesCheckpoint = false; + receiver = Interlocked.Exchange(ref this.receiver, null); + } + var exceptions = new List(); try { if (receiver is not null) { - await receiver.CloseAsync(cancellationToken); + using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); + await receiver.CloseAsync(cleanupCancellation.Token) + .WaitAsync(cleanupCancellation.Token); } } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } catch (Exception exception) { exceptions.Add(exception); } + if (cancellationToken.IsCancellationRequested) + { + if (exceptions.Count > 0) + { + LogWarningFailedToRecoverFromInvalidCheckpoint( + this.settings.Hub.EventHubName, + this.settings.Partition, + exceptions[0]); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + try { await Initialize(cancellationToken); @@ -454,9 +516,13 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke if (earliestSubscriptionToken is IEventHubPartitionLocation location && long.TryParse(location.EventHubOffset, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) { - IStreamQueueCheckpointer? checkpointer; lock (this.cacheLock) { + if (this.receiverState != ReceiverRunning) + { + return; + } + if (this.recoveredCursorProgress is { } recoveredProgress) { if (this.recoveryPendingCursors is { Count: > 0 } @@ -472,10 +538,11 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke this.recoveryPendingCursors = null; } - checkpointer = this.checkpointer; + this.checkpointer?.Update( + location.EventHubOffset, + utcNow, + CancellationToken.None); } - - checkpointer?.Update(location.EventHubOffset, utcNow, CancellationToken.None); } } @@ -523,9 +590,12 @@ public async Task Shutdown(TimeSpan timeout) try { // if receiver was already shutdown, do nothing - if (ReceiverShutdown == Interlocked.Exchange(ref this.receiverState, ReceiverShutdown)) + lock (this.cacheLock) { - return; + if (ReceiverShutdown == Interlocked.Exchange(ref this.receiverState, ReceiverShutdown)) + { + return; + } } LogInfoStoppingReadingFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); diff --git a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs index c027fcc909b..66587ac1846 100644 --- a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs @@ -77,24 +77,31 @@ public async Task Load(CancellationToken cancellationToken) /// public Task Reset(CancellationToken cancellationToken) { - Task resetTask; + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); lock (_lock) { _pendingResetCount++; _throttleSavesUntilUtc = DateTime.MaxValue; - resetTask = _inProgressSave = RunReset( + var inProgressSave = _inProgressSave; + _inProgressSave = completion.Task; + RunReset( + completion, + inProgressSave, _inProgressSave, _latestCheckpoint, _updateGeneration, - cancellationToken); - resetTask.Ignore(); + cancellationToken).Ignore(); + completion.Task.Ignore(); } - return resetTask.WaitAsync(cancellationToken); + return completion.Task.WaitAsync(cancellationToken); } private async Task RunReset( + TaskCompletionSource completion, Task inProgressSave, + Task resetTask, string enqueuedCheckpoint, long updateGeneration, CancellationToken cancellationToken) @@ -106,20 +113,100 @@ await ResetCore( enqueuedCheckpoint, updateGeneration, cancellationToken); + + while (true) + { + string checkpoint; + lock (_lock) + { + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedState.Checkpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + completion.TrySetResult(); + return; + } + + checkpoint = _latestCheckpoint; + } + + await Save(checkpoint, cancellationToken); + } } - finally + catch (OperationCanceledException exception) { - lock (_lock) + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: true); + } + catch (Exception exception) + { + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: false); + } + } + + private async Task CompleteFailedReset( + TaskCompletionSource completion, + Task resetTask, + Exception resetFailure, + bool canceled) + { + try + { + while (true) { - _pendingResetCount--; - if (_pendingResetCount == 0) + string checkpoint; + lock (_lock) { - _throttleSavesUntilUtc = null; + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedState.Checkpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + if (canceled) + { + completion.TrySetCanceled( + ((OperationCanceledException)resetFailure).CancellationToken); + } + else + { + completion.TrySetException(resetFailure); + } + + return; + } + + checkpoint = _latestCheckpoint; } + + await Save(checkpoint, CancellationToken.None); + } + } + catch (Exception persistenceFailure) + { + lock (_lock) + { + CompleteReset(); + completion.TrySetException( + new AggregateException(resetFailure, persistenceFailure)); } } } + private void CompleteReset() + { + _pendingResetCount--; + if (_pendingResetCount == 0) + { + _throttleSavesUntilUtc = null; + } + } + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index da4fc3def54..71008c49ef6 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -43,7 +43,7 @@ public virtual Task Reset(CancellationToken cancellationToken) return Task.CompletedTask; } - public void Update(string offset, DateTime utcNow) + public virtual void Update(string offset, DateTime utcNow) { if (LastOffset is not null && long.Parse(offset, CultureInfo.InvariantCulture) <= long.Parse(LastOffset, CultureInfo.InvariantCulture)) @@ -82,6 +82,47 @@ public override Task Reset(CancellationToken cancellationToken) } } + private sealed class BlockingResetCheckpointer : TestCheckpointer + { + public TaskCompletionSource ResetStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseReset { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override async Task Reset(CancellationToken cancellationToken) + { + await base.Reset(cancellationToken); + ResetStarted.TrySetResult(); + await ReleaseReset.Task.WaitAsync(cancellationToken); + } + } + + private sealed class BlockingUpdateCheckpointer : TestCheckpointer + { + public TaskCompletionSource UpdateStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseUpdate { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ResetStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override void Update(string offset, DateTime utcNow) + { + UpdateStarted.TrySetResult(); + ReleaseUpdate.Task.GetAwaiter().GetResult(); + base.Update(offset, utcNow); + } + + public override Task Reset(CancellationToken cancellationToken) + { + ResetStarted.TrySetResult(); + return base.Reset(cancellationToken); + } + } + private class TestEventHubQueueCache : IEventHubQueueCache { private readonly IStreamQueueCheckpointer? checkpointer; @@ -348,9 +389,14 @@ public Task> ReceiveAsync(int maxCount, TimeSpan waitTime private sealed class InvalidOffsetEventHubReceiver : IEventHubReceiver { public int CloseCount { get; private set; } + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) - => throw new ArgumentException("The supplied offset is invalid."); + { + ReceiveStarted.TrySetResult(); + throw new ArgumentException("The supplied offset is invalid."); + } public Task CloseAsync() { @@ -412,6 +458,24 @@ public Task CloseAsync(CancellationToken cancellationToken) } } + private sealed class CleanupTrackingEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + public CancellationToken CloseCancellationToken { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + CloseCancellationToken = cancellationToken; + return Task.CompletedTask; + } + } + private static EventHubSequenceToken MakeToken(long offset, long sequenceNumber = 0) { return new EventHubSequenceToken(offset.ToString(), sequenceNumber, 0); @@ -514,6 +578,51 @@ public async Task Initialize_WhenCheckpointDoesNotExist_UsesStartOfStream() Assert.Equal(EventHubConstants.StartOfStream, receiverOffset); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenReceiverCreationFails_DisposesCreatedCache() + { + var cache = new TestEventHubQueueCache(); + var expected = new InvalidOperationException("Receiver creation failed"); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + receiverFactory: _ => throw expected, + initialize: false); + + var actual = await Assert.ThrowsAsync( + () => receiver.Initialize(TimeSpan.FromSeconds(5))); + + Assert.Same(expected, actual); + Assert.Equal(1, cache.DisposeCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenCanceledAfterResourceCreation_UsesIndependentCleanupToken() + { + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new CleanupTrackingEventHubReceiver(); + using var cancellation = new CancellationTokenSource(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + receiverFactory: _ => + { + cancellation.Cancel(); + return eventHubReceiver; + }, + initialize: false); + + var exception = await Assert.ThrowsAnyAsync( + () => InvokeInitialize(receiver, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(eventHubReceiver.CloseCancellationToken.IsCancellationRequested); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task GetQueueMessagesAsync_TreatsNullReceiverResultAsEmpty() @@ -739,6 +848,69 @@ public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsRead Assert.Equal(1, checkpointer.ResetCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResources() + { + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var initialCache = new TestEventHubQueueCache(); + var replacementCache = new TestEventHubQueueCache(); + var caches = new Queue([initialCache, replacementCache]); + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver, + cacheFactory: caches.Dequeue); + + var recovery = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + checkpointer.ReleaseReset.TrySetResult(); + await Assert.ThrowsAsync(() => recovery); + + Assert.Equal(1, initialCache.DisposeCount); + Assert.Equal(1, replacementCache.DisposeCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(1, replacementReceiver.CloseCount); + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task UpdateDeliveryProgress_WhenRecoveryStarts_WritesBeforeReset() + { + var checkpointer = new BlockingUpdateCheckpointer + { + LoadedOffset = "123", + }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, eventHubReceiver: invalidReceiver); + + var update = Task.Run( + () => receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow), + TestContext.Current.CancellationToken); + await checkpointer.UpdateStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var recovery = Task.Run( + async () => await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + await invalidReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + + Assert.False(checkpointer.ResetStarted.Task.IsCompleted); + checkpointer.ReleaseUpdate.TrySetResult(); + + await update; + await Assert.ThrowsAsync(() => recovery); + Assert.Equal("124", checkpointer.LastOffset); + Assert.True(checkpointer.ResetStarted.Task.IsCompleted); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task NoOpCheckpointerFactory_CreateHonorsCancellation() @@ -1095,6 +1267,20 @@ public async Task Shutdown_FlushesLatestDeliveryProgress() Assert.Equal("100", checkpointer.FlushedOffset); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_RejectsSubsequentDeliveryProgress() + { + var checkpointer = new TestCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + receiver.UpdateDeliveryProgress(MakeToken(100), DateTime.UtcNow); + + Assert.Equal(0, checkpointer.UpdateCount); + Assert.Null(checkpointer.LastOffset); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task MultipleSubscriptions_CheckpointsMinimumWatermark() @@ -1294,6 +1480,20 @@ private static object InvokeCreateWriteEntity( return method.Invoke(checkpointer, [offset])!; } + private static Task InvokeInitialize( + EventHubAdapterReceiver receiver, + CancellationToken cancellationToken) + { + var method = typeof(EventHubAdapterReceiver).GetMethod( + "Initialize", + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [typeof(CancellationToken)], + modifiers: null); + Assert.NotNull(method); + return (Task)method.Invoke(receiver, [cancellationToken])!; + } + private static void InvokeRestoreLatestCheckpoint( AzureTableStreamQueueCheckpointer checkpointer, string offset) diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index 8039615559f..03470527867 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -188,7 +188,6 @@ public async Task Reset_WhenUpdatePrecedesFailedNestedReset_RestoresUpdate() blockedReset.SetResult(); await firstReset; await Assert.ThrowsAsync(() => secondReset); - await checkpointer.FlushAsync(CancellationToken.None); Assert.True(checkpointer.CheckpointExists); Assert.Equal([NoCheckpoint, NoCheckpoint, "20"], store.WriteAttempts); @@ -258,6 +257,23 @@ public async Task Reset_WhenQueuedBehindSave_PreservesSubsequentUpdate() Assert.Equal("30", store.PersistedCheckpoint); } + [Fact] + public async Task Reset_WhenUpdateArrivesDuringReset_PersistsBeforeReturning() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await reset; + + Assert.Equal([NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + [Fact] public async Task Update_PersistsCheckpoint() { From d7aa4e5795638bff51f826b627fc6685ef9467d1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 08:43:20 -0700 Subject: [PATCH 19/30] test(streaming): await Event Hubs cache initialization --- .../EHPurgeLogicTests.cs | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs index e552ee4fcdf..57da29fe5e8 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs @@ -70,7 +70,7 @@ public EHPurgeLogicTests() [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WontPurge_WhenUnderPressure() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache, make sure will allocate multiple buffers from the pool int itemAddToCache = 100; @@ -93,7 +93,7 @@ public async Task EventHubQueueCache_WontPurge_WhenUnderPressure() [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WontPurge_WhenTimePurgePredicateSaysDontPurge() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -118,7 +118,7 @@ public async Task EventHubQueueCache_WontPurge_WhenTimePurgePredicateSaysDontPur [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WillPurge_WhenTimePurgePredicateSaysPurge_And_NotUnderPressure() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -144,7 +144,7 @@ public async Task EventHubQueueCache_WillPurge_WhenTimePurgePredicateSaysPurge_A [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_EvictionStrategy_Behavior() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -200,7 +200,7 @@ public async Task EventHubQueueCache_EvictionStrategy_Behavior() expectedPurgedBuffers.ForEach(buffer => Assert.Contains(buffer, newBufferAllocated)); } - private void InitForTesting() + private async Task InitForTesting() { this.cacheList = new ConcurrentBag(); this.evictionStrategyList = new List(); @@ -211,11 +211,14 @@ private void InitForTesting() }; this.receiver1 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, - new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider); + new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, + static (_, _, _) => NoOpEventHubReceiver.Instance); this.receiver2 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, - new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider); - this.receiver1.Initialize(this.timeOut); - this.receiver2.Initialize(this.timeOut); + new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, + static (_, _, _) => NoOpEventHubReceiver.Instance); + await Task.WhenAll( + this.receiver1.Initialize(this.timeOut), + this.receiver2.Initialize(this.timeOut)); } private int GetItemCountInAllCache(ConcurrentBag caches) @@ -271,5 +274,15 @@ private IEventHubQueueCache CacheFactory(string partition, IStreamQueueCheckpoin this.cacheList.Add(cache); return cache; } + + private sealed class NoOpEventHubReceiver : IEventHubReceiver + { + public static NoOpEventHubReceiver Instance { get; } = new(); + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => Task.CompletedTask; + } } } From 7daeeb7cbe9179ee45f2f1e8e898405518a75d37 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 08:50:00 -0700 Subject: [PATCH 20/30] test(streaming): await Event Hubs purge setup --- .../EvictionStrategyTests/EHPurgeLogicTests.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs index 57da29fe5e8..509befbb3ea 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs @@ -212,10 +212,10 @@ private async Task InitForTesting() this.receiver1 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, - static (_, _, _) => NoOpEventHubReceiver.Instance); + static (_, _, _) => new NoOpEventHubReceiver()); this.receiver2 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, - static (_, _, _) => NoOpEventHubReceiver.Instance); + static (_, _, _) => new NoOpEventHubReceiver()); await Task.WhenAll( this.receiver1.Initialize(this.timeOut), this.receiver2.Initialize(this.timeOut)); @@ -277,8 +277,6 @@ private IEventHubQueueCache CacheFactory(string partition, IStreamQueueCheckpoin private sealed class NoOpEventHubReceiver : IEventHubReceiver { - public static NoOpEventHubReceiver Instance { get; } = new(); - public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) => Task.FromResult>([]); From cf20abb21a1525fe8d962892bed67deafe6dd6ac Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 09:36:42 -0700 Subject: [PATCH 21/30] fix(streaming): serialize Event Hubs recovery --- .../AzureTableStreamQueueCheckpointer.cs | 1 + .../EventHub/EventHubAdapterReceiver.cs | 151 +++++++++++++----- .../Streams/EventHub/EventHubCheckpointer.cs | 1 + .../Orleans.Streaming.EventHubs.cs | 2 + .../EventHubCheckpointerTests.cs | 86 ++++++++-- 5 files changed, 184 insertions(+), 57 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 5bf056faaf8..12407b99099 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -79,6 +79,7 @@ public bool CheckpointExists /// /// Creates and initializes an Azure Table stream queue checkpointer. /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public static Task> Create( AzureTableStreamCheckpointerOptions options, string streamProviderName, diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index bc28e0d68b0..6217fbe5e2e 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -66,6 +66,7 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private Dictionary? recoveredCursorProgress; private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance); private HashSet? recoveryPendingCursors; + private Task? recoveryTask; // Receiver life cycle private int receiverState = ReceiverShutdown; @@ -247,6 +248,12 @@ public async Task> GetQueueMessagesAsync( return new List(); } + var recoveryTask = Volatile.Read(ref this.recoveryTask); + if (recoveryTask is not null) + { + await recoveryTask.WaitAsync(cancellationToken); + } + // if receiver initialization failed, retry if (this.receiver == null) { @@ -258,13 +265,21 @@ public async Task> GetQueueMessagesAsync( return new List(); } } + + var receiver = Volatile.Read(ref this.receiver); + if (Volatile.Read(ref this.receiverState) != ReceiverRunning || receiver is null) + { + return new List(); + } + + var cache = Volatile.Read(ref this.cache); var watch = Stopwatch.StartNew(); List? messages; try { // Receivers built against older Orleans versions can still return null. - messages = (await this.receiver.ReceiveAsync( + messages = (await receiver.ReceiveAsync( maxCount, ReceiveTimeout, cancellationToken))?.ToList(); @@ -317,12 +332,14 @@ public async Task> GetQueueMessagesAsync( List messageStreamPositions; lock (this.cacheLock) { - if (this.cache is null) + if (!ReferenceEquals(receiver, this.receiver) + || !ReferenceEquals(cache, this.cache) + || cache is null) { return batches; } - messageStreamPositions = this.cache.Add(messages, dequeueTimeUtc); + messageStreamPositions = cache.Add(messages, dequeueTimeUtc); } foreach (var streamPosition in messageStreamPositions) @@ -339,39 +356,57 @@ private static bool IsInvalidOffsetException(Exception exception) private async Task ResetReceiver(CancellationToken cancellationToken) { - IStreamQueueCheckpointer checkpointer; + Task? existingRecovery; + TaskCompletionSource? recoveryCompletion = null; + IStreamQueueCheckpointer? checkpointer = null; + IEventHubQueueCache? cache = null; + IEventHubReceiver? receiver = null; lock (this.cacheLock) { - checkpointer = this.checkpointer!; - this.recoveryCache = null; - this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); - this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); + existingRecovery = this.recoveryTask; + if (existingRecovery is null) + { + recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + this.recoveryTask = recoveryCompletion.Task; + checkpointer = this.checkpointer!; + this.recoveryCache = null; + this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); + this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); + cache = this.cache; + this.cache = null; + this.flowController = new(MaxMessagesPerRead); + receiver = Interlocked.Exchange(ref this.receiver, null); + } + } + + if (existingRecovery is not null) + { + await existingRecovery.WaitAsync(cancellationToken); + return; } + var exceptions = new List(); + var resetSucceeded = false; try { - await checkpointer.Reset(cancellationToken); + await checkpointer!.Reset(cancellationToken); + this.receiverUsesCheckpoint = false; + resetSucceeded = true; } - catch + catch (Exception exception) { - lock (this.cacheLock) - { - this.recoveryCache = null; - this.recoveredCursorProgress = null; - this.recoveryPendingCursors = null; - } - - throw; + exceptions.Add(exception); } - IEventHubReceiver? receiver; - lock (this.cacheLock) + try { - this.receiverUsesCheckpoint = false; - receiver = Interlocked.Exchange(ref this.receiver, null); + cache?.Dispose(); + } + catch (Exception exception) + { + exceptions.Add(exception); } - var exceptions = new List(); try { if (receiver is not null) @@ -386,32 +421,37 @@ await receiver.CloseAsync(cleanupCancellation.Token) exceptions.Add(exception); } - if (cancellationToken.IsCancellationRequested) + if (exceptions.Count == 0 && !cancellationToken.IsCancellationRequested) { - if (exceptions.Count > 0) + try { - LogWarningFailedToRecoverFromInvalidCheckpoint( - this.settings.Hub.EventHubName, - this.settings.Partition, - exceptions[0]); + await Initialize(cancellationToken); + } + catch (Exception exception) + { + exceptions.Add(exception); } - - cancellationToken.ThrowIfCancellationRequested(); } - try - { - await Initialize(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception exception) + lock (this.cacheLock) { - exceptions.Add(exception); + if (!resetSucceeded) + { + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + } + + if (ReferenceEquals(this.recoveryTask, recoveryCompletion!.Task)) + { + this.recoveryTask = null; + } + + recoveryCompletion.TrySetResult(); } + cancellationToken.ThrowIfCancellationRequested(); + if (exceptions.Count == 1) { ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); @@ -454,7 +494,20 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList [Obsolete("Use IQueueCache.TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) { - return new Cursor(this, streamId, token); + lock (this.cacheLock) + { + var cache = this.cache + ?? throw new InvalidOperationException("The Event Hub receiver is recovering."); +#pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. + var cursor = cache.GetCursor(streamId, token); +#pragma warning restore CS0618 + return new Cursor( + this, + streamId, + cache, + cursor, + ReferenceEquals(cache, this.recoveryCache) ? token : null); + } } QueueCacheCursorResult IQueueCache.TryGetCacheCursor( @@ -463,7 +516,12 @@ QueueCacheCursorResult IQueueCache.TryGetCacheCursor( { lock (this.cacheLock) { - var cache = this.cache!; + var cache = this.cache; + if (cache is null) + { + return QueueCacheCursorResult.NotSupported; + } + return WrapCursorResult( streamId, cache, @@ -478,7 +536,12 @@ QueueCacheCursorResult IQueueCache.TryGetCacheCursorAtPositio { lock (this.cacheLock) { - var cache = this.cache!; + var cache = this.cache; + if (cache is null) + { + return QueueCacheCursorResult.NotSupported; + } + return WrapCursorResult( streamId, cache, diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index b3560074f29..5ad093703b4 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -71,6 +71,7 @@ public class EventHubCheckpointer : IStreamQueueCheckpointer /// /// /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) => await Create( options, diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index a5d8ec5ef59..ccd44301ef6 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -370,6 +370,7 @@ internal EventHubCheckpointer() { } public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -798,6 +799,7 @@ internal AzureTableStreamQueueCheckpointer() { } public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 71008c49ef6..8b6e5c38e60 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -98,6 +98,15 @@ public override async Task Reset(CancellationToken cancellationToken) } } + private sealed class CancelingResetCheckpointer(CancellationTokenSource cancellation) : TestCheckpointer + { + public override async Task Reset(CancellationToken cancellationToken) + { + await base.Reset(cancellationToken); + cancellation.Cancel(); + } + } + private sealed class BlockingUpdateCheckpointer : TestCheckpointer { public TaskCompletionSource UpdateStarted { get; } = new( @@ -391,6 +400,7 @@ private sealed class InvalidOffsetEventHubReceiver : IEventHubReceiver public int CloseCount { get; private set; } public TaskCompletionSource ReceiveStarted { get; } = new( TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken CloseCancellationToken { get; private set; } public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) { @@ -398,9 +408,12 @@ public Task> ReceiveAsync(int maxCount, TimeSpan waitTime throw new ArgumentException("The supplied offset is invalid."); } - public Task CloseAsync() + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) { CloseCount++; + CloseCancellationToken = cancellationToken; return Task.CompletedTask; } } @@ -742,8 +755,12 @@ public async Task GetQueueMessagesAsync_WhenArgumentFailureIsNotAnInvalidOffset_ } [TestSuite("BVT")] - [Fact, TestCategory("BVT")] - public async Task GetQueueMessagesAsync_AcceptsResumeTokenResolvedByReplacementCache() + [Theory] + [InlineData(false)] + [InlineData(true)] + [TestCategory("BVT")] + public async Task GetQueueMessagesAsync_AcceptsResumeTokenResolvedByReplacementCache( + bool useLegacyCursor) { var checkpointer = new TestCheckpointer { @@ -776,18 +793,30 @@ await Assert.ThrowsAsync( Assert.Single(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); var resumeToken = MakeToken(9); - using var cursor = AssertCursor( - receiver, - StreamId.Create("namespace", Guid.NewGuid()), - resumeToken); - Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + IQueueCacheCursor cursor; + if (useLegacyCursor) + { +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor API. + cursor = ((IQueueCache)receiver).GetCacheCursor(streamId, resumeToken); +#pragma warning restore CS0618 + } + else + { + cursor = AssertCursor(receiver, streamId, resumeToken); + } - receiver.UpdateDeliveryProgress(MakeToken(9), DateTime.UtcNow); - Assert.Equal(0, checkpointer.UpdateCount); + using (cursor) + { + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); - receiver.UpdateDeliveryProgress(resumeToken, DateTime.UtcNow); - Assert.Equal("9", checkpointer.LastOffset); - Assert.Equal(1, checkpointer.UpdateCount); + receiver.UpdateDeliveryProgress(MakeToken(9), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + receiver.UpdateDeliveryProgress(resumeToken, DateTime.UtcNow); + Assert.Equal("9", checkpointer.LastOffset); + Assert.Equal(1, checkpointer.UpdateCount); + } } [TestSuite("BVT")] @@ -880,6 +909,37 @@ public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResour Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task ResetReceiver_WhenCanceled_ClosesReceiverWithIndependentToken() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + var checkpointer = new CancelingResetCheckpointer(cancellation) + { + LoadedOffset = "123", + }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: invalidReceiver); + + var exception = await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.False(invalidReceiver.CloseCancellationToken.IsCancellationRequested); + + receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); + Assert.Null(checkpointer.LastOffset); + + var cursorResult = ((IQueueCache)receiver).TryGetCacheCursor( + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); + Assert.Equal(QueueCacheCursorResultKind.NotSupported, cursorResult.Kind); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task UpdateDeliveryProgress_WhenRecoveryStarts_WritesBeforeReset() From ca93fc88092941ac2038931c3b5006212211ac59 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 10:28:30 -0700 Subject: [PATCH 22/30] fix(streaming): harden checkpoint recovery cleanup --- .../Streams/DynamoDBStreamCheckpointStore.cs | 2 +- .../EventHub/EventHubAdapterReceiver.cs | 132 +++++++--- .../EventHubCheckpointerTests.cs | 242 ++++++++++++++++++ .../DynamoDBStreamQueueCheckpointerTests.cs | 14 + 4 files changed, 359 insertions(+), 31 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs index 0f023d86359..d878aad8e27 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs @@ -256,7 +256,7 @@ private async Task LoadCore(CancellationToken cancellationToken) } if (!item.TryGetValue(CheckpointAttribute, out var checkpoint) - || string.IsNullOrEmpty(checkpoint.S) + || checkpoint.S is null || !item.TryGetValue(VersionAttribute, out var version) || !long.TryParse(version.N, NumberStyles.None, CultureInfo.InvariantCulture, out _version) || _version <= 0) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 6217fbe5e2e..0134b18a95c 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -67,6 +67,11 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance); private HashSet? recoveryPendingCursors; private Task? recoveryTask; + private readonly SemaphoreSlim receiverCloseLock = new(1, 1); + private readonly SemaphoreSlim shutdownLock = new(1, 1); + private IEventHubReceiver? receiverPendingClose; + private Task? receiverCloseTask; + private bool shutdownCompleted; // Receiver life cycle private int receiverState = ReceiverShutdown; @@ -254,6 +259,11 @@ public async Task> GetQueueMessagesAsync( await recoveryTask.WaitAsync(cancellationToken); } + if (Volatile.Read(ref this.receiverPendingClose) is not null) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + } + // if receiver initialization failed, retry if (this.receiver == null) { @@ -376,6 +386,11 @@ private async Task ResetReceiver(CancellationToken cancellationToken) this.cache = null; this.flowController = new(MaxMessagesPerRead); receiver = Interlocked.Exchange(ref this.receiver, null); + if (receiver is not null) + { + this.receiverPendingClose = receiver; + this.receiverCloseTask = null; + } } } @@ -409,12 +424,7 @@ private async Task ResetReceiver(CancellationToken cancellationToken) try { - if (receiver is not null) - { - using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); - await receiver.CloseAsync(cleanupCancellation.Token) - .WaitAsync(cleanupCancellation.Token); - } + await ClosePendingReceiverWithTimeout(); } catch (Exception exception) { @@ -463,6 +473,61 @@ await receiver.CloseAsync(cleanupCancellation.Token) } } + private async Task ClosePendingReceiverWithTimeout() + { + using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); + await ClosePendingReceiver(cleanupCancellation.Token); + } + + private async Task ClosePendingReceiver(CancellationToken cancellationToken) + { + await this.receiverCloseLock.WaitAsync(cancellationToken); + try + { + IEventHubReceiver? receiver; + Task closeTask; + lock (this.cacheLock) + { + receiver = this.receiverPendingClose; + if (receiver is null) + { + return; + } + + if (this.receiverCloseTask is { IsCompletedSuccessfully: true }) + { + this.receiverPendingClose = null; + this.receiverCloseTask = null; + return; + } + + var existingCloseTask = this.receiverCloseTask; + if (existingCloseTask is null || existingCloseTask.IsCompleted) + { + existingCloseTask = receiver.CloseAsync(cancellationToken); + existingCloseTask.Ignore(); + this.receiverCloseTask = existingCloseTask; + } + + closeTask = existingCloseTask; + } + + await closeTask.WaitAsync(cancellationToken); + lock (this.cacheLock) + { + if (ReferenceEquals(this.receiverPendingClose, receiver)) + { + this.receiverPendingClose = null; + this.receiverCloseTask = null; + } + } + } + finally + { + this.receiverCloseLock.Release(); + } + } + public void AddToCache(IList messages) { // do nothing, we add data directly into cache. No need for agent involvement @@ -649,16 +714,22 @@ private void UnregisterCursor(Cursor cursor) public async Task Shutdown(TimeSpan timeout) { + using var shutdownCancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var shutdownCancellationToken = shutdownCancellation?.Token ?? CancellationToken.None; + await this.shutdownLock.WaitAsync(shutdownCancellationToken); var watch = Stopwatch.StartNew(); try { - // if receiver was already shutdown, do nothing + if (this.shutdownCompleted) + { + return; + } + lock (this.cacheLock) { - if (ReceiverShutdown == Interlocked.Exchange(ref this.receiverState, ReceiverShutdown)) - { - return; - } + this.receiverState = ReceiverShutdown; } LogInfoStoppingReadingFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); @@ -671,8 +742,7 @@ public async Task Shutdown(TimeSpan timeout) // so the latest processed offset is persisted and not replayed on restart. if (this.checkpointer != null) { - using var flushCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - await this.checkpointer.FlushAsync(flushCancellation?.Token ?? CancellationToken.None); + await this.checkpointer.FlushAsync(shutdownCancellationToken); } } catch (Exception ex) @@ -682,22 +752,14 @@ public async Task Shutdown(TimeSpan timeout) // clear receiver var localReceiver = Interlocked.Exchange(ref this.receiver, null); - - // start closing receiver - Task closeTask = Task.CompletedTask; - using var closeCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - var closeCancellationToken = closeCancellation?.Token ?? CancellationToken.None; - try + lock (this.cacheLock) { - if (localReceiver != null) + if (localReceiver is not null) { - closeTask = localReceiver.CloseAsync(closeCancellationToken); + this.receiverPendingClose = localReceiver; + this.receiverCloseTask = null; } } - catch (Exception ex) - { - shutdownExceptions.Add(ex); - } // dispose of cache try @@ -717,7 +779,7 @@ public async Task Shutdown(TimeSpan timeout) // finish return receiver closing task try { - await closeTask.WaitAsync(closeCancellationToken); + await ClosePendingReceiver(shutdownCancellationToken); } catch (Exception ex) { @@ -725,6 +787,7 @@ public async Task Shutdown(TimeSpan timeout) } ThrowIfAny(shutdownExceptions); + this.shutdownCompleted = true; watch.Stop(); this.monitor?.TrackShutdown(true, watch.Elapsed, null); @@ -735,6 +798,10 @@ public async Task Shutdown(TimeSpan timeout) this.monitor?.TrackShutdown(false, watch.Elapsed, ex); throw; } + finally + { + this.shutdownLock.Release(); + } static void ThrowIfAny(List exceptions) { @@ -965,19 +1032,24 @@ public void Refresh(StreamSequenceToken token) if (!ReferenceEquals(this.cache, cache)) { - this.owner.ClearRecoveredDeliveryToken(this); - this.cache = cache; - this.current = null; - this.recoveryResumeToken = null; var result = cache.TryGetCursor(this.streamId, token); switch (result.Kind) { case QueueCacheCursorResultKind.Success: + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = cache; this.cursor = result.Cursor; + this.current = null; + this.recoveryResumeToken = + ReferenceEquals(cache, this.owner.recoveryCache) ? token : null; this.pendingMoveResult = null; break; case QueueCacheCursorResultKind.CacheMiss: + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = cache; this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; this.pendingMoveResult = QueueCacheCursorMoveResult.FromCacheMiss(result.CacheMiss!.Value); break; default: diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 8b6e5c38e60..fd5dc49e5d1 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -73,6 +73,34 @@ public override Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class FailOnceFlushCheckpointer : TestCheckpointer + { + public int FlushAttempts { get; private set; } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + FlushAttempts++; + return FlushAttempts == 1 + ? Task.FromException(new InvalidOperationException("Flush failed")) + : base.FlushAsync(cancellationToken); + } + } + + private sealed class BlockingFlushCheckpointer : TestCheckpointer + { + public TaskCompletionSource FlushStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseFlush { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override async Task FlushAsync(CancellationToken cancellationToken) + { + FlushStarted.TrySetResult(); + await ReleaseFlush.Task.WaitAsync(cancellationToken); + await base.FlushAsync(cancellationToken); + } + } + private sealed class FailingResetCheckpointer : TestCheckpointer { public override Task Reset(CancellationToken cancellationToken) @@ -381,6 +409,20 @@ QueueCacheCursorResult IEventHubQueueCache.TryGetCursor( } } + private sealed class SequencedAcquisitionEventHubQueueCache : TestEventHubQueueCache, IEventHubQueueCache + { + public Queue> Results { get; } = new(); + public int RequestCount { get; private set; } + + QueueCacheCursorResult IEventHubQueueCache.TryGetCursor( + StreamId streamId, + StreamSequenceToken? token) + { + RequestCount++; + return Results.Dequeue(); + } + } + private sealed class NullReturningEventHubReceiver : IEventHubReceiver { public int ReceiveCount { get; private set; } @@ -432,6 +474,46 @@ public Task CloseAsync() } } + private sealed class RetryingCloseEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => throw new ArgumentException("The supplied offset is invalid."); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + return CloseCount == 1 + ? Task.FromException(new InvalidOperationException("Close failed")) + : Task.CompletedTask; + } + } + + private sealed class BlockingReadEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseReceive { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int CloseCount { get; private set; } + + public async Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + { + ReceiveStarted.TrySetResult(); + await ReleaseReceive.Task; + return [new EventData()]; + } + + public Task CloseAsync() + { + CloseCount++; + return Task.CompletedTask; + } + } + private sealed class CancellableEventHubReceiver : IEventHubReceiver { public TaskCompletionSource ReceiveStarted { get; } = new( @@ -471,6 +553,24 @@ public Task CloseAsync(CancellationToken cancellationToken) } } + private sealed class NonCancelableCloseEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReleaseClose { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + return ReleaseClose.Task; + } + } + private sealed class CleanupTrackingEventHubReceiver : IEventHubReceiver { public int CloseCount { get; private set; } @@ -858,6 +958,41 @@ await Assert.ThrowsAsync( Assert.Equal("high", exception.High); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorRefresh_WhenReplacementAcquisitionIsUnsupported_RetriesAcquisition() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var replacementCache = new SequencedAcquisitionEventHubQueueCache(); + replacementCache.Results.Enqueue(QueueCacheCursorResult.NotSupported); + replacementCache.Results.Enqueue( + QueueCacheCursorResult.FromCursor(replacementCache.Cursor)); + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor API. + using var cursor = ((IQueueCache)receiver).GetCacheCursor( + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); +#pragma warning restore CS0618 + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Throws(() => cursor.Refresh(MakeToken(2))); + cursor.Refresh(MakeToken(2)); + + Assert.Equal(2, replacementCache.RequestCount); + Assert.Null(replacementCache.RefreshedCursor); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsReadFailure() @@ -877,6 +1012,54 @@ public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsRead Assert.Equal(1, checkpointer.ResetCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_RetriesFailedReceiverCloseBeforeReplacement() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var invalidReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + onReceiverCreated: offsets.Add, + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(["123"], offsets); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + Assert.Equal(2, invalidReceiver.CloseCount); + Assert.Equal(["123", EventHubConstants.StartOfStream], offsets); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenShutdownDetachesReceiver_DiscardsInFlightRead() + { + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new BlockingReadEventHubReceiver(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + eventHubReceiver); + + var read = receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + await eventHubReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + eventHubReceiver.ReleaseReceive.TrySetResult(); + + Assert.Empty(await read); + Assert.Equal(0, cache.AddCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResources() @@ -1103,6 +1286,43 @@ public async Task Shutdown_DisposesCacheAndClosesReceiver_WhenFlushFails() Assert.Equal(1, eventHubReceiver.CloseCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_RetriesFailedFlushBeforeCompleting() + { + var checkpointer = new FailOnceFlushCheckpointer(); + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new TestEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, cache, eventHubReceiver); + + await Assert.ThrowsAsync( + () => receiver.Shutdown(TimeSpan.FromSeconds(5))); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, checkpointer.FlushAttempts); + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_ConcurrentCallerWaitsForCompletion() + { + var checkpointer = new BlockingFlushCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + + var first = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await checkpointer.FlushStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.False(second.IsCompleted); + checkpointer.ReleaseFlush.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, checkpointer.FlushCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Shutdown_CancelsBlockedReceiverClose() @@ -1122,6 +1342,28 @@ await Assert.ThrowsAnyAsync( Assert.Equal(1, eventHubReceiver.CloseCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterTimeoutAwaitsExistingReceiverClose() + { + var eventHubReceiver = new NonCancelableCloseEventHubReceiver(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + eventHubReceiver: eventHubReceiver); + + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + + var retry = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(retry.IsCompleted); + + eventHubReceiver.ReleaseClose.TrySetResult(); + await retry; + + Assert.Equal(1, eventHubReceiver.CloseCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task FlushBeforeLoad_DoesNotPersistUninitializedOffset() diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs index ba67802a1d4..e9bed8be8fc 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs @@ -58,6 +58,20 @@ public async ValueTask Update( [TestCategory("AWS"), TestCategory("Kinesis")] public sealed class DynamoDBStreamCheckpointStoreTests { + [Fact] + public async Task LoadTreatsPersistedEmptyCheckpointAsNoCheckpoint() + { + var client = Substitute.For(); + client.GetItemAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateReadResponse(string.Empty, 7))); + var checkpointer = new DynamoDBStreamQueueCheckpointer( + CreateStore(client), + new DynamoDBStreamQueueCheckpointerOptions()); + + Assert.Equal(string.Empty, await checkpointer.Load(TestContext.Current.CancellationToken)); + Assert.False(checkpointer.CheckpointExists); + } + [Fact] public async Task UpdatePersistsArbitrarySizeSequenceNumberAsString() { From ddc731dd6bba922aa192391acf4dfa9be3677f68 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 10:32:38 -0700 Subject: [PATCH 23/30] fix(streaming): reject progress while recovering --- .../Providers/Streams/EventHub/EventHubAdapterReceiver.cs | 4 +++- .../CheckpointerTests/EventHubCheckpointerTests.cs | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 0134b18a95c..4b5e93c2113 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -646,7 +646,9 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke { lock (this.cacheLock) { - if (this.receiverState != ReceiverRunning) + if (this.receiverState != ReceiverRunning + || this.receiver is null + || this.cache is null) { return; } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index fd5dc49e5d1..28d0e099fc0 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -1010,6 +1010,9 @@ public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsRead Assert.Equal("The supplied offset is invalid.", exception.Message); Assert.Equal(1, checkpointer.ResetCount); + + receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); } [TestSuite("BVT")] From c692bf1afd23599508dbdae82f56c8644459b609 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 11:07:30 -0700 Subject: [PATCH 24/30] fix(streaming): serialize receiver initialization --- .../EventHub/EventHubAdapterReceiver.cs | 241 ++++++++++++------ .../EventHubCheckpointerTests.cs | 113 +++++++- 2 files changed, 269 insertions(+), 85 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 4b5e93c2113..fcf1ba287c1 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -67,6 +67,7 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance); private HashSet? recoveryPendingCursors; private Task? recoveryTask; + private readonly SemaphoreSlim initializationLock = new(1, 1); private readonly SemaphoreSlim receiverCloseLock = new(1, 1); private readonly SemaphoreSlim shutdownLock = new(1, 1); private IEventHubReceiver? receiverPendingClose; @@ -130,15 +131,9 @@ public EventHubAdapterReceiver(EventHubPartitionSettings settings, public async Task Initialize(TimeSpan timeout) { LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); - - // if receiver was already running, do nothing - if (ReceiverRunning == Interlocked.Exchange(ref this.receiverState, ReceiverRunning)) - { - return; - } - + Interlocked.Exchange(ref this.receiverState, ReceiverRunning); using var cancellation = new CancellationTokenSource(timeout); - await Initialize(cancellation.Token); + await EnsureInitialized(cancellation.Token); } /// @@ -146,7 +141,56 @@ public async Task Initialize(TimeSpan timeout) /// it will be retried when messages are requested /// /// - private async Task Initialize(CancellationToken cancellationToken) + private async Task EnsureInitialized(CancellationToken cancellationToken) + { + while (true) + { + Task? recoveryTask; + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning || this.receiver is not null) + { + return; + } + + recoveryTask = this.recoveryTask; + } + + if (recoveryTask is not null) + { + await recoveryTask.WaitAsync(cancellationToken); + continue; + } + + await this.initializationLock.WaitAsync(cancellationToken); + try + { + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning || this.receiver is not null) + { + return; + } + + recoveryTask = this.recoveryTask; + } + + if (recoveryTask is null) + { + await InitializeCore(cancellationToken); + return; + } + } + finally + { + this.initializationLock.Release(); + } + + await recoveryTask.WaitAsync(cancellationToken); + } + } + + private async Task InitializeCore(CancellationToken cancellationToken) { var watch = Stopwatch.StartNew(); IEventHubQueueCache? cache = null; @@ -268,7 +312,7 @@ public async Task> GetQueueMessagesAsync( if (this.receiver == null) { LogWarningRetryingInitializationOfEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); - await Initialize(cancellationToken); + await EnsureInitialized(cancellationToken); if (this.receiver == null) { // should not get here, should throw instead, but just incase. @@ -303,11 +347,19 @@ public async Task> GetQueueMessagesAsync( this.monitor?.TrackRead(false, watch.Elapsed, ex); LogWarningFailedToReadFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition, ex); - if (this.receiverUsesCheckpoint && IsInvalidOffsetException(ex)) + bool shouldRecover; + lock (this.cacheLock) + { + shouldRecover = this.receiverState == ReceiverRunning + && ReferenceEquals(receiver, this.receiver) + && this.receiverUsesCheckpoint; + } + + if (shouldRecover && IsInvalidOffsetException(ex)) { try { - await ResetReceiver(cancellationToken); + await ResetReceiver(receiver, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -364,34 +416,14 @@ private static bool IsInvalidOffsetException(Exception exception) && exception.Message.StartsWith("The supplied offset", StringComparison.OrdinalIgnoreCase) && exception.Message.Contains(" is invalid.", StringComparison.OrdinalIgnoreCase); - private async Task ResetReceiver(CancellationToken cancellationToken) + private async Task ResetReceiver( + IEventHubReceiver failedReceiver, + CancellationToken cancellationToken) { Task? existingRecovery; - TaskCompletionSource? recoveryCompletion = null; - IStreamQueueCheckpointer? checkpointer = null; - IEventHubQueueCache? cache = null; - IEventHubReceiver? receiver = null; lock (this.cacheLock) { existingRecovery = this.recoveryTask; - if (existingRecovery is null) - { - recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); - this.recoveryTask = recoveryCompletion.Task; - checkpointer = this.checkpointer!; - this.recoveryCache = null; - this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); - this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); - cache = this.cache; - this.cache = null; - this.flowController = new(MaxMessagesPerRead); - receiver = Interlocked.Exchange(ref this.receiver, null); - if (receiver is not null) - { - this.receiverPendingClose = receiver; - this.receiverCloseTask = null; - } - } } if (existingRecovery is not null) @@ -400,76 +432,111 @@ private async Task ResetReceiver(CancellationToken cancellationToken) return; } - var exceptions = new List(); + await this.initializationLock.WaitAsync(cancellationToken); + TaskCompletionSource? recoveryCompletion = null; var resetSucceeded = false; + var exceptions = new List(); try { - await checkpointer!.Reset(cancellationToken); - this.receiverUsesCheckpoint = false; - resetSucceeded = true; - } - catch (Exception exception) - { - exceptions.Add(exception); - } + IStreamQueueCheckpointer checkpointer; + IEventHubQueueCache? cache; + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning + || !ReferenceEquals(failedReceiver, this.receiver)) + { + return; + } - try - { - cache?.Dispose(); - } - catch (Exception exception) - { - exceptions.Add(exception); - } + recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + this.recoveryTask = recoveryCompletion.Task; + checkpointer = this.checkpointer!; + this.recoveryCache = null; + this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); + this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); + cache = this.cache; + this.cache = null; + this.flowController = new(MaxMessagesPerRead); + this.receiverPendingClose = Interlocked.Exchange(ref this.receiver, null); + this.receiverCloseTask = null; + } - try - { - await ClosePendingReceiverWithTimeout(); - } - catch (Exception exception) - { - exceptions.Add(exception); - } + try + { + await checkpointer.Reset(cancellationToken); + this.receiverUsesCheckpoint = false; + resetSucceeded = true; + } + catch (Exception exception) + { + exceptions.Add(exception); + } - if (exceptions.Count == 0 && !cancellationToken.IsCancellationRequested) - { try { - await Initialize(cancellationToken); + cache?.Dispose(); } catch (Exception exception) { exceptions.Add(exception); } - } - lock (this.cacheLock) - { - if (!resetSucceeded) + try { - this.recoveryCache = null; - this.recoveredCursorProgress = null; - this.recoveryPendingCursors = null; + await ClosePendingReceiverWithTimeout(); + } + catch (Exception exception) + { + exceptions.Add(exception); } - if (ReferenceEquals(this.recoveryTask, recoveryCompletion!.Task)) + if (exceptions.Count == 0 && !cancellationToken.IsCancellationRequested) { - this.recoveryTask = null; + try + { + await InitializeCore(cancellationToken); + } + catch (Exception exception) + { + exceptions.Add(exception); + } } - recoveryCompletion.TrySetResult(); - } + cancellationToken.ThrowIfCancellationRequested(); - cancellationToken.ThrowIfCancellationRequested(); + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } - if (exceptions.Count == 1) - { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + if (exceptions.Count > 1) + { + throw new AggregateException(exceptions); + } } - - if (exceptions.Count > 1) + finally { - throw new AggregateException(exceptions); + if (recoveryCompletion is not null) + { + lock (this.cacheLock) + { + if (!resetSucceeded) + { + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + } + + if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task)) + { + this.recoveryTask = null; + } + + recoveryCompletion.TrySetResult(); + } + } + + this.initializationLock.Release(); } } @@ -907,8 +974,16 @@ public void Dispose() public IBatchContainer? GetCurrent(out Exception? exception) { - exception = null; - return this.current; + lock (this.owner.cacheLock) + { + exception = null; + if (!ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + } + + return this.current; + } } [Obsolete("Use MoveNextWithResult instead.")] diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 28d0e099fc0..b4419a3ab34 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -514,6 +514,23 @@ public Task CloseAsync() } } + private sealed class BlockingInvalidOffsetEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseReceive { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + { + ReceiveStarted.TrySetResult(); + await ReleaseReceive.Task; + throw new ArgumentException("The supplied offset is invalid."); + } + + public Task CloseAsync() => Task.CompletedTask; + } + private sealed class CancellableEventHubReceiver : IEventHubReceiver { public TaskCompletionSource ReceiveStarted { get; } = new( @@ -622,6 +639,7 @@ private static async Task CreateReceiver( Action? onReceiverCreated = null, Func? receiverFactory = null, Func? cacheFactory = null, + Func>>? checkpointerFactory = null, bool initialize = true) { var settings = new EventHubPartitionSettings @@ -639,7 +657,8 @@ private static async Task CreateReceiver( var receiver = new EventHubAdapterReceiver( settings, cacheFactory: (_, createdCheckpointer, _) => cacheFactory?.Invoke() ?? cache ?? new TestEventHubQueueCache(createdCheckpointer), - checkpointerFactory: _ => Task.FromResult>(checkpointer), + checkpointerFactory: (_, cancellationToken) => checkpointerFactory?.Invoke(cancellationToken) + ?? Task.FromResult>(checkpointer), loggerFactory: Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance, monitor: new Orleans.Streaming.EventHubs.DefaultEventHubReceiverMonitor( new EventHubReceiverMonitorDimensions @@ -691,6 +710,37 @@ public async Task Initialize_WhenCheckpointDoesNotExist_UsesStartOfStream() Assert.Equal(EventHubConstants.StartOfStream, receiverOffset); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ConcurrentCallersShareInitialization() + { + var checkpointer = new TestCheckpointer(); + var factoryStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var factoryCalls = 0; + var receiver = await CreateReceiver( + checkpointer, + checkpointerFactory: async cancellationToken => + { + Interlocked.Increment(ref factoryCalls); + factoryStarted.TrySetResult(); + await releaseFactory.Task.WaitAsync(cancellationToken); + return checkpointer; + }, + initialize: false); + + var first = receiver.Initialize(TimeSpan.FromSeconds(5)); + await factoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.False(second.IsCompleted); + releaseFactory.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, factoryCalls); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Initialize_WhenReceiverCreationFails_DisposesCreatedCache() @@ -830,6 +880,65 @@ await Assert.ThrowsAsync( Assert.Equal(1, checkpointer.UpdateCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorGetCurrent_WhenRecoveryDetachesCache_DiscardsSelectedBatch() + { + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var initialBatch = new TestBatchContainer(); + var initialCache = new TestEventHubQueueCache + { + NextMessage = initialBatch, + }; + var caches = new Queue( + [initialCache, new TestEventHubQueueCache()]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); + using var cursor = AssertCursor( + receiver, + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(initialBatch, cursor.GetCurrent(out _)); + + var recovery = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + + Assert.Null(cursor.GetCurrent(out _)); + + checkpointer.ReleaseReset.TrySetResult(); + await Assert.ThrowsAsync(() => recovery); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenShutdownDetachesFailedRead_DoesNotReset() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var eventHubReceiver = new BlockingInvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: eventHubReceiver); + + var read = receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + await eventHubReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + eventHubReceiver.ReleaseReceive.TrySetResult(); + + await Assert.ThrowsAsync(() => read); + Assert.Equal(0, checkpointer.ResetCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task GetQueueMessagesAsync_WhenArgumentFailureIsNotAnInvalidOffset_DoesNotReset() @@ -1790,7 +1899,7 @@ private static Task InvokeInitialize( CancellationToken cancellationToken) { var method = typeof(EventHubAdapterReceiver).GetMethod( - "Initialize", + "InitializeCore", BindingFlags.Instance | BindingFlags.NonPublic, binder: null, [typeof(CancellationToken)], From 8e17aaf7ba0e4b88e8cf28f2e6caedf47caa7ec4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 11:24:55 -0700 Subject: [PATCH 25/30] fix(streaming): retain failed initialization cleanup --- .../EventHub/EventHubAdapterReceiver.cs | 30 ++++++- .../EventHubCheckpointerTests.cs | 87 +++++++++++++++++++ 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index fcf1ba287c1..a7240d7a045 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -145,6 +145,11 @@ private async Task EnsureInitialized(CancellationToken cancellationToken) { while (true) { + if (Volatile.Read(ref this.receiverPendingClose) is not null) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + } + Task? recoveryTask; lock (this.cacheLock) { @@ -165,6 +170,7 @@ private async Task EnsureInitialized(CancellationToken cancellationToken) await this.initializationLock.WaitAsync(cancellationToken); try { + bool hasPendingClose; lock (this.cacheLock) { if (this.receiverState != ReceiverRunning || this.receiver is not null) @@ -173,6 +179,13 @@ private async Task EnsureInitialized(CancellationToken cancellationToken) } recoveryTask = this.recoveryTask; + hasPendingClose = this.receiverPendingClose is not null; + } + + if (hasPendingClose) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + continue; } if (recoveryTask is null) @@ -257,9 +270,20 @@ private async Task InitializeCore(CancellationToken cancellationToken) { if (receiver is not null) { - using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); - await receiver.CloseAsync(cleanupCancellation.Token) - .WaitAsync(cleanupCancellation.Token); + lock (this.cacheLock) + { + if (this.receiverPendingClose is not null + && !ReferenceEquals(this.receiverPendingClose, receiver)) + { + throw new InvalidOperationException( + "A previous Event Hub receiver is still pending cleanup."); + } + + this.receiverPendingClose = receiver; + this.receiverCloseTask = null; + } + + await ClosePendingReceiverWithTimeout(); } } catch (Exception exception) diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index b4419a3ab34..e6875b10785 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -786,6 +786,93 @@ public async Task Initialize_WhenCanceledAfterResourceCreation_UsesIndependentCl Assert.False(eventHubReceiver.CloseCancellationToken.IsCancellationRequested); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenCleanupFails_RetriesCloseBeforeCreatingReplacement() + { + using var cancellation = new CancellationTokenSource(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var receiverCreations = 0; + var receiver = await CreateReceiver( + new TestCheckpointer(), + receiverFactory: _ => + { + if (Interlocked.Increment(ref receiverCreations) == 1) + { + cancellation.Cancel(); + return failedReceiver; + } + + return replacementReceiver; + }, + initialize: false); + + await Assert.ThrowsAsync( + () => InvokeInitialize(receiver, cancellation.Token)); + + Assert.Equal(1, failedReceiver.CloseCount); + Assert.Equal(1, receiverCreations); + + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, failedReceiver.CloseCount); + Assert.Equal(2, receiverCreations); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WaitingCallerDrainsFailedCleanupBeforeCreatingReplacement() + { + var checkpointer = new TestCheckpointer(); + var firstCancellation = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var receiverFactoryStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + using var releaseReceiverFactory = new ManualResetEventSlim(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var checkpointerFactoryCalls = 0; + var receiverCreations = 0; + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: _ => + { + if (Interlocked.Increment(ref receiverCreations) == 1) + { + receiverFactoryStarted.TrySetResult(); + releaseReceiverFactory.Wait(TestContext.Current.CancellationToken); + return failedReceiver; + } + + return replacementReceiver; + }, + checkpointerFactory: cancellationToken => + { + if (Interlocked.Increment(ref checkpointerFactoryCalls) == 1) + { + _ = cancellationToken.Register(() => firstCancellation.TrySetResult()); + } + + return Task.FromResult>(checkpointer); + }, + initialize: false); + + var first = Task.Run( + () => receiver.Initialize(TimeSpan.FromMilliseconds(50)), + TestContext.Current.CancellationToken); + await receiverFactoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Initialize(TimeSpan.FromSeconds(5)); + await firstCancellation.Task.WaitAsync(TestContext.Current.CancellationToken); + releaseReceiverFactory.Set(); + + await Assert.ThrowsAsync(() => first); + await second; + + Assert.Equal(2, failedReceiver.CloseCount); + Assert.Equal(2, receiverCreations); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task GetQueueMessagesAsync_TreatsNullReceiverResultAsEmpty() From 3fdac89407093ab2125736d4585d1c862ac97f62 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 12:06:10 -0700 Subject: [PATCH 26/30] fix(streaming): keep reset recovery in flight --- .../EventHub/EventHubAdapterReceiver.cs | 98 +++++++++++++------ .../EventHubCheckpointerTests.cs | 37 +++++-- 2 files changed, 97 insertions(+), 38 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index a7240d7a045..e347860f055 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -457,13 +457,12 @@ private async Task ResetReceiver( } await this.initializationLock.WaitAsync(cancellationToken); - TaskCompletionSource? recoveryCompletion = null; - var resetSucceeded = false; - var exceptions = new List(); + var releaseInitializationLock = true; try { IStreamQueueCheckpointer checkpointer; IEventHubQueueCache? cache; + TaskCompletionSource recoveryCompletion; lock (this.cacheLock) { if (this.receiverState != ReceiverRunning @@ -473,6 +472,7 @@ private async Task ResetReceiver( } recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + recoveryCompletion.Task.Ignore(); this.recoveryTask = recoveryCompletion.Task; checkpointer = this.checkpointer!; this.recoveryCache = null; @@ -485,10 +485,37 @@ private async Task ResetReceiver( this.receiverCloseTask = null; } + RunRecovery(checkpointer, cache, recoveryCompletion).Ignore(); + releaseInitializationLock = false; + cancellationToken.ThrowIfCancellationRequested(); + await recoveryCompletion.Task.WaitAsync(cancellationToken); + } + finally + { + if (releaseInitializationLock) + { + this.initializationLock.Release(); + } + } + } + + private async Task RunRecovery( + IStreamQueueCheckpointer checkpointer, + IEventHubQueueCache? cache, + TaskCompletionSource recoveryCompletion) + { + var exceptions = new List(); + var resetSucceeded = false; + try + { try { - await checkpointer.Reset(cancellationToken); - this.receiverUsesCheckpoint = false; + await checkpointer.Reset(CancellationToken.None); + lock (this.cacheLock) + { + this.receiverUsesCheckpoint = false; + } + resetSucceeded = true; } catch (Exception exception) @@ -514,49 +541,56 @@ private async Task ResetReceiver( exceptions.Add(exception); } - if (exceptions.Count == 0 && !cancellationToken.IsCancellationRequested) + bool isRunning; + lock (this.cacheLock) + { + isRunning = this.receiverState == ReceiverRunning; + } + + if (exceptions.Count == 0 && isRunning) { try { - await InitializeCore(cancellationToken); + using var initializationCancellation = new CancellationTokenSource(ReceiveTimeout); + await InitializeCore(initializationCancellation.Token); } catch (Exception exception) { exceptions.Add(exception); } } - - cancellationToken.ThrowIfCancellationRequested(); - - if (exceptions.Count == 1) + } + catch (Exception exception) + { + exceptions.Add(exception); + } + finally + { + if (exceptions.Count == 0) { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + recoveryCompletion.TrySetResult(); } - - if (exceptions.Count > 1) + else if (exceptions.Count == 1) { - throw new AggregateException(exceptions); + recoveryCompletion.TrySetException(exceptions[0]); } - } - finally - { - if (recoveryCompletion is not null) + else { - lock (this.cacheLock) - { - if (!resetSucceeded) - { - this.recoveryCache = null; - this.recoveredCursorProgress = null; - this.recoveryPendingCursors = null; - } + recoveryCompletion.TrySetException(new AggregateException(exceptions)); + } - if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task)) - { - this.recoveryTask = null; - } + lock (this.cacheLock) + { + if (!resetSucceeded) + { + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + } - recoveryCompletion.TrySetResult(); + if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task)) + { + this.recoveryTask = null; } } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index e6875b10785..767569cfc44 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -1285,9 +1285,9 @@ public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResour await Assert.ThrowsAsync(() => recovery); Assert.Equal(1, initialCache.DisposeCount); - Assert.Equal(1, replacementCache.DisposeCount); + Assert.Equal(0, replacementCache.DisposeCount); Assert.Equal(1, invalidReceiver.CloseCount); - Assert.Equal(1, replacementReceiver.CloseCount); + Assert.Equal(0, replacementReceiver.CloseCount); Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); } @@ -1315,11 +1315,36 @@ public async Task ResetReceiver_WhenCanceled_ClosesReceiverWithIndependentToken( receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); Assert.Null(checkpointer.LastOffset); + } - var cursorResult = ((IQueueCache)receiver).TryGetCacheCursor( - StreamId.Create("namespace", Guid.NewGuid()), - MakeToken(1)); - Assert.Equal(QueueCacheCursorResultKind.NotSupported, cursorResult.Kind); + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task ResetReceiver_WhenCallerCancels_ResetRemainsRecoveryBarrier() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver()); + + var canceledRead = receiver.GetQueueMessagesAsync(10, cancellation.Token); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledRead); + + var waitingRead = receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken); + Assert.False(waitingRead.IsCompleted); + + checkpointer.ReleaseReset.TrySetResult(); + Assert.Empty(await waitingRead); } [TestSuite("BVT")] From 0163ee97b62d10d3eb9408c81a73b8d78bedd42b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 12 Sep 2026 02:45:47 -0700 Subject: [PATCH 27/30] fix(streaming): preserve recovery after reset failure --- .../EventHub/EventHubAdapterReceiver.cs | 14 +++----- .../EventHubCheckpointerTests.cs | 34 ++++++++++++++++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index e347860f055..9518d1c8b05 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -505,7 +505,6 @@ private async Task RunRecovery( TaskCompletionSource recoveryCompletion) { var exceptions = new List(); - var resetSucceeded = false; try { try @@ -516,7 +515,6 @@ private async Task RunRecovery( this.receiverUsesCheckpoint = false; } - resetSucceeded = true; } catch (Exception exception) { @@ -581,13 +579,6 @@ private async Task RunRecovery( lock (this.cacheLock) { - if (!resetSucceeded) - { - this.recoveryCache = null; - this.recoveredCursorProgress = null; - this.recoveryPendingCursors = null; - } - if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task)) { this.recoveryTask = null; @@ -906,7 +897,10 @@ public async Task Shutdown(TimeSpan timeout) // finish return receiver closing task try { - await ClosePendingReceiver(shutdownCancellationToken); + using var cleanupCancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + await ClosePendingReceiver(cleanupCancellation?.Token ?? CancellationToken.None); } catch (Exception ex) { diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 767569cfc44..e4ed9584610 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -101,6 +101,14 @@ public override async Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class CanceledFlushCheckpointer : TestCheckpointer + { + public override async Task FlushAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + } + private sealed class FailingResetCheckpointer : TestCheckpointer { public override Task Reset(CancellationToken cancellationToken) @@ -1197,9 +1205,14 @@ public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsRead { LoadedOffset = "123", }; + var caches = new Queue( + [new TestEventHubQueueCache(), new TestEventHubQueueCache()]); var receiver = await CreateReceiver( checkpointer, - eventHubReceiver: new InvalidOffsetEventHubReceiver()); + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); var exception = await Assert.ThrowsAsync( () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); @@ -1207,6 +1220,9 @@ public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsRead Assert.Equal("The supplied offset is invalid.", exception.Message); Assert.Equal(1, checkpointer.ResetCount); + Assert.Empty(await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)); receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); Assert.Equal(0, checkpointer.UpdateCount); } @@ -1510,6 +1526,22 @@ public async Task Shutdown_DisposesCacheAndClosesReceiver_WhenFlushFails() Assert.Equal(1, eventHubReceiver.CloseCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_WhenFlushTimesOut_ClosesReceiverWithIndependentToken() + { + var eventHubReceiver = new CleanupTrackingEventHubReceiver(); + var receiver = await CreateReceiver( + new CanceledFlushCheckpointer(), + eventHubReceiver: eventHubReceiver); + + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(eventHubReceiver.CloseCancellationToken.IsCancellationRequested); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Shutdown_RetriesFailedFlushBeforeCompleting() From ac25afab78e85e8667e1f7321bf610d6f0ed19ad Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 12 Sep 2026 07:24:46 -0700 Subject: [PATCH 28/30] fix(streaming): propagate receiver initialization cancellation --- .../EventHub/EventHubAdapterReceiver.cs | 20 ++++++- .../EventHubCheckpointerTests.cs | 60 ++++++++++++++++--- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 9518d1c8b05..dfcea7122ea 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -128,12 +128,26 @@ public EventHubAdapterReceiver(EventHubPartitionSettings settings, this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory; } - public async Task Initialize(TimeSpan timeout) + public Task Initialize(TimeSpan timeout) + => Initialize(timeout, CancellationToken.None); + + public async Task Initialize( + TimeSpan timeout, + CancellationToken cancellationToken) { LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); Interlocked.Exchange(ref this.receiverState, ReceiverRunning); - using var cancellation = new CancellationTokenSource(timeout); - await EnsureInitialized(cancellation.Token); + using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCancellation.CancelAfter(timeout); + try + { + await EnsureInitialized(timeoutCancellation.Token); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } } /// diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index e4ed9584610..9668c35eb0b 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -685,7 +685,9 @@ private static async Task CreateReceiver( if (initialize) { - await receiver.Initialize(TimeSpan.FromSeconds(5)); + await receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); } return receiver; @@ -739,9 +741,13 @@ public async Task Initialize_ConcurrentCallersShareInitialization() }, initialize: false); - var first = receiver.Initialize(TimeSpan.FromSeconds(5)); + var first = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); await factoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); - var second = receiver.Initialize(TimeSpan.FromSeconds(5)); + var second = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.False(second.IsCompleted); releaseFactory.TrySetResult(); @@ -749,6 +755,38 @@ public async Task Initialize_ConcurrentCallersShareInitialization() Assert.Equal(1, factoryCalls); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ThroughInterface_ForwardsCancellationToken() + { + var checkpointer = new TestCheckpointer(); + var initializationStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var receiver = await CreateReceiver( + checkpointer, + checkpointerFactory: async cancellationToken => + { + initializationStarted.TrySetResult(cancellationToken); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return checkpointer; + }, + initialize: false); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + + var initialization = ((IQueueAdapterReceiver)receiver).Initialize( + TimeSpan.FromSeconds(30), + cancellation.Token); + var forwardedToken = await initializationStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.True(forwardedToken.IsCancellationRequested); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Initialize_WhenReceiverCreationFails_DisposesCreatedCache() @@ -762,7 +800,9 @@ public async Task Initialize_WhenReceiverCreationFails_DisposesCreatedCache() initialize: false); var actual = await Assert.ThrowsAsync( - () => receiver.Initialize(TimeSpan.FromSeconds(5))); + () => receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken)); Assert.Same(expected, actual); Assert.Equal(1, cache.DisposeCount); @@ -822,7 +862,9 @@ await Assert.ThrowsAsync( Assert.Equal(1, failedReceiver.CloseCount); Assert.Equal(1, receiverCreations); - await receiver.Initialize(TimeSpan.FromSeconds(5)); + await receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(2, failedReceiver.CloseCount); Assert.Equal(2, receiverCreations); @@ -867,10 +909,14 @@ public async Task Initialize_WaitingCallerDrainsFailedCleanupBeforeCreatingRepla initialize: false); var first = Task.Run( - () => receiver.Initialize(TimeSpan.FromMilliseconds(50)), + () => receiver.Initialize( + TimeSpan.FromMilliseconds(50), + TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); await receiverFactoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); - var second = receiver.Initialize(TimeSpan.FromSeconds(5)); + var second = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); await firstCancellation.Task.WaitAsync(TestContext.Current.CancellationToken); releaseReceiverFactory.Set(); From eeb9c058d866fd3eed8097f3867a4d859cdf1790 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 21:21:30 -0700 Subject: [PATCH 29/30] fix(streaming): drain Event Hubs receiver lifetimes --- .../EventHub/EventHubAdapterReceiver.cs | 45 ++- .../Orleans.Streaming.EventHubs/README.md | 10 +- .../EventHubCheckpointerTests.cs | 288 +++++++++++++++++- 3 files changed, 324 insertions(+), 19 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index dfcea7122ea..13425608692 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -72,13 +72,13 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly SemaphoreSlim shutdownLock = new(1, 1); private IEventHubReceiver? receiverPendingClose; private Task? receiverCloseTask; - private bool shutdownCompleted; // Receiver life cycle private int receiverState = ReceiverShutdown; private const int ReceiverShutdown = 0; private const int ReceiverRunning = 1; + private const int ReceiverShutdownCompleted = 2; public int GetMaxAddCount() { @@ -135,10 +135,15 @@ public async Task Initialize( TimeSpan timeout, CancellationToken cancellationToken) { - LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); - Interlocked.Exchange(ref this.receiverState, ReceiverRunning); + cancellationToken.ThrowIfCancellationRequested(); using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutCancellation.CancelAfter(timeout); + LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); + lock (this.cacheLock) + { + this.receiverState = ReceiverRunning; + } + try { await EnsureInitialized(timeoutCancellation.Token); @@ -330,7 +335,7 @@ public async Task> GetQueueMessagesAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (this.receiverState == ReceiverShutdown || maxCount <= 0) + if (this.receiverState != ReceiverRunning || maxCount <= 0) { return new List(); } @@ -449,6 +454,7 @@ public async Task> GetQueueMessagesAsync( return batches; } + // The SDK translates this service rejection to ArgumentException, preserving the service description. private static bool IsInvalidOffsetException(Exception exception) => exception is ArgumentException && exception.Message.StartsWith("The supplied offset", StringComparison.OrdinalIgnoreCase) @@ -852,20 +858,25 @@ public async Task Shutdown(TimeSpan timeout) var shutdownCancellationToken = shutdownCancellation?.Token ?? CancellationToken.None; await this.shutdownLock.WaitAsync(shutdownCancellationToken); var watch = Stopwatch.StartNew(); + var initializationLockHeld = false; try { - if (this.shutdownCompleted) - { - return; - } - lock (this.cacheLock) { + if (this.receiverState == ReceiverShutdownCompleted) + { + return; + } + this.receiverState = ReceiverShutdown; } LogInfoStoppingReadingFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); + // Recovery owns this lock until reset and detached-resource cleanup finish, even if its pull was canceled. + await this.initializationLock.WaitAsync(shutdownCancellationToken); + initializationLockHeld = true; + var shutdownExceptions = new List(); try @@ -922,7 +933,16 @@ public async Task Shutdown(TimeSpan timeout) } ThrowIfAny(shutdownExceptions); - this.shutdownCompleted = true; + lock (this.cacheLock) + { + this.checkpointer = null; + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + this.cursors.Clear(); + this.flowController = new(MaxMessagesPerRead); + this.receiverState = ReceiverShutdownCompleted; + } watch.Stop(); this.monitor?.TrackShutdown(true, watch.Elapsed, null); @@ -935,6 +955,11 @@ public async Task Shutdown(TimeSpan timeout) } finally { + if (initializationLockHeld) + { + this.initializationLock.Release(); + } + this.shutdownLock.Release(); } diff --git a/src/Azure/Orleans.Streaming.EventHubs/README.md b/src/Azure/Orleans.Streaming.EventHubs/README.md index df8309cfa3f..7179ba21605 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/README.md +++ b/src/Azure/Orleans.Streaming.EventHubs/README.md @@ -74,13 +74,17 @@ Set `GrainStreamQueueCheckpointerOptions.StorageProviderName` to use another reg Both `UseGrainCheckpointer` and `UseAzureTableCheckpointer` extend `ISiloPersistentStreamConfigurator`, so they can also be used with other persistent stream providers. Event Hubs configures numeric checkpoint ordering by default; other providers can set the corresponding `CheckpointComparer` option for their checkpoint format. -### Recovering from an expired checkpoint +### Recovering from an invalid checkpoint -When Event Hubs rejects a persisted partition offset as invalid, the provider clears that partition's checkpoint, closes and recreates its receiver, and resumes on the next pull using the configured no-checkpoint position. `EventHubReceiverOptions.StartFromNow` selects that position. Its default value, `true`, starts at the partition tail and reads newly enqueued events, skipping the retained backlog. Set it to `false` to replay from the earliest retained event, which can duplicate delivery. Choose the policy according to the application's recovery requirements and keep consumers idempotent. +When Event Hubs rejects a persisted partition offset with its known invalid-offset argument error, the provider clears that partition's checkpoint, closes and recreates its receiver, and resumes on the next pull using the configured no-checkpoint position. Recovery recognizes an `ArgumentException` whose service message starts with `The supplied offset` and contains ` is invalid.`. Azure.Messaging.EventHubs 5.12.2 exposes this rejection through the service message, so recognition depends on that message format. + +`EventHubReceiverOptions.StartFromNow` selects the recovery position. Its default value, `true`, starts at the partition tail and reads newly enqueued events, skipping the retained backlog. Set it to `false` to replay from the earliest retained event, which can duplicate delivery. Choose the policy according to the application's recovery requirements and keep consumers idempotent. The built-in Azure Table and grain-backed checkpointers support this reset contract. A custom `IStreamQueueCheckpointer` participates in recovery by implementing `Reset(CancellationToken)`; the default implementation surfaces unsupported recovery with `NotSupportedException`. -Existing Orleans subscription cursors rebind to the replacement partition cache, so subscriptions continue without application re-registration. Checkpoint reset, receiver shutdown, and receiver initialization observe the pulling agent's cancellation token. +Existing Orleans subscription cursors rebind to the replacement partition cache, so subscriptions continue automatically. Initial receiver creation and reads observe the pulling agent's cancellation token. Once recovery detaches a failed receiver, checkpoint reset runs to completion as a barrier before replacement initialization. Canceling a pull cancels its wait for that recovery; the recovery continues with independent cleanup and replacement-initialization timeouts. + +Receiver shutdown stops initialization and recovery from installing new resources, drains admitted initialization and recovery, then flushes the checkpoint and closes the current lifetime's resources. Its timeout bounds the drain and flush; receiver close has an independent cleanup timeout. Successful shutdown completes cleanup for that lifetime, and a subsequent initialization starts a new lifetime with its own shutdown. ## Example - Using Event Hub Streams in a Grain diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 9668c35eb0b..696ff81ec45 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -120,6 +120,8 @@ public override Task Reset(CancellationToken cancellationToken) private sealed class BlockingResetCheckpointer : TestCheckpointer { + public CancellationToken ResetCancellationToken { get; private set; } + public TaskCompletionSource ResetStarted { get; } = new( TaskCreationOptions.RunContinuationsAsynchronously); @@ -128,6 +130,7 @@ private sealed class BlockingResetCheckpointer : TestCheckpointer public override async Task Reset(CancellationToken cancellationToken) { + ResetCancellationToken = cancellationToken; await base.Reset(cancellationToken); ResetStarted.TrySetResult(); await ReleaseReset.Task.WaitAsync(cancellationToken); @@ -471,9 +474,10 @@ public Task CloseAsync(CancellationToken cancellationToken) private sealed class InvalidArgumentEventHubReceiver : IEventHubReceiver { public int CloseCount { get; private set; } + public Exception Failure { get; init; } = new ArgumentException("The receiver rejected an unrelated argument."); public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) - => throw new ArgumentException("The receiver rejected an unrelated argument."); + => throw Failure; public Task CloseAsync() { @@ -1104,6 +1108,37 @@ public async Task GetQueueMessagesAsync_WhenArgumentFailureIsNotAnInvalidOffset_ Assert.Equal(["123"], offsets); } + [TestSuite("BVT")] + [Theory, TestCategory("BVT")] + [InlineData(true, true, "The supplied offset '13953' is invalid. The last offset in the system is '13952' TrackingId:example\nFor troubleshooting information, see https://aka.ms/azsdk/net/eventhubs/exceptions/troubleshoot", true)] + [InlineData(false, true, "The supplied offset '13953' is invalid.", false)] + [InlineData(true, false, "The supplied offset '13953' is invalid.", false)] + [InlineData(true, true, "The supplied offset must be numeric.", false)] + [InlineData(true, true, "The consumer group is invalid.", false)] + public async Task GetQueueMessagesAsync_ClassifiesKnownServiceOffsetError( + bool checkpointExists, bool argumentException, string message, bool recover) + { + var checkpointer = new TestCheckpointer { CheckpointExists = checkpointExists, LoadedOffset = "13953" }; + var cache = new TestEventHubQueueCache(); + var failure = argumentException ? (Exception)new ArgumentException(message) : new InvalidOperationException(message); + var failedReceiver = new InvalidArgumentEventHubReceiver { Failure = failure }; + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => offsets.Count == 0 ? cache : new TestEventHubQueueCache(), + onReceiverCreated: offsets.Add, + receiverFactory: _ => offsets.Count == 1 ? failedReceiver : new TestEventHubReceiver()); + + Assert.Same(failure, await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken))); + Assert.Equal(recover ? 1 : 0, checkpointer.ResetCount); + Assert.Equal(recover ? 1 : 0, failedReceiver.CloseCount); + Assert.Equal(recover ? 1 : 0, cache.DisposeCount); + Assert.Equal( + recover ? ["13953", EventHubConstants.StartOfStream] : new[] { checkpointExists ? "13953" : EventHubConstants.StartOfStream }, + offsets); + } + [TestSuite("BVT")] [Theory] [InlineData(false)] @@ -1322,8 +1357,11 @@ public async Task GetQueueMessagesAsync_WhenShutdownDetachesReceiver_DiscardsInF } [TestSuite("BVT")] - [Fact, TestCategory("BVT")] - public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResources() + [Theory, TestCategory("BVT")] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ResetReceiver_WhenShutdownWins_DrainsRecoveryBeforeNextLifetime(bool cancelRead, bool timeoutShutdown) { var checkpointer = new BlockingResetCheckpointer { @@ -1339,18 +1377,53 @@ public async Task ResetReceiver_WhenShutdownWins_DoesNotInstallReplacementResour receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver, cacheFactory: caches.Dequeue); - var recovery = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var recovery = receiver.GetQueueMessagesAsync(10, cancellation.Token); await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); - await receiver.Shutdown(TimeSpan.FromSeconds(5)); + if (cancelRead) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => recovery); + } + + if (timeoutShutdown) + { + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + } + + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.False(shutdown.IsCompleted); + Assert.Equal(CancellationToken.None, checkpointer.ResetCancellationToken); + Assert.Equal(0, checkpointer.FlushCount); + Assert.Equal(0, initialCache.DisposeCount); + Assert.Equal(0, invalidReceiver.CloseCount); checkpointer.ReleaseReset.TrySetResult(); - await Assert.ThrowsAsync(() => recovery); + await shutdown; + if (!cancelRead) + { + await Assert.ThrowsAsync(() => recovery); + } + Assert.Equal(1, checkpointer.FlushCount); Assert.Equal(1, initialCache.DisposeCount); Assert.Equal(0, replacementCache.DisposeCount); Assert.Equal(1, invalidReceiver.CloseCount); Assert.Equal(0, replacementReceiver.CloseCount); Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(42)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal("42", checkpointer.FlushedOffset); + Assert.Equal(2, checkpointer.FlushCount); + Assert.Equal(1, initialCache.DisposeCount); + Assert.Equal(1, replacementCache.DisposeCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(1, replacementReceiver.CloseCount); + Assert.Equal(1, checkpointer.ResetCount); } [TestSuite("BVT")] @@ -1556,6 +1629,209 @@ public async Task LegacyReceiverCursorPreservesProviderExceptions() Assert.Same(movementInnerException, actualMovementException.InnerException); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterReinitialization_CleansUpEachLifetimeOnce() + { + var checkpointers = new[] { new TestCheckpointer(), new TestCheckpointer() }; + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new TestEventHubReceiver(), new TestEventHubReceiver() }; + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointers[0], + cacheFactory: () => caches[lifetime], + receiverFactory: _ => eventHubReceivers[lifetime], + checkpointerFactory: _ => Task.FromResult>(checkpointers[lifetime]), + initialize: false); + + for (; lifetime < 2; lifetime++) + { + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(100 + lifetime)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal((100 + lifetime).ToString(CultureInfo.InvariantCulture), checkpointers[lifetime].FlushedOffset); + Assert.Equal(1, checkpointers[lifetime].FlushCount); + Assert.Equal(1, caches[lifetime].DisposeCount); + Assert.Equal(1, eventHubReceivers[lifetime].CloseCount); + } + + Assert.All(checkpointers, checkpointer => Assert.Equal(1, checkpointer.FlushCount)); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.All(eventHubReceivers, eventHubReceiver => Assert.Equal(1, eventHubReceiver.CloseCount)); + } + + [TestSuite("BVT")] + [Theory, TestCategory("BVT")] + [InlineData(false)] + [InlineData(true)] + public async Task Initialize_AfterShutdownAndFailedReinitialization_RetriesAndCleansUp(bool cancelInitialization) + { + var checkpointers = new[] { new TestCheckpointer(), new TestCheckpointer(), new TestCheckpointer() }; + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new CleanupTrackingEventHubReceiver(), new CleanupTrackingEventHubReceiver(), new CleanupTrackingEventHubReceiver() }; + var attempt = 0; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var expected = new InvalidOperationException("Receiver creation failed"); + var receiver = await CreateReceiver( + checkpointers[0], + cacheFactory: () => caches[attempt], + receiverFactory: _ => + { + if (attempt == 1) + { + if (!cancelInitialization) + { + throw expected; + } + + cancellation.Cancel(); + } + + return eventHubReceivers[attempt]; + }, + checkpointerFactory: _ => Task.FromResult>(checkpointers[attempt])); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + attempt++; + var initialization = receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token); + if (cancelInitialization) + { + var exception = await Assert.ThrowsAnyAsync(() => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False(eventHubReceivers[1].CloseCancellationToken.IsCancellationRequested); + } + else + { + Assert.Same(expected, await Assert.ThrowsAsync(() => initialization)); + } + + attempt++; + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + UpdateDeliveryProgress(receiver, MakeToken(200)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointers[0].FlushCount); + Assert.Equal(0, checkpointers[1].FlushCount); + Assert.Equal(1, checkpointers[2].FlushCount); + Assert.Equal("200", checkpointers[2].FlushedOffset); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.Equal(1, eventHubReceivers[0].CloseCount); + Assert.Equal(cancelInitialization ? 1 : 0, eventHubReceivers[1].CloseCount); + Assert.Equal(1, eventHubReceivers[2].CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_AfterShutdownWithCanceledToken_PreservesCompletedLifetime() + { + var checkpointer = new TestCheckpointer(); + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new TestEventHubReceiver(); + var creations = 0; + var receiver = await CreateReceiver( + checkpointer, cache, eventHubReceiver, onReceiverCreated: _ => creations++); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, creations); + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterCanceledReinitialization_RetriesPendingCleanup() + { + var checkpointer = new TestCheckpointer(); + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var originalReceiver = new TestEventHubReceiver(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + using var cancellation = new CancellationTokenSource(); + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => caches[lifetime], + receiverFactory: _ => + { + if (lifetime == 0) + { + return originalReceiver; + } + + cancellation.Cancel(); + return failedReceiver; + }); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + lifetime++; + var failure = await Assert.ThrowsAsync( + () => receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.IsAssignableFrom(failure.InnerExceptions[0]); + Assert.Equal("Close failed", failure.InnerExceptions[1].Message); + Assert.Equal(1, failedReceiver.CloseCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, originalReceiver.CloseCount); + Assert.Equal(2, failedReceiver.CloseCount); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_DrainsInitializationBeforeNextLifetime() + { + var checkpointer = new TestCheckpointer(); + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new TestEventHubReceiver(), new TestEventHubReceiver() }; + var factoryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => caches[lifetime], + receiverFactory: _ => eventHubReceivers[lifetime], + checkpointerFactory: async cancellationToken => + { + factoryStarted.TrySetResult(); + await releaseFactory.Task.WaitAsync(cancellationToken); + return checkpointer; + }, + initialize: false); + + var initialization = receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await factoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.False(shutdown.IsCompleted); + releaseFactory.TrySetResult(); + await Task.WhenAll(initialization, shutdown); + Assert.Equal(0, checkpointer.FlushCount); + Assert.Equal(1, caches[0].DisposeCount); + Assert.Equal(1, eventHubReceivers[0].CloseCount); + + lifetime++; + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(300)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal("300", checkpointer.FlushedOffset); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.All(eventHubReceivers, eventHubReceiver => Assert.Equal(1, eventHubReceiver.CloseCount)); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Shutdown_DisposesCacheAndClosesReceiver_WhenFlushFails() From b1637903ffc61172544163d8c18f26367f8089d8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 18 Sep 2026 02:35:19 -0700 Subject: [PATCH 30/30] fix(streaming): serialize receiver lifetime transitions --- .../EventHub/EventHubAdapterReceiver.cs | 18 +++-- .../EventHubCheckpointerTests.cs | 71 +++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 13425608692..8642c99857e 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -139,13 +139,21 @@ public async Task Initialize( using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutCancellation.CancelAfter(timeout); LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); - lock (this.cacheLock) - { - this.receiverState = ReceiverRunning; - } - try { + await this.shutdownLock.WaitAsync(timeoutCancellation.Token); + try + { + lock (this.cacheLock) + { + this.receiverState = ReceiverRunning; + } + } + finally + { + this.shutdownLock.Release(); + } + await EnsureInitialized(timeoutCancellation.Token); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 696ff81ec45..52b7e208f74 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -1662,6 +1662,77 @@ public async Task Shutdown_AfterReinitialization_CleansUpEachLifetimeOnce() Assert.All(eventHubReceivers, eventHubReceiver => Assert.Equal(1, eventHubReceiver.CloseCount)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ConcurrentWithShutdown_StartsAfterShutdownCompletes() + { + var firstCheckpointer = new BlockingFlushCheckpointer(); + var secondCheckpointer = new TestCheckpointer(); + var checkpointers = new Queue>( + [firstCheckpointer, secondCheckpointer]); + var caches = new Queue( + [new TestEventHubQueueCache(), new TestEventHubQueueCache()]); + var eventHubReceivers = new Queue( + [new TestEventHubReceiver(), new TestEventHubReceiver()]); + var receiver = await CreateReceiver( + firstCheckpointer, + cacheFactory: () => caches.Dequeue(), + receiverFactory: _ => eventHubReceivers.Dequeue(), + checkpointerFactory: _ => Task.FromResult(checkpointers.Dequeue())); + + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await firstCheckpointer.FlushStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + var initialization = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.False(initialization.IsCompleted); + Assert.Single(checkpointers); + Assert.Single(caches); + Assert.Single(eventHubReceivers); + + firstCheckpointer.ReleaseFlush.TrySetResult(); + await shutdown; + await initialization; + + Assert.Empty(await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Empty(checkpointers); + Assert.Empty(caches); + Assert.Empty(eventHubReceivers); + Assert.Equal(1, firstCheckpointer.FlushCount); + Assert.Equal(1, secondCheckpointer.FlushCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_CanceledWhileWaitingForShutdown_PreservesCallerToken() + { + var checkpointer = new BlockingFlushCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await checkpointer.FlushStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + + var initialization = receiver.Initialize( + TimeSpan.FromSeconds(30), + cancellation.Token); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + + checkpointer.ReleaseFlush.TrySetResult(); + await shutdown; + } + [TestSuite("BVT")] [Theory, TestCategory("BVT")] [InlineData(false)]