From cc3b4e6638be93ae418002a3b054481bf9c192ad Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 09:19:53 -0700 Subject: [PATCH 01/90] feat(streaming): report delivery progress periodically Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../PersistentStreamProviderOptions.cs | 10 + .../PersistentStreamPullingAgent.cs | 55 ++++- .../Orleans.Streaming/Orleans.Streaming.cs | 3 + .../PersistentStreamPullingAgentTests.cs | 230 +++++++++++++++++- 4 files changed, 294 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs b/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs index 1793d47434b..e99b328844c 100644 --- a/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs +++ b/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs @@ -152,6 +152,16 @@ public class StreamPullingAgentOptions /// public static readonly TimeSpan DEFAULT_MAX_EVENT_DELIVERY_TIME = TimeSpan.FromMinutes(1); + /// + /// Gets or sets the period between delivery progress updates. + /// + public TimeSpan DeliveryProgressUpdateInterval { get; set; } = DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL; + + /// + /// The default period between delivery progress updates. + /// + public static readonly TimeSpan DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL = TimeSpan.FromSeconds(1); + /// /// Gets or sets the stream inactivity period. /// diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index b66a28afee7..faebc3f343d 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -43,6 +43,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private IQueueAdapterReceiver? receiver; private DateTime lastTimeCleanedPubSubCache; private IGrainTimer? timer; + private ITimer? deliveryProgressTimer; private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; @@ -90,6 +91,14 @@ internal PersistentStreamPullingAgent( pubSubCache = new Dictionary(); this.options = options; options.InitialSubscriptionStartPosition.Validate(); + if (options.DeliveryProgressUpdateInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(options.DeliveryProgressUpdateInterval), + options.DeliveryProgressUpdateInterval, + "The delivery progress update interval must be greater than zero."); + } + this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler)); this.queueAdapterCache = queueAdapterCache; @@ -216,6 +225,11 @@ public async Task Initialize(CancellationToken cancellationToken) // Even if the receiver failed to initialize, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = RandomTimeSpan.Next(this.options.GetQueueMsgsTimerPeriod); timer = RegisterGrainTimer(RunQueuePump, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); + deliveryProgressTimer = _timeProvider.CreateTimer( + static state => ((PersistentStreamPullingAgent)state!).ScheduleDeliveryProgressUpdate(), + this, + this.options.DeliveryProgressUpdateInterval, + this.options.DeliveryProgressUpdateInterval); StreamingEvents.EmitPullingAgentStarted(streamProviderName, Silo, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); _streamInstruments?.RegisterPersistentStreamPubSubCacheSizeObserve(() => new Measurement(pubSubCache.Count, new KeyValuePair("name", StatisticUniquePostfix))); @@ -251,6 +265,9 @@ private async Task ShutdownCore(CancellationToken cancellationToken) var drainTask = _workAdmission.CloseAsync(); var asyncTimer = timer; timer = null; + var localDeliveryProgressTimer = deliveryProgressTimer; + deliveryProgressTimer = null; + localDeliveryProgressTimer?.Dispose(); if (asyncTimer is not null) { asyncTimer.Dispose(); @@ -1007,16 +1024,23 @@ private void CleanupPubSubCache(DateTime now) } /// - /// Computes delivery progress before shutdown so the queue can persist the latest handoff checkpoint. + /// Computes delivery progress so the queue can persist the latest handoff checkpoint. /// private void NotifyDeliveryProgress() { if (queueCache is null) return; var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - if (TryGetDeliveryProgress(out var earliest)) + try + { + if (TryGetDeliveryProgress(out var earliest)) + { + queueCache.UpdateDeliveryProgress(earliest, utcNow); + } + } + catch (ArgumentException exception) { - queueCache.UpdateDeliveryProgress(earliest, utcNow); + LogWarningDeliveryProgressComparison(new(QueueId), exception); } } @@ -1070,6 +1094,24 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return true; } + private void ScheduleDeliveryProgressUpdate() + { + this.RunOrQueueTask(() => + { + if (!IsShutdown && deliveryProgressTimer is not null) + { + NotifyDeliveryProgress(); + } + + return Task.CompletedTask; + }) + .LogException( + logger, + ErrorCode.PersistentStreamPullingAgent_28, + $"Failed to update delivery progress for queue {QueueId}.") + .Ignore(); + } + private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => EventSequenceTokenCompatibility.Compare(current, other) < 0; @@ -1899,6 +1941,13 @@ private readonly struct QueueIdLogRecord(QueueId queueId) )] private partial void LogWarningMessagesDeliveredAsync(QueueIdLogRecord myQueueId, Exception exception); + [LoggerMessage( + Level = LogLevel.Warning, + EventId = (int)ErrorCode.PersistentStreamPullingAgent_28, + Message = "Unable to compare delivery progress tokens for queue {QueueId}. The checkpoint will not advance." + )] + private partial void LogWarningDeliveryProgressComparison(QueueIdLogRecord queueId, Exception exception); + [LoggerMessage( Level = LogLevel.Information, EventId = (int)ErrorCode.PersistentStreamPullingAgent_24, diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 609eca4357d..65496c3dc3b 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -152,6 +152,7 @@ public partial class StreamPubSubOptions public partial class StreamPullingAgentOptions { public static readonly int DEFAULT_BATCH_CONTAINER_BATCH_SIZE; + public static readonly System.TimeSpan DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL; public static readonly System.TimeSpan DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD; public static readonly System.TimeSpan DEFAULT_INIT_QUEUE_TIMEOUT; public static readonly Streams.StreamSubscriptionStartPosition DEFAULT_INITIAL_SUBSCRIPTION_START_POSITION; @@ -159,6 +160,8 @@ public partial class StreamPullingAgentOptions public static readonly System.TimeSpan DEFAULT_STREAM_INACTIVITY_PERIOD; public int BatchContainerBatchSize { get { throw null; } set { } } + public System.TimeSpan DeliveryProgressUpdateInterval { get { throw null; } set { } } + public System.TimeSpan GetQueueMsgsTimerPeriod { get { throw null; } set { } } public Streams.StreamSubscriptionStartPosition InitialSubscriptionStartPosition { get { throw null; } set { } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index fb2b5e73f6c..8d00ef5d1ab 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -460,6 +460,7 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, StreamPullingAgentOptions? options = null, + IStreamFilter? streamFilter = null, IStreamFailureHandler? failureHandler = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); @@ -497,7 +498,7 @@ private static PersistentStreamPullingAgent CreateAgent( SystemTargetGrainId.Create(SystemTargetGrainId.CreateGrainType("persistent-stream-pulling-agent-test"), siloAddress), "provider", pubSub!, - new NoOpStreamFilter(), + streamFilter ?? new NoOpStreamFilter(), queueId, options ?? new StreamPullingAgentOptions(), queueAdapter, @@ -520,8 +521,12 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke private sealed class RecordingQueueCache : IQueueCache { + private TaskCompletionSource deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int DeliveryProgressCallCount { get; private set; } public List DeliveryProgressTokens { get; } = new(); + public List DeliveryProgressUtcTimes { get; } = new(); + public Task DeliveryProgressUpdated => deliveryProgressUpdated.Task; public int GetMaxAddCount() => 1000; @@ -546,12 +551,16 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke { DeliveryProgressCallCount++; DeliveryProgressTokens.Add(earliestSubscriptionToken); + DeliveryProgressUtcTimes.Add(utcNow); + deliveryProgressUpdated.TrySetResult(true); } public void ClearDeliveryProgress() { DeliveryProgressCallCount = 0; DeliveryProgressTokens.Clear(); + DeliveryProgressUtcTimes.Clear(); + deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); } } @@ -744,9 +753,11 @@ public void RecordDeliveryFailure() private sealed class ScriptedQueueCache : IQueueCache { private readonly List messages = new(); + private TaskCompletionSource deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); public int DeliveryProgressCallCount { get; private set; } public List DeliveryProgressTokens { get; } = new(); + public Task DeliveryProgressUpdated => deliveryProgressUpdated.Task; public int GetMaxAddCount() => 1000; @@ -772,12 +783,14 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke { DeliveryProgressCallCount++; DeliveryProgressTokens.Add(earliestSubscriptionToken); + deliveryProgressUpdated.TrySetResult(true); } public void ClearDeliveryProgress() { DeliveryProgressCallCount = 0; DeliveryProgressTokens.Clear(); + deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); } } @@ -1193,6 +1206,26 @@ private sealed class DerivedEventSequenceTokenV2(long sequenceNumber, int eventI protected override Type SequenceTokenCompatibilityDomain => typeof(EventSequenceToken); } + private sealed class ProviderSequenceToken(int providerOrder, long sequenceNumber) : StreamSequenceToken + { + public int ProviderOrder { get; } = providerOrder; + public override long SequenceNumber { get; protected set; } = sequenceNumber; + public override int EventIndex { get; protected set; } + + public override int CompareTo(StreamSequenceToken? other) + => other is ProviderSequenceToken token + ? ProviderOrder.CompareTo(token.ProviderOrder) + : throw new ArgumentException($"Cannot compare {GetType()} with {other?.GetType()}.", nameof(other)); + + public override bool Equals(StreamSequenceToken? other) + => other is ProviderSequenceToken token && ProviderOrder == token.ProviderOrder && SequenceNumber == token.SequenceNumber; + } + + private sealed class RejectingStreamFilter : IStreamFilter + { + public bool ShouldDeliver(StreamId streamId, object item, string? filterData) => false; + } + private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = null) : IStreamConsumerExtension { private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -4128,6 +4161,201 @@ public async Task Shutdown_PushesFinalDeliveryProgress() // Shutdown should push a final delivery progress snapshot before tearing down. Assert.Single(queueCache.DeliveryProgressTokens); } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void EventSequenceTokens_CompareAndEqualAcrossVersions() + { + StreamSequenceToken v1 = new EventSequenceToken(10, 2); + StreamSequenceToken v2 = new EventSequenceTokenV2(10, 2); + + Assert.Equal(0, v1.CompareTo(v2)); + Assert.Equal(0, v2.CompareTo(v1)); + Assert.Equal(v1, v2); + Assert.Equal(v2, v1); + Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparison() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var receiver = Substitute.For(); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var providerEarliest = new ProviderSequenceToken(providerOrder: 95, sequenceNumber: 200); + var providerNewest = new ProviderSequenceToken(providerOrder: 200, sequenceNumber: 1); + + var earliestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + earliestConsumer.IsRegistered = true; + earliestConsumer.LastProcessedToken = providerEarliest; + + var newestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + newestConsumer.IsRegistered = true; + newestConsumer.LastProcessedToken = providerNewest; + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval - TimeSpan.FromTicks(1)); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + timeProvider.Advance(TimeSpan.FromTicks(1)); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(providerEarliest, Assert.Single(queueCache.DeliveryProgressTokens)); + Assert.Equal(timeProvider.GetUtcNow().UtcDateTime, Assert.Single(queueCache.DeliveryProgressUtcTimes)); + + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_PendingRegistrationBlocksPeriodicUpdate() + { + var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default).ReturnsForAnyArgs(_ => registration.Task); + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult>([new TestBatchContainer(streamId, new EventSequenceTokenV2(1))]), + Task.FromResult>([])); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RunQueuePump(queueId, CancellationToken.None); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + Assert.NotNull(streamData.RegistrationTask); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + registration.SetResult(new HashSet()); + await streamData.RegistrationTask!; + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_IncludesFilteredBatches() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var previousToken = new EventSequenceTokenV2(1); + var filteredToken = new EventSequenceTokenV2(2); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([new TestBatchContainer(streamId, filteredToken)])); + var queueCache = new ScriptedQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver, + queueAdapterCache, + timeProvider, + options, + new RejectingStreamFilter()); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, previousToken, timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + streamConsumer: null!, + filterData: "reject", + now: timeProvider.GetUtcNow().UtcDateTime); + consumerData.IsRegistered = true; + consumerData.LastProcessedToken = previousToken; + consumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + queueCache.ClearDeliveryProgress(); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + Assert.Equal(filteredToken, consumerData.LastProcessedToken); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(filteredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_StopsPeriodicDeliveryProgressUpdates() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var receiver = Substitute.For(); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var agent = CreateAgent(pubSub: null, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.Shutdown(); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + + Assert.Empty(queueCache.DeliveryProgressTokens); + } } } From 30a87706648c049c7edc4b01f7008c7f46d278bc Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 09:29:27 -0700 Subject: [PATCH 02/90] test(streaming): cover delivery progress barriers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../PersistentStreamPullingAgentTests.cs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 8d00ef5d1ab..330ee9c7b50 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -2422,6 +2422,29 @@ private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamC public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(rewindToken); } + private sealed class ImmediateConsumer : IStreamConsumerExtension + { + public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + => throw new NotSupportedException(); + + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + => throw new NotSupportedException(); + + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken) + { + Delivered.TrySetResult(true); + return Task.FromResult(null); + } + + public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId) => Task.FromResult(null); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4331,6 +4354,74 @@ public async Task DeliveryProgress_IncludesFilteredBatches() await testAccessor.Shutdown(); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var previousToken = new EventSequenceTokenV2(1); + var currentToken = new EventSequenceTokenV2(2); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([new TestBatchContainer(streamId, currentToken)])); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = new ScriptedQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, previousToken, timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var slowConsumer = new RecordingConsumer(); + var slowConsumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + slowConsumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + slowConsumerData.IsRegistered = true; + slowConsumerData.LastProcessedToken = previousToken; + slowConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + + var fastConsumer = new ImmediateConsumer(); + var fastConsumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + fastConsumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + fastConsumerData.IsRegistered = true; + fastConsumerData.LastProcessedToken = previousToken; + fastConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + queueCache.ClearDeliveryProgress(); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + await slowConsumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await fastConsumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await testAccessor.GetPubSubCache(); + Assert.Equal(previousToken, slowConsumerData.LastProcessedToken); + Assert.Equal(currentToken, fastConsumerData.LastProcessedToken); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(previousToken, Assert.Single(queueCache.DeliveryProgressTokens)); + + slowConsumer.ReleaseDelivery(); + await testAccessor.GetPubSubCache(); + await testAccessor.Shutdown(); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4356,6 +4447,20 @@ public async Task Shutdown_StopsPeriodicDeliveryProgressUpdates() Assert.Empty(queueCache.DeliveryProgressTokens); } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void DeliveryProgress_RejectsNonPositiveUpdateInterval() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var options = new StreamPullingAgentOptions { DeliveryProgressUpdateInterval = TimeSpan.Zero }; + + var exception = Assert.Throws(() => CreateAgent(pubSub: null, queueId, options: options)); + + Assert.Equal(nameof(options.DeliveryProgressUpdateInterval), exception.ParamName); + } } } From 54bca27cab65808828528ddbcd71b754d05827b4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 09:58:11 -0700 Subject: [PATCH 03/90] feat(streaming): extract reusable checkpointing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../DynamoDBStreamQueueCheckpointer.cs | 23 ++- .../AzureTableStreamQueueCheckpointer.cs | 187 ++++++++---------- .../Orleans.Streaming.EventHubs.cs | 4 + .../EventHubCheckpointerTests.cs | 52 +++-- 4 files changed, 136 insertions(+), 130 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 56ba751fb24..7cd46a09efe 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -11,15 +11,15 @@ namespace Orleans.Streaming.Kinesis /// internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer { - private readonly GrainStreamQueueCheckpointer _inner; + private readonly StreamQueueCheckpointer _inner; internal DynamoDBStreamQueueCheckpointer( IDynamoDBStreamCheckpointStore store, DynamoDBStreamQueueCheckpointerOptions options) { - _inner = new GrainStreamQueueCheckpointer( + _inner = new StreamQueueCheckpointer( new StreamCheckpointStoreAdapter(store), - new GrainStreamQueueCheckpointerOptions + new StreamQueueCheckpointerOptions { CheckpointComparer = StreamCheckpointComparers.Numeric, PersistInterval = options.PersistInterval, @@ -63,15 +63,22 @@ public void Update(string offset, DateTime utcNow, CancellationToken cancellatio /// public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); - private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain + private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointStore { - public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); + public async ValueTask Load(CancellationToken cancellationToken) + { + var checkpoint = await store.Load(cancellationToken).ConfigureAwait(false); + return new(checkpoint, checkpoint); + } - public ValueTask Update( + public async ValueTask Update( string checkpoint, - string expectedCheckpoint, + string expectedVersion, CancellationToken cancellationToken) - => store.Update(checkpoint, expectedCheckpoint, cancellationToken); + { + var persistedCheckpoint = await store.Update(checkpoint, expectedVersion, cancellationToken).ConfigureAwait(false); + return new(persistedCheckpoint, persistedCheckpoint); + } } } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 7c69c12b365..b6601b8c0f6 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Azure; using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Streaming.EventHubs; @@ -13,16 +14,8 @@ namespace Orleans.Streams /// public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer { - private readonly AzureTableDataManager _dataManager; - private readonly TimeSpan _persistInterval; - private readonly IComparer? _checkpointComparer; - private readonly object _lock = new(); - - private StreamQueueCheckpointEntity _entity; - private Task _inProgressSave = Task.CompletedTask; - private DateTime? _throttleSavesUntilUtc; - private string _latestCheckpoint = string.Empty; - private string _persistedCheckpoint = string.Empty; + private readonly AzureTableCheckpointStore _store; + private readonly StreamQueueCheckpointer _inner; private AzureTableStreamQueueCheckpointer( AzureTableStreamCheckpointerOptions options, @@ -45,16 +38,23 @@ private AzureTableStreamQueueCheckpointer( $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); } - _persistInterval = options.PersistInterval; - _checkpointComparer = options.CheckpointComparer ?? defaultComparer; - _dataManager = new AzureTableDataManager( + var dataManager = new AzureTableDataManager( options, loggerFactory.CreateLogger()); - _entity = StreamQueueCheckpointEntity.Create( - partitionKeyPrefix ?? options.PartitionKeyPrefix, - streamProviderName, - serviceId, - partition); + _store = new AzureTableCheckpointStore( + dataManager, + StreamQueueCheckpointEntity.Create( + partitionKeyPrefix ?? options.PartitionKeyPrefix, + streamProviderName, + serviceId, + partition)); + _inner = new StreamQueueCheckpointer( + _store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = options.CheckpointComparer ?? defaultComparer, + PersistInterval = options.PersistInterval, + }); LogCreatingCheckpointer( loggerFactory.CreateLogger(), partition, @@ -63,16 +63,7 @@ private AzureTableStreamQueueCheckpointer( } /// - public bool CheckpointExists - { - get - { - lock (_lock) - { - return !string.IsNullOrEmpty(_latestCheckpoint); - } - } - } + public bool CheckpointExists => _inner.CheckpointExists; /// /// Creates and initializes an Azure Table stream queue checkpointer. @@ -104,115 +95,105 @@ internal static async Task> Create( loggerFactory, defaultComparer, partitionKeyPrefix); - await checkpointer._dataManager.InitTableAsync(); + await checkpointer._store.Initialize(); return checkpointer; } /// - public async Task Load() - { - var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); - var checkpoint = result.Entity?.Offset ?? string.Empty; - lock (_lock) - { - if (result.Entity is not null) - { - _entity = result.Entity; - } - - _latestCheckpoint = checkpoint; - _persistedCheckpoint = checkpoint; - } + public Task Load() => Load(CancellationToken.None); - return checkpoint; - } + /// + public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); /// public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); + + /// + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); + + /// + public Task FlushAsync(CancellationToken cancellationToken) + => _inner.FlushAsync(cancellationToken); + + private sealed class AzureTableCheckpointStore( + AzureTableDataManager dataManager, + StreamQueueCheckpointEntity entity) : IStreamCheckpointStore { - ArgumentNullException.ThrowIfNull(offset); + public StreamQueueCheckpointEntity Entity { get; private set; } = entity; - lock (_lock) - { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_checkpointComparer is { } comparer - && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) - { - return; - } + public Task Initialize() => dataManager.InitTableAsync(); - _latestCheckpoint = offset; - if (_throttleSavesUntilUtc.HasValue - && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) + public async ValueTask Load(CancellationToken cancellationToken) + { + var result = await dataManager.ReadSingleTableEntryAsync( + Entity.PartitionKey, + Entity.RowKey, + cancellationToken); + if (result.Entity is null) { - return; + return new(string.Empty, string.Empty); } - _throttleSavesUntilUtc = utcNow + _persistInterval; - _inProgressSave = Save(offset); - _inProgressSave.Ignore(); + Entity = result.Entity; + return new(Entity.Offset, result.ETag ?? Entity.ETag.ToString()); } - } - /// - public async Task FlushAsync(CancellationToken cancellationToken) - { - var retryingSave = false; - while (true) + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) { - Task inProgressSave; - lock (_lock) + var updatedEntity = new StreamQueueCheckpointEntity { - inProgressSave = _inProgressSave; - } + PartitionKey = Entity.PartitionKey, + RowKey = Entity.RowKey, + Offset = checkpoint, + }; - if (retryingSave) + try { - await inProgressSave.WaitAsync(cancellationToken); - } - else - { - try + string version; + if (string.IsNullOrEmpty(expectedVersion)) { - await inProgressSave.WaitAsync(cancellationToken); + var result = await dataManager.InsertTableEntryAsync(updatedEntity); + if (!result.isSuccess) + { + return await Load(cancellationToken); + } + + version = result.eTag!; } - catch (Exception) when (!cancellationToken.IsCancellationRequested) + else { + version = await dataManager.UpdateTableEntryAsync(updatedEntity, expectedVersion); } - cancellationToken.ThrowIfCancellationRequested(); + updatedEntity.ETag = new ETag(version); + Entity = updatedEntity; + return new(checkpoint, version); } - - lock (_lock) + catch (RequestFailedException exception) when (exception.Status is 409 or 412) { - if (!ReferenceEquals(inProgressSave, _inProgressSave)) - { - retryingSave = false; - continue; - } - - if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + return await Load(cancellationToken); + } + catch (RequestFailedException exception) when (exception.Status == 404) + { + var result = await dataManager.InsertTableEntryAsync(updatedEntity); + if (!result.isSuccess) { - return; + return await Load(cancellationToken); } - _inProgressSave = Save(_latestCheckpoint); - retryingSave = true; + var version = result.eTag!; + updatedEntity.ETag = new ETag(version); + Entity = updatedEntity; + return new(checkpoint, version); } } } - private async Task Save(string checkpoint) - { - _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity); - lock (_lock) - { - _persistedCheckpoint = checkpoint; - } - } - [LoggerMessage( Level = LogLevel.Information, Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index ab34b90eaa2..f81ae0e6658 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -784,7 +784,11 @@ internal AzureTableStreamQueueCheckpointer() { } public System.Threading.Tasks.Task Load() { throw null; } + public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow) { } + + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } } public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 97846df0868..dd9d01b5c20 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -585,7 +585,6 @@ public void Update_WhenOffsetAdvances_TracksLatestCheckpoint() Assert.True(checkpointer.CheckpointExists); Assert.Equal("21", GetLatestOffset(checkpointer)); - Assert.Equal("21", GetEntityOffset(checkpointer)); } [TestSuite("BVT")] @@ -593,12 +592,12 @@ public void Update_WhenOffsetAdvances_TracksLatestCheckpoint() public void Update_WithNoComparer_TracksOpaqueCheckpoint() { var checkpointer = CreateUninitializedCheckpointer(useNumericComparer: false); + ThrottleSaves(checkpointer); checkpointer.Update("opaque-checkpoint", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); Assert.True(checkpointer.CheckpointExists); Assert.Equal("opaque-checkpoint", GetLatestOffset(checkpointer)); - Assert.Equal("opaque-checkpoint", GetEntityOffset(checkpointer)); } [TestSuite("BVT")] @@ -621,7 +620,7 @@ public void EventHubCheckpointEntity_PreservesLegacyAzureTableSchema() streamProviderName: "provider/name", partition: "partition?1", serviceId: "service#id"); - var entity = GetField(checkpointer, "_entity"); + var entity = GetEntity(checkpointer); var entityType = entity.GetType(); Assert.Equal("EventHubCheckpoints_provider_name_service_id", GetEntityPartitionKey(checkpointer)); @@ -844,53 +843,68 @@ private static AzureTableStreamQueueCheckpointer CreateUninitializedCheckpointer private static void SetPersistedOffset(AzureTableStreamQueueCheckpointer checkpointer, string offset) { - SetField(checkpointer, "_latestCheckpoint", offset); - SetField(checkpointer, "_persistedCheckpoint", offset); + var inner = GetField(checkpointer, "_inner"); + SetField(inner, "_latestCheckpoint", offset); + SetField(inner, "_persistedState", new StreamCheckpointStoreState(offset, offset)); + SetField(inner, "_throttleSavesUntilUtc", DateTime.MaxValue); SetEntityOffset(checkpointer, offset); } private static string GetLatestOffset(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_latestCheckpoint"); + => (string)GetField(GetField(checkpointer, "_inner"), "_latestCheckpoint"); private static string GetEntityOffset(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("Offset")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static string GetEntityPartitionKey(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("PartitionKey")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static string GetEntityRowKey(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("RowKey")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static void SetEntityOffset(AzureTableStreamQueueCheckpointer checkpointer, string offset) { - var entity = GetField(checkpointer, "_entity"); + var entity = GetEntity(checkpointer); entity.GetType().GetProperty("Offset")!.SetValue(entity, offset); } - private static object GetField(AzureTableStreamQueueCheckpointer checkpointer, string name) + private static object GetEntity(AzureTableStreamQueueCheckpointer checkpointer) { - var field = typeof(AzureTableStreamQueueCheckpointer).GetField( + var store = GetField(checkpointer, "_store"); + var property = store.GetType().GetProperty( + "Entity", + BindingFlags.Instance | BindingFlags.Public); + Assert.NotNull(property); + return property.GetValue(store)!; + } + + private static void ThrottleSaves(AzureTableStreamQueueCheckpointer checkpointer) + => SetField(GetField(checkpointer, "_inner"), "_throttleSavesUntilUtc", DateTime.MaxValue); + + private static object GetField(object instance, string name) + { + var field = instance.GetType().GetField( name, BindingFlags.Instance | BindingFlags.NonPublic); Assert.NotNull(field); - return field.GetValue(checkpointer)!; + return field.GetValue(instance)!; } - private static void SetField(AzureTableStreamQueueCheckpointer checkpointer, string name, object value) + private static void SetField(object instance, string name, object value) { - var field = typeof(AzureTableStreamQueueCheckpointer).GetField( + var field = instance.GetType().GetField( name, BindingFlags.Instance | BindingFlags.NonPublic); Assert.NotNull(field); - field.SetValue(checkpointer, value); + field.SetValue(instance, value); } } From 99ecb39b7aa3c0249f89aea783718d6ee771cf8e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 09:51:51 -0700 Subject: [PATCH 04/90] fix(streaming): make Kinesis recovery restart-safe Order Kinesis tokens by durable arbitrary-precision shard sequence, preserve legacy token and record formats, and advertise rewind support. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Streams/KinesisAdapterFactory.cs | 2 +- .../Streams/KinesisAdapterReceiver.cs | 3 +- .../Streams/KinesisBatchContainer.cs | 2 +- .../KinesisBatchContainerTests.cs | 83 ++++++++++++ .../KinesisRuntimeTests.cs | 126 ++++++++++++++++++ 5 files changed, 213 insertions(+), 3 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 8fbd667b379..772936973fd 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -68,7 +68,7 @@ public KinesisAdapterFactory( public string Name { get; } - public bool IsRewindable => false; + public bool IsRewindable => true; public StreamProviderDirection Direction => StreamProviderDirection.ReadWrite; diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs index dcfb81b18c0..d4e0159a17c 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs @@ -163,7 +163,8 @@ public async Task> GetQueueMessagesAsync( foreach (var record in getRecordsResponse.Records) { - // Kinesis only has a long string sequence ID, so we fake one based on the order we read from the partition. + // Retain the receiver-local ordinal for compatibility with existing serialized tokens. + // KinesisSequenceToken orders records using the durable shard sequence. batch.Add(KinesisBatchContainer.FromKinesisRecord(_serializer, record, _lastReadMessage++)); } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 97ea841a767..4d67dbbc569 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -81,7 +81,7 @@ public bool ImportRequestContext() } public int CompareTo(KinesisBatchContainer? other) - => other is null ? 1 : Token.SequenceNumber.CompareTo(other.SequenceToken.SequenceNumber); + => other is null ? 1 : Token.CompareTo(other.Token); [Serializable] [GenerateSerializer] diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index e70e7ea9de2..3c78b30c2fa 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -40,5 +40,88 @@ public void GetEventsFiltersByRequestedType() Assert.Equal([1, 3], batch.GetEvents().Select(item => item.Item1)); Assert.Equal(["two"], batch.GetEvents().Select(item => item.Item1)); + + Assert.False(batch.ImportRequestContext()); + } + + [Fact] + public void GetEventsAssignsDistinctEventIndexPerEventWithinSameRecord() + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new object[] { "first", "second", "third" }, + requestContext: null); + var record = new KinesisRecord + { + Data = new MemoryStream(payload), + SequenceNumber = "999999999999999999999", + }; + var batch = KinesisBatchContainer.FromKinesisRecord(serializer, record, sequenceId: 0); + + var tokens = batch.GetEvents() + .Select(item => (Value: item.Item1, Token: (KinesisSequenceToken)item.Item2)) + .ToArray(); + + Assert.Equal(["first", "second", "third"], tokens.Select(t => t.Value)); + Assert.Equal([0, 1, 2], tokens.Select(t => t.Token.EventIndex)); + + Assert.True(tokens[0].Token.CompareTo(tokens[1].Token) < 0); + Assert.True(tokens[1].Token.CompareTo(tokens[2].Token) < 0); + Assert.All(tokens, t => Assert.Equal(record.SequenceNumber, t.Token.ShardSequence)); + } + + [Fact] + public void OldPayloadShapeWithRequestContextDecodesUnchanged() + { + const string legacyPayload = "ICAABQkDHUgFGWxlZ2FjeS1ldmVudOAhAQNBH2xlZ2FjeS10cmFjZS1pZEgFEXRyYWNlLTQy4CFAYWxlZ2FjeS1uYW1lc3BhY2UxMTExMTExMTIyMjIzMzMzNDQ0NDU1NTU1NTU1NTU1NQEhYSSIghng4A=="; + var streamId = StreamId.Create("legacy-namespace", Guid.Parse("11111111-2222-3333-4444-555555555555")); + var requestContext = new Dictionary { ["legacy-trace-id"] = "trace-42" }; + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new object[] { 7, "legacy-event" }, + requestContext); + Assert.Equal(legacyPayload, Convert.ToBase64String(payload)); + + var record = new KinesisRecord + { + Data = new MemoryStream(Convert.FromBase64String(legacyPayload)), + SequenceNumber = "123456789012345678901234567890", + }; + + var batch = KinesisBatchContainer.FromKinesisRecord(serializer, record, sequenceId: 0); + + Assert.Equal(streamId, batch.StreamId); + Assert.Equal([7], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(["legacy-event"], batch.GetEvents().Select(item => item.Item1)); + + Assert.True(batch.ImportRequestContext()); + try + { + Assert.Equal("trace-42", RequestContext.Get("legacy-trace-id")); + } + finally + { + RequestContext.Clear(); + } + } + + [Fact] + public void CompareToOrdersByDurableShardSequenceNotReceiverLocalOrdinal() + { + var readFirstButNewer = KinesisBatchContainer.FromKinesisRecord( + serializer, + new KinesisRecord { Data = new MemoryStream(), SequenceNumber = "200000000000000000000000000000" }, + sequenceId: 0); + + var readSecondButOlder = KinesisBatchContainer.FromKinesisRecord( + serializer, + new KinesisRecord { Data = new MemoryStream(), SequenceNumber = "1" }, + sequenceId: 1); + + Assert.True(readFirstButNewer.CompareTo(readSecondButOlder) > 0); + Assert.True(readSecondButOlder.CompareTo(readFirstButNewer) < 0); } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index e539f5a364e..6796a8a71b6 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -1,8 +1,11 @@ using Amazon.Kinesis; using Amazon.Kinesis.Model; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using NSubstitute; +using Orleans.Configuration; +using Orleans.Serialization; using Orleans.Streaming.Kinesis; using Orleans.Streams; using TestExtensions; @@ -93,6 +96,44 @@ public void ExplicitRegionOverridesServiceUrl() Assert.Equal("eu-west-1", KinesisAdapterFactory.GetRegionName(options)); } + [Fact] + public void QueueAdapterFactoryIsRewindable() + { + var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + using var factory = new KinesisAdapterFactory( + "Kinesis", + new KinesisStreamOptions(), + new SimpleQueueCacheOptions(), + serializer, + checkpointerFactory: null, + NullLoggerFactory.Instance); + + Assert.True(factory.IsRewindable); + Assert.Equal(StreamProviderDirection.ReadWrite, factory.Direction); + } + + [Fact] + public async Task InitialShardIteratorUsesTrimHorizonWhenNoCheckpointExists() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + await client.Received(1).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == ShardIteratorType.TRIM_HORIZON + && request.StartingSequenceNumber == null), + Arg.Any()); + } + [Fact] public async Task TopologyMonitorLatchesWhenShardSetChanges() { @@ -149,6 +190,91 @@ await client.Received(1).GetShardIteratorAsync( Arg.Any()); } + [Fact] + public async Task ReceiverAssignsMonotonicallyIncreasingLocalOrdinalsAcrossReads() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-2", + Records = [ + new Amazon.Kinesis.Model.Record { SequenceNumber = "10", Data = new MemoryStream() }, + new Amazon.Kinesis.Model.Record { SequenceNumber = "20", Data = new MemoryStream() }, + ], + }), + Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-3", + Records = [new Amazon.Kinesis.Model.Record { SequenceNumber = "30", Data = new MemoryStream() }], + })); + var timeProvider = new FakeTimeProvider { AutoAdvanceAmount = TimeSpan.FromMilliseconds(200) }; + var receiver = CreateReceiver(client, checkpointerFactory, timeProvider); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var firstBatch = (await receiver.GetQueueMessagesAsync(10)).Cast().ToArray(); + var secondBatch = (await receiver.GetQueueMessagesAsync(10)).Cast().ToArray(); + + Assert.Equal([0L, 1L], firstBatch.Select(container => container.Token.SequenceNumber)); + Assert.Equal([2L], secondBatch.Select(container => container.Token.SequenceNumber)); + } + + [Fact] + public async Task MessagesDeliveredCommitsNumericallyHighestShardSequence() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var hugeButReadFirst = KinesisBatchContainer.FromKinesisRecord( + null!, + new Amazon.Kinesis.Model.Record { SequenceNumber = "170141183460469231731687303715884105727", Data = new MemoryStream() }, + sequenceId: 0); + var smallButReadSecond = KinesisBatchContainer.FromKinesisRecord( + null!, + new Amazon.Kinesis.Model.Record { SequenceNumber = "42", Data = new MemoryStream() }, + sequenceId: 1); + + await receiver.MessagesDeliveredAsync([hugeButReadFirst, smallButReadSecond]); + + checkpointer.Received(1).Update( + "170141183460469231731687303715884105727", + Arg.Any(), + Arg.Any()); + checkpointer.DidNotReceive().Update("42", Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task MessagesDeliveredWithEmptyListDoesNotUpdateCheckpointOrThrow() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + await receiver.MessagesDeliveredAsync(Array.Empty()); + + checkpointer.DidNotReceive().Update(Arg.Any(), Arg.Any(), Arg.Any()); + } + [Fact] public async Task ReceiverLimitsGetRecordsToFiveCallsPerSecond() { From f3c5a660df176d63a340e6f46d5f3005fa2d83cb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 10:16:14 -0700 Subject: [PATCH 05/90] feat(streaming): add recoverable partition pipeline Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../EventHub/EventHubAdapterFactory.cs | 9 +- .../IRecoverableStreamDataAdapter.cs | 37 +++ .../IRecoverableStreamSource.cs | 31 ++ .../QueueAdapterReceiverRegistry.cs | 36 +++ .../RecoverableStreamQueueCache.cs | 184 ++++++++++++ .../RecoverableStreamReceiver.cs | 155 ++++++++++ .../RecoverableStreamStartPosition.cs | 29 ++ .../Orleans.Streaming.EventHubs.cs | 8 +- .../Orleans.Streaming/Orleans.Streaming.cs | 104 +++++++ .../RecoverableStreamReceiverTests.cs | 276 ++++++++++++++++++ 10 files changed, 862 insertions(+), 7 deletions(-) create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs index 3d2fbb385d9..db52d81ea57 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -50,7 +49,7 @@ public class EventHubAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueu private readonly StreamCacheEvictionOptions cacheEvictionOptions; private HashRingBasedPartitionedStreamQueueMapper streamQueueMapper = null!; private string[] partitionIds = null!; - private ConcurrentDictionary receivers = null!; + private QueueAdapterReceiverRegistry receivers = null!; private EventHubProducerClient client = null!; /// @@ -101,7 +100,7 @@ public class EventHubAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueu /// Factory to create a IEventHubReceiver /// protected Func EventHubReceiverFactory = null!; - internal ConcurrentDictionary EventHubReceivers => receivers; + internal IReadOnlyDictionary EventHubReceivers => receivers.Receivers; internal HashRingBasedPartitionedStreamQueueMapper EventHubQueueMapper => streamQueueMapper; /// @@ -147,7 +146,7 @@ public EventHubAdapterFactory( /// public virtual void Init() { - this.receivers = new ConcurrentDictionary(); + this.receivers = new QueueAdapterReceiverRegistry(MakeReceiver); InitEventHubClient(); @@ -263,7 +262,7 @@ public IQueueCache CreateQueueCache(QueueId queueId) private EventHubAdapterReceiver GetOrCreateReceiver(QueueId queueId) { - return this.receivers.GetOrAdd(queueId, (q, instance) => instance.MakeReceiver(q), this); + return this.receivers.GetOrCreate(queueId); } /// diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs new file mode 100644 index 00000000000..db6c5a351d8 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs @@ -0,0 +1,37 @@ +using System; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Adapts immutable source records to and from pooled cache storage. + /// + /// The source record type. + public interface IRecoverableStreamDataAdapter : ICacheDataAdapter + { + /// + /// Gets the stream and provider position for a source record. + /// + StreamPosition GetStreamPosition(TQueueMessage queueMessage); + + /// + /// Packs a source record into pooled cache storage. + /// + CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment); + + /// + /// Gets the provider offset encoded in a cached message. + /// + string GetOffset(ref CachedMessage cachedMessage); + + /// + /// Tries to extract a provider offset from a delivery token. + /// + bool TryGetOffset(StreamSequenceToken token, out string offset); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs new file mode 100644 index 00000000000..dbae621b76e --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Reads ordered records from a recoverable stream partition. + /// + /// The source record type. + public interface IRecoverableStreamSource + { + /// + /// Initializes the source at the requested position. + /// + /// The durable checkpoint or configured start policy. + /// The cancellation token. + /// A durable checkpoint always takes precedence and reads must begin strictly after it. + Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken); + + /// + /// Reads an ordered batch of immutable source records. + /// + Task> Read(int maxCount, CancellationToken cancellationToken); + + /// + /// Shuts the partition source down. + /// + Task Shutdown(CancellationToken cancellationToken); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs new file mode 100644 index 00000000000..03ee2db9c8f --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Ensures receiver and cache factories share one coordinator instance per queue. + /// + /// The combined receiver and cache type. + public sealed class QueueAdapterReceiverRegistry + where TReceiver : class, IQueueAdapterReceiver, IQueueCache + { + private readonly ConcurrentDictionary _receivers = new(); + private readonly Func _factory; + + /// + /// Initializes a new instance of the class. + /// + public QueueAdapterReceiverRegistry(Func factory) + { + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + } + + /// + /// Gets the registered receiver instances. + /// + public IReadOnlyDictionary Receivers => _receivers; + + /// + /// Gets or creates the coordinator for a queue. + /// + public TReceiver GetOrCreate(QueueId queueId) => _receivers.GetOrAdd(queueId, _factory); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs new file mode 100644 index 00000000000..5826e36769d --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Provider-neutral pooled cache for immutable recoverable stream records. + /// + /// The source record type. + public sealed class RecoverableStreamQueueCache : IQueueCache, IDisposable + { + private readonly int _defaultMaxAddCount; + private readonly IObjectPool _bufferPool; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly IEvictionStrategy _evictionStrategy; + private readonly IQueueFlowController? _flowController; + private readonly PooledQueueCache _cache; + private FixedSizeBuffer? _currentBuffer; + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamQueueCache( + int defaultMaxAddCount, + IObjectPool bufferPool, + IRecoverableStreamDataAdapter dataAdapter, + IEvictionStrategy evictionStrategy, + ILogger logger, + IQueueFlowController? flowController = null, + ICacheMonitor? cacheMonitor = null, + TimeSpan? cacheMonitorWriteInterval = null, + TimeSpan? metadataMinTimeInCache = null) + { + if (defaultMaxAddCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(defaultMaxAddCount)); + } + + _defaultMaxAddCount = defaultMaxAddCount; + _bufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _evictionStrategy = evictionStrategy ?? throw new ArgumentNullException(nameof(evictionStrategy)); + _flowController = flowController; + _cache = new PooledQueueCache( + dataAdapter, + logger ?? throw new ArgumentNullException(nameof(logger)), + cacheMonitor, + cacheMonitorWriteInterval, + metadataMinTimeInCache); + _evictionStrategy.PurgeObservable = _cache; + _evictionStrategy.OnPurged = OnPurged; + } + + /// + /// Gets the most recently purged provider offset. + /// + public string? LastPurgedOffset { get; private set; } + + /// + /// Packs and adds ordered source records to the cache. + /// + public IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc) + { + ArgumentNullException.ThrowIfNull(messages); + var positions = new List(messages.Count); + var cachedMessages = new List(messages.Count); + foreach (var message in messages) + { + var position = _dataAdapter.GetStreamPosition(message); + cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetSegment)); + positions.Add(position); + } + + _cache.Add(cachedMessages, dequeueTimeUtc); + return positions; + } + + /// + public int GetMaxAddCount() + => _flowController is null + ? _defaultMaxAddCount + : Math.Min(_defaultMaxAddCount, _flowController.GetMaxAddCount()); + + /// + public void AddToCache(IList messages) + { + // Source records are admitted directly by the receiver coordinator. + } + + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + { + purgedItems = null; + if (!IsUnderPressure()) + { + _evictionStrategy.PerformPurge(DateTime.UtcNow); + } + + return false; + } + + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => new Cursor(_cache, streamId, token); + + /// + public bool IsUnderPressure() => GetMaxAddCount() <= 0; + + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + } + + /// + public void Dispose() + { + _evictionStrategy.OnPurged = null; + } + + private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) + { + if (lastPurged is { } message) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref message); + } + } + + private ArraySegment GetSegment(int size) + { + if (_currentBuffer is not null && _currentBuffer.TryGetSegment(size, out var segment)) + { + return segment; + } + + var buffer = _bufferPool.Allocate(); + if (!buffer.TryGetSegment(size, out segment)) + { + buffer.Dispose(); + throw new ArgumentOutOfRangeException(nameof(size), $"Message size is too big. MessageSize: {size}"); + } + + _currentBuffer = buffer; + _evictionStrategy.OnBlockAllocated(buffer); + return segment; + } + + private sealed class Cursor : IQueueCacheCursor + { + private readonly PooledQueueCache _cache; + private readonly object _cursor; + private IBatchContainer? _current; + + public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? token) + { + _cache = cache; + _cursor = cache.GetCursor(streamId, token); + } + + public void Dispose() + { + } + + public IBatchContainer? GetCurrent(out Exception? exception) + { + exception = null; + return _current; + } + + public bool MoveNext() => _cache.TryGetNextMessage(_cursor, out _current); + + public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); + + public void RecordDeliveryFailure() + { + } + } + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs new file mode 100644 index 00000000000..1e49f966c8f --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Coordinates a recoverable partition source, pooled cache, and durable checkpoint. + /// + /// The source record type. + public sealed class RecoverableStreamReceiver : IQueueAdapterReceiver, IQueueCache + { + private readonly IRecoverableStreamSource _source; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly RecoverableStreamQueueCache _cache; + private readonly IStreamQueueCheckpointer _checkpointer; + private readonly bool _startFromNow; + private int _running; + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + RecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) + { + _source = source ?? throw new ArgumentNullException(nameof(source)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _checkpointer = checkpointer ?? throw new ArgumentNullException(nameof(checkpointer)); + _startFromNow = startFromNow; + } + + /// + public async Task Initialize(TimeSpan timeout) + { + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + var checkpoint = await _checkpointer.Load(cancellationToken); + await _source.Initialize( + new RecoverableStreamStartPosition( + _checkpointer.CheckpointExists ? checkpoint : null, + _startFromNow), + cancellationToken); + Volatile.Write(ref _running, 1); + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> GetQueueMessagesAsync(int maxCount) + => GetQueueMessagesAsync(maxCount, CancellationToken.None); + + /// + public async Task> GetQueueMessagesAsync( + int maxCount, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref _running) == 0 || maxCount <= 0) + { + return []; + } + + var messages = await _source.Read(maxCount, cancellationToken); + if (messages.Count == 0) + { + return []; + } + + var positions = _cache.Add(messages, DateTime.UtcNow); + var result = new List(positions.Count); + foreach (var position in positions) + { + result.Add(new StreamActivityNotificationBatch(position)); + } + + return result; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task MessagesDeliveredAsync(IList messages) + => Task.CompletedTask; + + /// + public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + + /// + public async Task Shutdown(TimeSpan timeout) + { + if (Interlocked.Exchange(ref _running, 0) == 0) + { + return; + } + + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + await _checkpointer.FlushAsync(cancellationToken); + await _source.Shutdown(cancellationToken); + _cache.Dispose(); + } + + /// + public int GetMaxAddCount() => _cache.GetMaxAddCount(); + + /// + public void AddToCache(IList messages) + { + } + + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _cache.TryPurgeFromCache(out purgedItems); + + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.GetCacheCursor(streamId, token); + + /// + public bool IsUnderPressure() => _cache.IsUnderPressure(); + + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + if (earliestSubscriptionToken is not null + && _dataAdapter.TryGetOffset(earliestSubscriptionToken, out var offset)) + { + _checkpointer.Update(offset, utcNow, CancellationToken.None); + } + } + + private sealed class StreamActivityNotificationBatch(StreamPosition position) : IBatchContainer + { + public StreamId StreamId => position.StreamId; + + public StreamSequenceToken SequenceToken => position.SequenceToken; + + public IEnumerable> GetEvents() => throw new NotSupportedException(); + + public bool ImportRequestContext() => throw new NotSupportedException(); + } + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs new file mode 100644 index 00000000000..0b9af8d59a7 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs @@ -0,0 +1,29 @@ +namespace Orleans.Providers.Streams.Common +{ + /// + /// Describes where a recoverable stream source begins reading. + /// + public readonly struct RecoverableStreamStartPosition + { + /// + /// Initializes a new instance of the struct. + /// + /// The durable checkpoint, or if one does not exist. + /// Whether a source without a checkpoint starts at its current tail. + public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) + { + Checkpoint = checkpoint; + StartFromNow = startFromNow; + } + + /// + /// Gets the durable checkpoint. Sources must begin strictly after this value. + /// + public string? Checkpoint { get; } + + /// + /// Gets a value indicating whether a source without a checkpoint starts at its current tail. + /// + public bool StartFromNow { get; } + } +} diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index f81ae0e6658..aa43e70a6a1 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -303,6 +303,7 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public static EventHubAdapterFactory Create(System.IServiceProvider services, string name) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task CreateAdapter() { throw null; } protected virtual IEventHubQueueCacheFactory CreateCacheFactory(Configuration.EventHubStreamCachePressureOptions eventHubCacheOptions) { throw null; } @@ -313,6 +314,7 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public System.Threading.Tasks.Task GetDeliveryFailureHandler(Streams.QueueId queueId) { throw null; } + [System.Diagnostics.DebuggerStepThrough] protected virtual System.Threading.Tasks.Task GetPartitionIdsAsync() { throw null; } public Streams.IQueueAdapterCache GetQueueAdapterCache() { throw null; } @@ -368,6 +370,7 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -375,6 +378,7 @@ internal EventHubCheckpointer() { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } @@ -786,9 +790,9 @@ internal AzureTableStreamQueueCheckpointer() { } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } - public void Update(string offset, System.DateTime utcNow) { } - public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } + + public void Update(string offset, System.DateTime utcNow) { } } public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 65496c3dc3b..36607f8fa60 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -429,6 +429,7 @@ public MemoryAdapterFactory(string providerName, Configuration.StreamCacheEvicti [System.Diagnostics.CodeAnalysis.MemberNotNull(new[] { "CacheMonitorFactory", "BlockPoolMonitorFactory", "ReceiverMonitorFactory" })] public void Init() { } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task QueueMessageBatchAsync(Runtime.StreamId streamId, System.Collections.Generic.IEnumerable events, Orleans.Streams.StreamSequenceToken? token, System.Collections.Generic.Dictionary? requestContext) { throw null; } } @@ -799,6 +800,21 @@ public partial interface IQueueAdapterReceiverMonitor void TrackShutdown(bool success, System.TimeSpan callTime, System.Exception? exception); } + public partial interface IRecoverableStreamDataAdapter : ICacheDataAdapter + { + CachedMessage FromQueueMessage(Orleans.Streams.StreamPosition streamPosition, TQueueMessage queueMessage, System.DateTime dequeueTimeUtc, System.Func> getSegment); + string GetOffset(ref CachedMessage cachedMessage); + Orleans.Streams.StreamPosition GetStreamPosition(TQueueMessage queueMessage); + bool TryGetOffset(Orleans.Streams.StreamSequenceToken token, out string offset); + } + + public partial interface IRecoverableStreamSource + { + System.Threading.Tasks.Task Initialize(RecoverableStreamStartPosition position, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task> Read(int maxCount, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task Shutdown(System.Threading.CancellationToken cancellationToken); + } + public partial class ObjectPoolMonitorBridge : IObjectPoolMonitor { public ObjectPoolMonitorBridge(IBlockPoolMonitor blockPoolMonitor, int blockSizeInBytes) { } @@ -896,6 +912,16 @@ public virtual void OnResetState() { } public virtual void SignalPurge() { } } + public sealed partial class QueueAdapterReceiverRegistry + where TReceiver : class, Orleans.Streams.IQueueAdapterReceiver, Orleans.Streams.IQueueCache + { + public QueueAdapterReceiverRegistry(System.Func factory) { } + + public System.Collections.Generic.IReadOnlyDictionary Receivers { get { throw null; } } + + public TReceiver GetOrCreate(Orleans.Streams.QueueId queueId) { throw null; } + } + public partial class ReceiverMonitorDimensions { public ReceiverMonitorDimensions() { } @@ -905,6 +931,74 @@ public ReceiverMonitorDimensions(string queueId) { } public string QueueId { get { throw null; } set { } } } + public sealed partial class RecoverableStreamQueueCache : Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable + { + public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null) { } + + public string? LastPurgedOffset { get { throw null; } } + + public System.Collections.Generic.IReadOnlyList Add(System.Collections.Generic.IReadOnlyList messages, System.DateTime dequeueTimeUtc) { throw null; } + + public void AddToCache(System.Collections.Generic.IList messages) { } + + public void Dispose() { } + + public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public int GetMaxAddCount() { throw null; } + + public bool IsUnderPressure() { throw null; } + + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } + + public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } + } + + public sealed partial class RecoverableStreamReceiver : Orleans.Streams.IQueueAdapterReceiver, Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController + { + public RecoverableStreamReceiver(IRecoverableStreamSource source, IRecoverableStreamDataAdapter dataAdapter, RecoverableStreamQueueCache cache, Orleans.Streams.IStreamQueueCheckpointer checkpointer, bool startFromNow) { } + + public void AddToCache(System.Collections.Generic.IList messages) { } + + public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public int GetMaxAddCount() { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task> GetQueueMessagesAsync(int maxCount, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task> GetQueueMessagesAsync(int maxCount) { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task Initialize(System.TimeSpan timeout) { throw null; } + + public bool IsUnderPressure() { throw null; } + + public System.Threading.Tasks.Task MessagesDeliveredAsync(System.Collections.Generic.IList messages, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task MessagesDeliveredAsync(System.Collections.Generic.IList messages) { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task Shutdown(System.TimeSpan timeout) { throw null; } + + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } + + public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } + } + + public readonly partial struct RecoverableStreamStartPosition + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) { } + + public string? Checkpoint { get { throw null; } } + + public bool StartFromNow { get { throw null; } } + } + public static partial class SegmentBuilder { public static void Append(System.ArraySegment segment, ref int writerOffset, System.ReadOnlySpan bytes) { } @@ -1543,13 +1637,17 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain) { } public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient) { throw null; } @@ -1876,10 +1974,12 @@ public LeaseBasedQueueBalancer(string name, Configuration.LeaseBasedQueueBalance public override System.Collections.Generic.IEnumerable GetMyQueues() { throw null; } + [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Initialize(IStreamQueueMapper queueMapper) { throw null; } protected override void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos) { } + [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Shutdown() { throw null; } } @@ -2010,6 +2110,7 @@ protected QueueBalancerBase(System.IServiceProvider sp, Microsoft.Extensions.Log protected System.Threading.Tasks.Task NotifyListeners() { throw null; } protected abstract void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos); + [System.Diagnostics.DebuggerStepThrough] public virtual System.Threading.Tasks.Task Shutdown() { throw null; } public bool SubscribeToQueueDistributionChangeEvents(IStreamQueueBalanceListener observer) { throw null; } @@ -2222,6 +2323,7 @@ public StreamCheckpointGrain(Runtime.IPersistentState Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.ValueTask Update(string offset, string expectedCheckpoint, System.Threading.CancellationToken cancellationToken) { throw null; } } @@ -2326,11 +2428,13 @@ public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpoi public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs new file mode 100644 index 00000000000..af43dcbafc7 --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -0,0 +1,276 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streams; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +[TestCategory("BVT")] +public sealed class RecoverableStreamReceiverTests +{ + [Fact] + public async Task Receiver_ResumesAfterCheckpointAndPersistsDeliveryProgress() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource( + [ + new TestQueueMessage(streamId, 11, "payload"), + ]); + var adapter = new TestDataAdapter(); + var bufferPool = new ObjectPool(() => new FixedSizeBuffer(4 * 1024)); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + var checkpointer = new TestCheckpointer("10"); + var receiver = new RecoverableStreamReceiver( + source, + adapter, + cache, + checkpointer, + startFromNow: true); + + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.Equal("10", source.StartPosition.Checkpoint); + Assert.True(source.StartPosition.StartFromNow); + var notifications = await receiver.GetQueueMessagesAsync(100, CancellationToken.None); + var notification = Assert.Single(notifications); + Assert.Equal(streamId, notification.StreamId); + Assert.Equal(11, notification.SequenceToken.SequenceNumber); + + using var cursor = receiver.GetCacheCursor(streamId, notification.SequenceToken); + Assert.True(cursor.MoveNext()); + var batch = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); + Assert.Equal("payload", batch.Payload); + Assert.True(adapter.CompareCallCount > 0); + + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(11), DateTime.UtcNow); + Assert.Equal("11", checkpointer.LastUpdatedCheckpoint); + + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, checkpointer.FlushCount); + Assert.True(source.IsShutdown); + } + + [Fact] + public void Registry_ReturnsSameReceiverAndCacheInstanceForQueue() + { + var created = 0; + var registry = new QueueAdapterReceiverRegistry(_ => + { + created++; + return new TestCombinedReceiver(); + }); + var queue = QueueId.GetQueueId("queue", 0, 0); + + IQueueAdapterReceiver receiver = registry.GetOrCreate(queue); + IQueueCache cache = registry.GetOrCreate(queue); + + Assert.Same(receiver, cache); + Assert.Equal(1, created); + Assert.Single(registry.Receivers); + } + + private sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); + + private sealed class TestDataAdapter : IRecoverableStreamDataAdapter + { + public int CompareCallCount { get; private set; } + + public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) + => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TestQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + var size = SegmentBuilder.CalculateAppendSize(queueMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + + SegmentBuilder.CalculateAppendSize(queueMessage.Payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture)); + SegmentBuilder.Append(segment, ref offset, queueMessage.Payload); + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.SequenceNumber, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = dequeueTimeUtc, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + var offset = 0; + _ = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); + var payload = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); + return new TestBatchContainer( + cachedMessage.StreamId, + GetSequenceToken(ref cachedMessage), + payload!); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex); + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + { + CompareCallCount++; + return cachedMessage.SequenceNumber != token.SequenceNumber + ? cachedMessage.SequenceNumber.CompareTo(token.SequenceNumber) + : cachedMessage.EventIndex.CompareTo(token.EventIndex); + } + + public string GetOffset(ref CachedMessage cachedMessage) + { + var offset = 0; + return SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + } + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return true; + } + } + + private sealed class TestBatchContainer( + StreamId streamId, + StreamSequenceToken sequenceToken, + string payload) : IBatchContainer + { + public StreamId StreamId { get; } = streamId; + + public StreamSequenceToken SequenceToken { get; } = sequenceToken; + + public string Payload { get; } = payload; + + public IEnumerable> GetEvents() => []; + + public bool ImportRequestContext() => false; + } + + private sealed class TestSource(IReadOnlyList messages) : IRecoverableStreamSource + { + private bool read; + + public RecoverableStreamStartPosition StartPosition { get; private set; } + + public bool IsShutdown { get; private set; } + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + StartPosition = position; + return Task.CompletedTask; + } + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (read) + { + return Task.FromResult>([]); + } + + read = true; + return Task.FromResult(messages); + } + + public Task Shutdown(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IsShutdown = true; + return Task.CompletedTask; + } + } + + private sealed class TestCheckpointer(string checkpoint) : IStreamQueueCheckpointer + { + public bool CheckpointExists => !string.IsNullOrEmpty(checkpoint); + + public string? LastUpdatedCheckpoint { get; private set; } + + public int FlushCount { get; private set; } + + public Task Load() => Task.FromResult(checkpoint); + + public Task Load(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.FromResult(checkpoint); + + public void Update(string offset, DateTime utcNow) => LastUpdatedCheckpoint = offset; + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + LastUpdatedCheckpoint = offset; + } + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + FlushCount++; + return Task.CompletedTask; + } + } + + private sealed class NoOpEvictionStrategy : IEvictionStrategy + { + public IPurgeObservable PurgeObservable { private get; set; } = null!; + + public Action? OnPurged { get; set; } + + public void PerformPurge(DateTime utcNow) + { + } + + public void OnBlockAllocated(FixedSizeBuffer newBlock) + { + } + } + + private sealed class TestCombinedReceiver : IQueueAdapterReceiver, IQueueCache + { + public Task Initialize(TimeSpan timeout) => Task.CompletedTask; + + public Task> GetQueueMessagesAsync(int maxCount) + => Task.FromResult>([]); + + public Task MessagesDeliveredAsync(IList messages) => Task.CompletedTask; + + public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; + + public int GetMaxAddCount() => 1; + + public void AddToCache(IList messages) + { + } + + public bool TryPurgeFromCache(out IList purgedItems) + { + purgedItems = null!; + return false; + } + + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => throw new NotSupportedException(); + + public bool IsUnderPressure() => false; + } +} From 603918f9c88d62dc5ec536efdba21bd27a5ef093 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 10:12:02 -0700 Subject: [PATCH 06/90] feat(streaming)!: add ADO.NET retained partition log Replace the alpha destructive queue schema with versioned immutable partition logs, epoch-fenced checkpoints, bounded retention cleanup, and updated configuration and benchmarks. BREAKING CHANGE: The alpha ADO.NET streaming schema and queue-oriented options are incompatible with this retained-log model and require destructive schema recreation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../docs/streaming/stream-providers.md | 6 +- .../AdoNetBatchContainer.cs | 2 +- .../AdoNetQueueAdapter.cs | 16 +- .../AdoNetQueueAdapterReceiver.cs | 241 +--- .../AdoNetStreamCheckpointUpdate.cs | 12 + .../AdoNetStreamCleanupResult.cs | 15 + .../AdoNetStreamConfirmation.cs | 13 - .../AdoNetStreamConfirmationAck.cs | 15 - .../AdoNetStreamDeadLetter.cs | 23 - .../AdoNetStreamFailureHandler.cs | 68 +- .../AdoNetStreamMessage.cs | 17 +- .../AdoNetStreamOptions.cs | 41 +- .../AdoNetStreamOptionsValidator.cs | 29 +- .../AdoNetStreamPartitionState.cs | 13 + .../Orleans.Streaming.AdoNet/Extensions.cs | 20 - .../MySQL-Streaming.sql | 1098 ++++++-------- .../PostgreSQL-Streaming.sql | 982 +++++-------- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 15 + .../SQLServer-Streaming.sql | 1265 +++++++---------- src/AdoNet/Shared/Storage/DbStoredQueries.cs | 134 +- .../Storage/RelationalOrleansQueries.cs | 295 ++-- .../Orleans.Streaming.AdoNet.cs | 22 +- .../Streaming/MessageDequeueingBenchmark.cs | 127 -- .../Streaming/MessageQueueingBenchmark.cs | 89 -- .../Streaming/RetainedLogAppendBenchmark.cs | 84 ++ .../RetainedLogReadCheckpointBenchmark.cs | 257 ++++ .../AdoNetOptionsValidatorTests.cs | 139 ++ .../Streaming/AdoNetBatchContainerTests.cs | 22 +- .../AdoNetQueueAdapterReceiverTests.cs | 552 ------- .../Streaming/AdoNetQueueAdapterTests.cs | 107 +- .../AdoNetStreamFailureHandlerTests.cs | 222 --- .../AdoNetStreamMessageStreamIdTests.cs | 131 ++ .../AdoNetStreamPartitionLogTests.cs | 979 +++++++++++++ .../Streaming/AdoNetStreamSchemaTests.cs | 75 + .../RelationalOrleansQueriesTests.cs | 1230 ---------------- 35 files changed, 3480 insertions(+), 4876 deletions(-) create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs delete mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs delete mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs delete mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs delete mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs delete mode 100644 test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs delete mode 100644 test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs create mode 100644 test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs create mode 100644 test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs delete mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs delete mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs create mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs create mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs create mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs delete mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index e75fa76a6e0..95d289ebda3 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -18,7 +18,7 @@ A stream provider connects the Orleans streaming API to a transport and defines | Azure Event Hubs | [`Microsoft.Orleans.Streaming.EventHubs`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.EventHubs) | Stable | Yes, within Event Hubs retention | Yes | Event Hubs namespace, hub, consumer group, and checkpoint storage | | Amazon Kinesis | [`Microsoft.Orleans.Streaming.Kinesis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Kinesis) | Stable | Yes, within Kinesis retention | Yes | Kinesis data stream, AWS credentials, region, and durable checkpoint storage | | Amazon SQS | [`Microsoft.Orleans.Streaming.SQS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.SQS) | Stable | Yes, within SQS retention | No | AWS account, queue permissions, region/endpoint configuration | -| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational tables until expiry/dead-letter eviction | No | Supported database, ADO.NET driver, and Orleans streaming SQL schema | +| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in a retained relational partition log | No | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | | NATS JetStream | [`Microsoft.Orleans.Streaming.NATS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.NATS) | **Alpha** | Configurable; file storage is the default | No | NATS server with JetStream and sufficient storage; subject/stream administration | | Redis Streams | [`Microsoft.Orleans.Streaming.Redis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Redis) | **Alpha** | Configurable through Redis persistence and stream retention | Yes, while entries remain | Redis deployment, persistence/HA policy, and retention sizing | @@ -70,7 +70,9 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S ## ADO.NET streaming (alpha) -Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with `AddAdoNetStreams`. Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Messages are durable in relational tables but expire and can move to dead letters according to `AdoNetStreamOptions`. The provider isn't rewindable. +Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue is an immutable, ordered partition log with a durable, ownership-fenced checkpoint. retains checkpointed messages for one day by default. can impose a hard storage ceiling, but it can delete unprocessed messages and therefore produces gap diagnostics. Cleanup is bounded by . + +The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. ## NATS JetStream streaming (alpha) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs index b02b90394d5..713fff77e5d 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs @@ -81,7 +81,7 @@ public static AdoNetBatchContainer FromMessage(Serializer // A stored stream message always contains a serialized batch container. var container = serializer.Deserialize(message.Payload)!; container.SequenceToken = new(message.MessageId); - container.Dequeued = message.Dequeued; + container.Dequeued = 0; return container; } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index 3aaaa5ec07a..f01b513eda3 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -45,14 +45,20 @@ public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable ev // create the payload from the events var payload = AdoNetBatchContainer.ToMessagePayload(serializer, streamId, events.Cast().ToList(), requestContext); - // we can enqueue the message now + // Append the immutable payload together with the canonical stream identifier. try { - await queries.QueueStreamMessageAsync(clusterOptions.ServiceId, Name, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); + await queries.AppendStreamMessageAsync( + clusterOptions.ServiceId, + Name, + queueId, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + payload); } catch (Exception ex) { - LogFailedToQueueStreamMessage(ex, clusterOptions.ServiceId, Name, queueId); + LogFailedToAppendStreamMessage(ex, clusterOptions.ServiceId, Name, queueId); throw; } } @@ -64,8 +70,8 @@ public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable ev #region Logging - [LoggerMessage(1, LogLevel.Error, "Failed to queue stream message with ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogFailedToQueueStreamMessage(Exception ex, string serviceId, string providerId, string queueId); + [LoggerMessage(1, LogLevel.Error, "Failed to append stream message with ({ServiceId}, {ProviderId}, {QueueId})")] + private partial void LogFailedToAppendStreamMessage(Exception ex, string serviceId, string providerId, string queueId); #endregion Logging } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index e6284795d69..0d2245bd84a 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -1,42 +1,12 @@ -using Orleans.Internal; - namespace Orleans.Streaming.AdoNet; -internal interface IStreamMessageQueries -{ - Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize); - - Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages); - - Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages); -} - /// /// Receives message batches from an individual queue of an ADO.NET provider. /// -internal partial class AdoNetQueueAdapterReceiver(string providerId, string queueId, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, IStreamMessageQueries queries, Serializer serializer, ILogger logger) : IQueueAdapterReceiver +internal class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver { - private readonly ILogger _logger = logger; - private readonly AdmissionGate _admissionGate = new(); - private readonly object _lock = new(); - private readonly Dictionary _pendingMessages = []; + private readonly string _providerId; + private readonly string _queueId; public AdoNetQueueAdapterReceiver( string providerId, @@ -47,204 +17,35 @@ public AdoNetQueueAdapterReceiver( RelationalOrleansQueries queries, Serializer serializer, ILogger logger) - : this(providerId, queueId, streamOptions, clusterOptions, cacheOptions, new RelationalStreamMessageQueries(queries), serializer, logger) { + _providerId = providerId; + _queueId = queueId; + _ = streamOptions; + _ = clusterOptions; + _ = cacheOptions; + _ = queries; + _ = serializer; + _ = logger; } /// - /// Completes initialization immediately. + /// Receiver integration is supplied by the shared recoverable stream pipeline. /// - public Task Initialize(TimeSpan timeout) => Task.CompletedTask; + public Task Initialize(TimeSpan timeout) => Task.FromException(CreatePipelineUnavailableException()); /// - /// Closes admission, drains accepted operations, and releases pending messages. + /// No receiver work is started by this storage-only implementation. /// - public async Task Shutdown(TimeSpan timeout) - { - try - { - await _admissionGate.CloseAsync().WaitAsync(timeout); - } - catch (Exception ex) - { - LogShutdownFault(ex, clusterOptions.ServiceId, providerId, queueId); - return; - } - - List pending; - lock (_lock) - { - RemoveExpiredPendingMessages(); - if (_pendingMessages.Count == 0) - { - return; - } - - pending = _pendingMessages - .Select(static item => new AdoNetStreamConfirmation(item.Key, item.Value.Dequeued)) - .ToList(); - } - - try - { - var released = await queries.ReleaseStreamMessagesAsync(clusterOptions.ServiceId, providerId, queueId, pending).WaitAsync(timeout); - lock (_lock) - { - foreach (var message in released) - { - _pendingMessages.Remove(message.MessageId); - } - } - } - catch (Exception ex) - { - LogReleaseFailed(ex, clusterOptions.ServiceId, providerId, queueId, pending); - } - } + public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; /// - public async Task> GetQueueMessagesAsync(int maxCount) - { - using var admission = _admissionGate.TryEnter(); - if (!admission.Entered) - { - return []; - } - - // cap max count as appropriate - maxCount = Math.Min(maxCount, cacheOptions.CacheSize); - - try - { - var messages = await queries.GetStreamMessagesAsync( - clusterOptions.ServiceId, - providerId, - queueId, - maxCount, - streamOptions.MaxAttempts, - streamOptions.VisibilityTimeout.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - - lock (_lock) - { - RemoveExpiredPendingMessages(); - foreach (var message in messages) - { - _pendingMessages[message.MessageId] = new(message.Dequeued, message.ExpiresOn); - } - } - - // convert the messages into standard batch containers - return messages.Select(x => AdoNetBatchContainer.FromMessage(serializer, x)).Cast().ToList(); - } - catch (Exception ex) - { - LogDequeueFailed(ex, clusterOptions.ServiceId, providerId, queueId); - throw; - } - } + public Task> GetQueueMessagesAsync(int maxCount) => + Task.FromException>(CreatePipelineUnavailableException()); /// - public async Task MessagesDeliveredAsync(IList messages) - { - // skip work if there are no messages to deliver - if (messages.Count == 0) - { - return; - } - - using var admission = _admissionGate.TryEnter(); - if (!admission.Entered) - { - return; - } - - // get the identifiers for the messages to confirm - var items = messages.Cast().Select(x => new AdoNetStreamConfirmation(x.SequenceToken.SequenceNumber, x.Dequeued)).ToList(); - - try - { - var confirmed = await queries.ConfirmStreamMessagesAsync(clusterOptions.ServiceId, providerId, queueId, items); - var receipts = items.ToDictionary(static item => item.MessageId, static item => item.Dequeued); - lock (_lock) - { - foreach (var message in confirmed) - { - if (receipts.TryGetValue(message.MessageId, out var receipt) - && _pendingMessages.TryGetValue(message.MessageId, out var pending) - && receipt == pending.Dequeued) - { - _pendingMessages.Remove(message.MessageId); - } - } - } - } - catch (Exception ex) - { - LogConfirmationFailed(ex, clusterOptions.ServiceId, providerId, queueId, items); - throw; - } - } - - private void RemoveExpiredPendingMessages() - { - var now = DateTime.UtcNow; - var expired = _pendingMessages - .Where(message => message.Value.ExpiresOn <= now) - .Select(static message => message.Key) - .ToList(); - foreach (var messageId in expired) - { - _pendingMessages.Remove(messageId); - } - } - - private readonly record struct PendingMessage(int Dequeued, DateTime ExpiresOn); - - private sealed class RelationalStreamMessageQueries(RelationalOrleansQueries queries) : IStreamMessageQueries - { - public Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize) => - queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - public Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) => - queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, messages); - - public Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) => - queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, messages); - } - - #region Logging - - [LoggerMessage(1, LogLevel.Error, "Failed to get messages from ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogDequeueFailed(Exception exception, string serviceId, string providerId, string queueId); - - [LoggerMessage(2, LogLevel.Error, "Failed to confirm messages for ({ServiceId}, {ProviderId}, {QueueId}, {@Items})")] - private partial void LogConfirmationFailed(Exception exception, string serviceId, string providerId, string queueId, List items); - - [LoggerMessage(3, LogLevel.Warning, "Handled fault while shutting down receiver for ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogShutdownFault(Exception exception, string serviceId, string providerId, string queueId); - - [LoggerMessage(4, LogLevel.Warning, "Failed to release messages while shutting down receiver for ({ServiceId}, {ProviderId}, {QueueId}, {@Items})")] - private partial void LogReleaseFailed(Exception exception, string serviceId, string providerId, string queueId, List items); + public Task MessagesDeliveredAsync(IList messages) => + Task.FromException(CreatePipelineUnavailableException()); - #endregion Logging + private InvalidOperationException CreatePipelineUnavailableException() => + new($"The ADO.NET retained-log receiver for provider '{_providerId}', queue '{_queueId}' requires the shared recoverable stream cache and receiver pipeline."); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs new file mode 100644 index 00000000000..2c7ab53dba1 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs @@ -0,0 +1,12 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes the result of an epoch-fenced checkpoint update. +/// +internal record AdoNetStreamCheckpointUpdate( + string ServiceId, + string ProviderId, + string QueueId, + long OwnerEpoch, + long? Checkpoint, + bool Updated); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs new file mode 100644 index 00000000000..b55c620c674 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs @@ -0,0 +1,15 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes one bounded retained-log cleanup operation. +/// +internal record AdoNetStreamCleanupResult( + bool Ran, + int DeletedCount, + long? DeletedThroughMessageId, + int HardDeletedCount, + long? HardDeletedFromMessageId, + long? HardDeletedThroughMessageId, + long? Checkpoint, + long? EarliestMessageId, + long? TailMessageId); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs deleted file mode 100644 index 0c3c127a976..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a message that can be confirmed. -/// -internal record AdoNetStreamConfirmation( - long MessageId, - int Dequeued) -{ - public AdoNetStreamConfirmation() : this(0, 0) - { - } -} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs deleted file mode 100644 index a975aec7f37..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a message that was successfully confirmed. -/// -internal record AdoNetStreamConfirmationAck( - string ServiceId, - string ProviderId, - string QueueId, - long MessageId) -{ - public AdoNetStreamConfirmationAck() : this("", "", "", 0) - { - } -} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs deleted file mode 100644 index 5ceb5e81d10..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a dead letter in an ADONET streaming provider. -/// -internal record AdoNetStreamDeadLetter( - string ServiceId, - string ProviderId, - string QueueId, - long MessageId, - int Dequeued, - DateTime VisibleOn, - DateTime ExpiresOn, - DateTime CreatedOn, - DateTime ModifiedOn, - DateTime DeadOn, - DateTime RemoveOn, - byte[] Payload) -{ - public AdoNetStreamDeadLetter() : this("", "", "", 0, 0, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, []) - { - } -} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs index 31f48fcc72b..97a35ba4e6a 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs @@ -1,57 +1,43 @@ namespace Orleans.Streaming.AdoNet; /// -/// An that attempts to move the message to dead letters. +/// A placeholder for the retained-log receiver integration. /// -internal partial class AdoNetStreamFailureHandler(bool faultOnFailure, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, ILogger logger) : IStreamFailureHandler +internal class AdoNetStreamFailureHandler : IStreamFailureHandler { - private readonly ILogger _logger = logger; + public AdoNetStreamFailureHandler( + bool faultOnFailure, + AdoNetStreamOptions streamOptions, + ClusterOptions clusterOptions, + AdoNetStreamQueueMapper mapper, + RelationalOrleansQueries queries, + ILogger logger) + { + ShouldFaultSubsriptionOnError = faultOnFailure; + _ = streamOptions; + _ = clusterOptions; + _ = mapper; + _ = queries; + _ = logger; + } /// /// Gets a value indicating whether the subscription should fault when there is an error. /// - public bool ShouldFaultSubsriptionOnError { get; } = faultOnFailure; + public bool ShouldFaultSubsriptionOnError { get; } /// - /// Attempts to move the message to dead letters on delivery failure. + /// Failure handling is supplied by the shared recoverable stream pipeline. /// - public Task OnDeliveryFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => OnFailureAsync(streamProviderName, streamIdentity, sequenceToken); + public Task OnDeliveryFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => + Task.FromException(CreatePipelineUnavailableException()); /// - /// Attempts to move the message to dead letters on delivery failure. + /// Failure handling is supplied by the shared recoverable stream pipeline. /// - public Task OnSubscriptionFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => OnFailureAsync(streamProviderName, streamIdentity, sequenceToken); - - /// - /// Attempts to move the message to dead letters on delivery failure. - /// - private async Task OnFailureAsync(string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) - { - ArgumentNullException.ThrowIfNull(streamProviderName); - ArgumentNullException.ThrowIfNull(sequenceToken); - - var queueId = mapper.GetAdoNetQueueId(streamIdentity); - - try - { - await queries.FailStreamMessageAsync(clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber, streamOptions.MaxAttempts, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); - - LogMovedMessage(clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber); - } - catch (Exception ex) - { - LogFailedToMoveMessage(ex, clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber); - throw; - } - } - - #region Logging - - [LoggerMessage(1, LogLevel.Warning, "Moved failed delivery to dead letters: ({ServiceId}, {ProviderId}, {QueueId}, {MessageId})")] - private partial void LogMovedMessage(string serviceId, string providerId, string queueId, long messageId); - - [LoggerMessage(2, LogLevel.Error, "Failed to move failed delivery to dead letters: ({ServiceId}, {ProviderId}, {QueueId}, {MessageId}")] - private partial void LogFailedToMoveMessage(Exception ex, string serviceId, string providerId, string queueId, long messageId); + public Task OnSubscriptionFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => + Task.FromException(CreatePipelineUnavailableException()); - #endregion Logging -} + private static InvalidOperationException CreatePipelineUnavailableException() => + new("The ADO.NET retained-log failure policy requires the shared recoverable stream cache and receiver pipeline."); +} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs index 3d295f355a7..c5b2231ecf9 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs @@ -8,14 +8,19 @@ internal record AdoNetStreamMessage( string ProviderId, string QueueId, long MessageId, - int Dequeued, - DateTime VisibleOn, - DateTime ExpiresOn, + byte[] StreamIdBytes, + int StreamNamespaceLength, DateTime CreatedOn, - DateTime ModifiedOn, byte[] Payload) { - public AdoNetStreamMessage() : this("", "", "", 0, 0, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, []) + public AdoNetStreamMessage() : this("", "", "", 0, [], 0, DateTime.MinValue, []) { } -} + + /// + /// Gets the canonical stream identifier stored with this message. + /// + public StreamId StreamId => StreamId.Create( + StreamIdBytes.AsSpan(0, StreamNamespaceLength), + StreamIdBytes.AsSpan(StreamNamespaceLength)); +} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index 7823292b4ea..66d541bc116 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -28,38 +28,47 @@ public class AdoNetStreamOptions public DbDataSource? DataSource { get; set; } /// - /// The maximum number of attempts to deliver a message. - /// The message is eventually moved to dead letters if these many attempts are made without success. + /// Gets or sets a value indicating whether a new partition checkpoint starts at the retained log tail. /// - public int MaxAttempts { get; set; } = 5; + /// + /// When , a new checkpoint starts immediately before the earliest retained message. + /// This setting is only used while initializing a partition which does not have a checkpoint. + /// + public bool StartFromNow { get; set; } /// - /// The timeout until a message is allowed to be dequeued again if not yet confirmed. + /// Gets or sets the maximum number of messages returned by a partition read. /// - public TimeSpan VisibilityTimeout { get; set; } = TimeSpan.FromMinutes(1); + public int MaxMessagesPerRead { get; set; } = 1_000; /// - /// The expiry timeout until a message is considered expired and moved to dead letters regardless of attempts. - /// The message is only moved if the current attempt is also past its visibility timeout. + /// Gets or sets the interval between checkpoint persistence attempts. /// - public TimeSpan ExpiryTimeout { get; set; } = TimeSpan.FromMinutes(10); + public TimeSpan CheckpointPersistInterval { get; set; } = TimeSpan.FromSeconds(5); /// - /// The removal timeout until a failed message is deleted from the dead letters table. + /// Gets or sets the minimum amount of time that a message is retained after it is checkpointed. /// - public TimeSpan DeadLetterEvictionTimeout { get; set; } = TimeSpan.FromDays(7); + public TimeSpan RetentionPeriod { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets an optional hard retention ceiling. + /// + /// + /// Messages older than this value can be deleted even when they are newer than the checkpoint. + /// Such deletions are reported by storage so that the receiver can emit gap diagnostics. + /// + public TimeSpan? MaximumRetentionPeriod { get; set; } /// - /// The period of time between eviction activities. - /// These include moving expired messages to dead letters and removing dead letters after their own lifetime. - /// This period is cluster wide and will not change with the number of silos. + /// Gets or sets the interval between cleanup attempts for a partition. /// - public TimeSpan EvictionInterval { get; set; } = TimeSpan.FromSeconds(10); + public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(1); /// - /// The maximum number of messages affected by an eviction batch. + /// Gets or sets the maximum number of messages deleted by one cleanup operation. /// - public int EvictionBatchSize { get; set; } = 1000; + public int CleanupBatchSize { get; set; } = 1_000; /// /// A safety timeout for underlying database initialization. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs index a55326e5f47..a9ea7669196 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs @@ -20,34 +20,39 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': configure exactly one of {nameof(options.ConnectionString)} or {nameof(options.DataSource)}."); } - if (options.MaxAttempts < 0) + if (options.MaxMessagesPerRead <= 0) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxAttempts)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxMessagesPerRead)} must be greater than zero."); } - if (options.VisibilityTimeout < TimeSpan.Zero) + if (options.CheckpointPersistInterval <= TimeSpan.Zero) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.VisibilityTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CheckpointPersistInterval)} must be greater than zero."); } - if (options.EvictionInterval < TimeSpan.Zero) + if (options.RetentionPeriod <= TimeSpan.Zero) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.EvictionInterval)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.RetentionPeriod)} must be greater than zero."); } - if (options.ExpiryTimeout < TimeSpan.Zero) + if (options.MaximumRetentionPeriod is { } maximumRetentionPeriod && maximumRetentionPeriod < options.RetentionPeriod) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.ExpiryTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaximumRetentionPeriod)} must be greater than or equal to {nameof(options.RetentionPeriod)}."); } - if (options.DeadLetterEvictionTimeout < TimeSpan.Zero) + if (options.CleanupInterval <= TimeSpan.Zero) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.DeadLetterEvictionTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupInterval)} must be greater than zero."); } - if (options.EvictionBatchSize < 0) + if (options.CleanupBatchSize <= 0) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.EvictionBatchSize)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupBatchSize)} must be greater than zero."); + } + + if (options.InitializationTimeout <= TimeSpan.Zero) + { + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.InitializationTimeout)} must be greater than zero."); } } } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs new file mode 100644 index 00000000000..0d8e44f7933 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs @@ -0,0 +1,13 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes the durable position and retained bounds of a stream partition. +/// +internal record AdoNetStreamPartitionState( + string ServiceId, + string ProviderId, + string QueueId, + long OwnerEpoch, + long? Checkpoint, + long? EarliestMessageId, + long? TailMessageId); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs deleted file mode 100644 index 0ae0149b736..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// Internal syntax sugar. -/// -internal static class Extensions -{ - /// - public static int Int32Ceiling(this double value) => (int)Math.Ceiling(value); - - /// - /// Rounds up the specified time span to the nearest upper second and returns the total number of seconds as an integer. - /// - public static int TotalSecondsCeiling(this TimeSpan value) => value.TotalSeconds.Int32Ceiling(); - - /// - /// Rounds up the specified time span to the nearest upper second and returns the total number of seconds as an integer. - /// - public static TimeSpan SecondsCeiling(this TimeSpan value) => TimeSpan.FromSeconds(value.TotalSecondsCeiling()); -} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 7caa8b2e4e0..26712881988 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -1,389 +1,166 @@ -CREATE TABLE OrleansStreamMessageSequence -( - MessageId BIGINT NOT NULL -); -INSERT INTO OrleansStreamMessageSequence -SELECT 0 -WHERE NOT EXISTS (SELECT * FROM OrleansStreamMessageSequence); - -DELIMITER $$ - -CREATE TABLE OrleansStreamMessage -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME(6) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload LONGBLOB NOT NULL, - - /* This PK supports the various ordered scanning queries. */ - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); - -DELIMITER $$ - -CREATE TABLE OrleansStreamDeadLetter -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, +/* +ADO.NET streaming schema version 2. - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. +*/ - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was given up on - troubleshooting only */ - DeadOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event is scheduled to be removed from dead letters */ - RemoveOn DATETIME(6) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload LONGBLOB NULL, - - /* This PK supports the various ordered scanning queries. */ - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); - -DELIMITER $$ - -CREATE TABLE OrleansStreamControl -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider */ - QueueId NVARCHAR(150) NOT NULL, - - /* The next due schedule for messages to be evicted */ - EvictOn DATETIME(6) NOT NULL, - - /* Each row represents a flat configuration object for an individual queue */ - PRIMARY KEY (ServiceId, ProviderId, QueueId) -); +DROP PROCEDURE IF EXISTS ValidateOrleansStreamingSchemaUpgrade; DELIMITER $$ -CREATE PROCEDURE QueueStreamMessage -( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _Payload LONGBLOB, - IN _ExpiryTimeout INT -) +CREATE PROCEDURE ValidateOrleansStreamingSchemaUpgrade() BEGIN + IF EXISTS + ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = DATABASE() + AND table_name IN + ( + 'OrleansStreamPartition', + 'OrleansStreamMessage', + 'OrleansStreamDeadLetter', + 'OrleansStreamControl', + 'OrleansStreamMessageSequence' + ) + ) + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) + THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'Incompatible alpha ADO.NET streaming schema. Drop old streaming objects and query rows; no in-place migration.'; + END IF; +END$$ -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _ExpiresOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _ExpiryTimeout SECOND); -DECLARE _MessageId BIGINT; - -UPDATE OrleansStreamMessageSequence -SET MessageId = LAST_INSERT_ID(MessageId + 1); +DELIMITER ; -SET _MessageId = LAST_INSERT_ID(); +CALL ValidateOrleansStreamingSchemaUpgrade(); +DROP PROCEDURE ValidateOrleansStreamingSchemaUpgrade; -INSERT INTO OrleansStreamMessage -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -VALUES +CREATE TABLE OrleansStreamPartition ( - _ServiceId, - _ProviderId, - _QueueId, - _MessageId, - 0, - _Now, - _ExpiresOn, - _Now, - _Now, - _Payload -); - -SELECT - _ServiceId AS ServiceId, - _ProviderId AS ProviderId, - _QueueId AS QueueId, - _MessageId AS MessageId; - -END; + ServiceId VARCHAR(150) NOT NULL, + ProviderId VARCHAR(150) NOT NULL, + QueueId VARCHAR(150) NOT NULL, + NextMessageId BIGINT NOT NULL, + Checkpoint BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn DATETIME(6) NOT NULL, + CreatedOn DATETIME(6) NOT NULL, + ModifiedOn DATETIME(6) NOT NULL, + + PRIMARY KEY (ServiceId, ProviderId, QueueId) +) ENGINE = InnoDB; -DELIMITER $$ - -INSERT INTO OrleansQuery +CREATE TABLE OrleansStreamMessage ( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'CALL QueueStreamMessage(@ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout)' + ServiceId VARCHAR(150) NOT NULL, + ProviderId VARCHAR(150) NOT NULL, + QueueId VARCHAR(150) NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes LONGBLOB NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn DATETIME(6) NOT NULL, + Payload LONGBLOB NOT NULL, + + PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) +) ENGINE = InnoDB; DELIMITER $$ -CREATE PROCEDURE GetStreamMessages +CREATE PROCEDURE AppendStreamMessage ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _MaxCount INT, - IN _MaxAttempts INT, - IN _VisibilityTimeout INT, - IN _RemovalTimeout INT, - IN _EvictionInterval INT, - IN _EvictionBatchSize INT + IN _ServiceId VARCHAR(150), + IN _ProviderId VARCHAR(150), + IN _QueueId VARCHAR(150), + IN _StreamIdBytes LONGBLOB, + IN _StreamNamespaceLength INT, + IN _Payload LONGBLOB, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _MessageId BIGINT; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _VisibleOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _VisibilityTimeout SECOND); -DECLARE _NextEvictOn TIMESTAMP(6) DEFAULT DATE_ADD(_Now, INTERVAL _EvictionInterval SECOND); -DECLARE _EvictOn DATETIME(6); -DECLARE _Count INT; - --- get the next eviction schedule -SET _EvictOn = -( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId -); - --- initialize the control row as necessary -IF _EvictOn IS NULL THEN - - -- race to initialize the control row - INSERT OrleansStreamControl + INSERT INTO OrleansStreamPartition ( ServiceId, ProviderId, QueueId, - EvictOn + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn ) VALUES ( _ServiceId, _ProviderId, _QueueId, - _NextEvictOn + 1, + NULL, + 0, + _Now, + _Now, + _Now ) - ON DUPLICATE KEY - UPDATE - EvictOn = EvictOn; + ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; - -- read the winning update - SET _EvictOn = - ( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - ); - -END IF; - -IF _EvictOn <= _Now THEN - - -- race to update the control row - UPDATE OrleansStreamControl - SET EvictOn = _NextEvictOn - WHERE - ServiceId = _ServiceId + SELECT NextMessageId + INTO _MessageId + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId - AND EvictOn <= _Now; - - -- if we won the race then we also run eviction - IF ROW_COUNT() > 0 THEN - CALL EvictStreamMessages(_ServiceId, _ProviderId, _QueueId, _MaxAttempts, _RemovalTimeout, _EvictionBatchSize); - CALL EvictStreamDeadLetters(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize); - END IF; - -END IF; - -START TRANSACTION; - -/* elect the batch of messages to dequeue and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamMessage -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND Dequeued < _MaxAttempts - AND VisibleOn <= _Now - AND ExpiresOn > _Now -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _MaxCount -FOR UPDATE SKIP LOCKED; - -/* update the message batch */ -UPDATE OrleansStreamMessage AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -SET - M.Dequeued = M.Dequeued + 1, - M.VisibleOn = _VisibleOn, - M.ModifiedOn = _Now; - -/* return the updated batch */ -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload -FROM - OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -ORDER BY - M.MessageId; - -DROP TEMPORARY TABLE _Batch; - -COMMIT; - -END; - -DELIMITER $$ - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'CALL GetStreamMessages(@ServiceId, @ProviderId, @QueueId, @MaxCount, @MaxAttempts, @VisibilityTimeout, @RemovalTimeout, @EvictionInterval, @EvictionBatchSize)'; - -DELIMITER $$ - -CREATE PROCEDURE ConfirmStreamMessages -( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _Items LONGTEXT -) -BEGIN - -DECLARE _Delimiter1 NVARCHAR(1) DEFAULT '|'; -DECLARE _Delimiter2 NVARCHAR(1) DEFAULT ':'; -DECLARE _Value LONGTEXT; -DECLARE _MessageId BIGINT; -DECLARE _Dequeued INT; + FOR UPDATE; -SET _Items = CONCAT(_Items, _Delimiter1); - -/* parse the message identifiers to be deleted */ -DROP TEMPORARY TABLE IF EXISTS _ItemsTable; -CREATE TEMPORARY TABLE _ItemsTable -( - ServiceId NVARCHAR(150) NOT NULL, - ProviderId NVARCHAR(150) NOT NULL, - QueueId NVARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); - -WHILE LOCATE(_Delimiter1, _Items) > 0 DO - - SET _Value = SUBSTRING_INDEX(_Items, _Delimiter1, 1); - SET _MessageId = CAST(SUBSTRING_INDEX(_Value, _Delimiter2, 1) AS UNSIGNED); - SET _Dequeued = CAST(SUBSTRING_INDEX(_Value, _Delimiter2, -1) AS SIGNED); + UPDATE OrleansStreamPartition + SET + NextMessageId = _MessageId + 1, + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; - INSERT INTO _ItemsTable + INSERT INTO OrleansStreamMessage ( ServiceId, ProviderId, QueueId, MessageId, - Dequeued + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload ) VALUES ( @@ -391,353 +168,364 @@ WHILE LOCATE(_Delimiter1, _Items) > 0 DO _ProviderId, _QueueId, _MessageId, - _Dequeued + _StreamIdBytes, + _StreamNamespaceLength, + _Now, + _Payload ); - SET _Items = SUBSTRING(_Items, LOCATE(_Delimiter1, _Items) + 1); - -END WHILE; - -START TRANSACTION; - -/* elect the batch of messages to confirm and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId -FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON M.ServiceId = I.ServiceId - AND M.ProviderId = I.ProviderId - AND M.QueueId = I.QueueId - AND M.MessageId = I.MessageId - AND M.Dequeued = ABS(I.Dequeued) -ORDER BY - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId -FOR UPDATE; - -IF EXISTS (SELECT 1 FROM _ItemsTable WHERE Dequeued < 0) THEN - /* negative dequeue receipts release messages for immediate redelivery */ - UPDATE OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - SET - M.VisibleOn = UTC_TIMESTAMP(6), - M.ModifiedOn = UTC_TIMESTAMP(6); -ELSE - /* delete the elected batch */ - DELETE M - FROM OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; -END IF; - -/* return the ack */ -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - _Batch; - -DROP TEMPORARY TABLE _Batch; -DROP TEMPORARY TABLE _ItemsTable; - -COMMIT; -END; + IF _ManageTransaction THEN + COMMIT; + END IF; -DELIMITER $$ + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _MessageId AS MessageId; +END$$ -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'CALL ConfirmStreamMessages(@ServiceId, @ProviderId, @QueueId, @Items)'; +DELIMITER ; DELIMITER $$ -CREATE PROCEDURE FailStreamMessage +CREATE PROCEDURE AcquireStreamPartition ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _MessageId BIGINT, - IN _MaxAttempts INT, - IN _RemovalTimeout INT + IN _ServiceId VARCHAR(150), + IN _ProviderId VARCHAR(150), + IN _QueueId VARCHAR(150), + IN _StartFromNow BOOLEAN, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _NextMessageId BIGINT; + DECLARE _Checkpoint BIGINT; + DECLARE _OwnerEpoch BIGINT; + DECLARE _EarliestMessageId BIGINT; + DECLARE _TailMessageId BIGINT; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _RemoveOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _RemovalTimeout SECOND); - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = _Now, - ModifiedOn = _Now -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - AND Dequeued < _MaxAttempts; - -IF ROW_COUNT() = 0 THEN - - START TRANSACTION; - - /* otherwise attempt to move the message to dead letters */ - CREATE TEMPORARY TABLE Deleted AS - SELECT - * - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId; - - DELETE FROM OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId; - - INSERT INTO OrleansStreamDeadLetter + INSERT INTO OrleansStreamPartition ( ServiceId, ProviderId, QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload + ModifiedOn ) - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now AS DeadOn, - _RemoveOn AS RemoveOn, - Payload - FROM - Deleted; + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + 1, + NULL, + 0, + _Now, + _Now, + _Now + ) + ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; - COMMIT; + SELECT NextMessageId, Checkpoint + INTO _NextMessageId, _Checkpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; -END IF; + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; -END; + IF _Checkpoint IS NULL THEN + SET _Checkpoint = CASE + WHEN _StartFromNow THEN _NextMessageId - 1 + ELSE COALESCE(_EarliestMessageId - 1, _NextMessageId - 1) + END; + END IF; -DELIMITER $$ + UPDATE OrleansStreamPartition + SET + Checkpoint = _Checkpoint, + OwnerEpoch = OwnerEpoch + 1, + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'FailStreamMessageKey', - 'CALL FailStreamMessage(@ServiceId, @ProviderId, @QueueId, @MessageId, @MaxAttempts, @RemovalTimeout)' + SELECT OwnerEpoch + INTO _OwnerEpoch + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _OwnerEpoch AS OwnerEpoch, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; +END$$ + +DELIMITER ; DELIMITER $$ -CREATE PROCEDURE EvictStreamMessages +CREATE PROCEDURE AdvanceStreamCheckpoint ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _BatchSize INT, - IN _MaxAttempts INT, - IN _RemovalTimeout INT + IN _ServiceId VARCHAR(150), + IN _ProviderId VARCHAR(150), + IN _QueueId VARCHAR(150), + IN _OwnerEpoch BIGINT, + IN _Checkpoint BIGINT, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _CurrentOwnerEpoch BIGINT; + DECLARE _CurrentCheckpoint BIGINT; + DECLARE _Updated BOOLEAN DEFAULT FALSE; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(); -DECLARE _RemoveOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _RemovalTimeout SECOND); - -START TRANSACTION; - -/* elect the batch of messages to move and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamMessage -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND - ( - -- a message is no longer dequeueable if the last attempt timed out - (Dequeued >= _MaxAttempts AND VisibleOn <= _Now) - OR - -- a message is no longer dequeueable if it has expired regardless - (ExpiresOn <= _Now) - ) -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _BatchSize -FOR UPDATE SKIP LOCKED; - -/* copy the messages to dead letters */ -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - _Now, - _RemoveOn, - M.Payload -FROM - OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -/* delete elected messages from the source now */ -DELETE M -FROM OrleansStreamMessage AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -DROP TEMPORARY TABLE _Batch; - -COMMIT; - -END; + UPDATE OrleansStreamPartition + SET + Checkpoint = _Checkpoint, + ModifiedOn = UTC_TIMESTAMP(6) + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND OwnerEpoch = _OwnerEpoch + AND (Checkpoint IS NULL OR Checkpoint < _Checkpoint) + AND _Checkpoint < NextMessageId; -DELIMITER $$ + SET _Updated = ROW_COUNT() = 1; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'CALL EvictStreamMessages(@ServiceId, @ProviderId, @QueueId, @BatchSize, @MaxAttempts, @RemovalTimeout)' -; + SELECT OwnerEpoch, Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _CurrentOwnerEpoch AS OwnerEpoch, + _CurrentCheckpoint AS Checkpoint, + _Updated AS Updated + FROM DUAL + WHERE _CurrentOwnerEpoch IS NOT NULL; +END$$ + +DELIMITER ; DELIMITER $$ -CREATE PROCEDURE EvictStreamDeadLetters +CREATE PROCEDURE CleanupStreamMessages ( - _ServiceId NVARCHAR(150), - _ProviderId NVARCHAR(150), - _QueueId NVARCHAR(150), - _BatchSize INT + IN _ServiceId VARCHAR(150), + IN _ProviderId VARCHAR(150), + IN _QueueId VARCHAR(150), + IN _RetentionPeriodSeconds INT, + IN _MaximumRetentionPeriodSeconds INT, + IN _CleanupIntervalSeconds INT, + IN _CleanupBatchSize INT, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _Checkpoint BIGINT; + DECLARE _CleanupOn DATETIME(6); + DECLARE _DeletedCount INT DEFAULT 0; + DECLARE _DeletedThroughMessageId BIGINT; + DECLARE _HardDeletedCount INT DEFAULT 0; + DECLARE _HardDeletedFromMessageId BIGINT; + DECLARE _HardDeletedThroughMessageId BIGINT; + DECLARE _EarliestMessageId BIGINT; + DECLARE _TailMessageId BIGINT; + DECLARE _PartitionExists BOOLEAN DEFAULT FALSE; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(); - -/* elect the batch of messages to remove */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamDeadLetter -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND RemoveOn <= _Now -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _BatchSize -FOR UPDATE SKIP LOCKED; - -/* now delete the locked messages */ -DELETE M -FROM OrleansStreamDeadLetter AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -DROP TEMPORARY TABLE _Batch; - -END; + SELECT TRUE, Checkpoint, CleanupOn + INTO _PartitionExists, _Checkpoint, _CleanupOn + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; -DELIMITER $$ + IF NOT _PartitionExists OR _CleanupOn > _Now THEN + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + FALSE AS Ran, + 0 AS DeletedCount, + NULL AS DeletedThroughMessageId, + 0 AS HardDeletedCount, + NULL AS HardDeletedFromMessageId, + NULL AS HardDeletedThroughMessageId, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; + ELSE + UPDATE OrleansStreamPartition + SET + CleanupOn = DATE_ADD(_Now, INTERVAL _CleanupIntervalSeconds SECOND), + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + DROP TEMPORARY TABLE IF EXISTS OrleansStreamCleanupBatch; + CREATE TEMPORARY TABLE OrleansStreamCleanupBatch + ( + MessageId BIGINT NOT NULL, + PRIMARY KEY (MessageId) + ); + + INSERT INTO OrleansStreamCleanupBatch (MessageId) + SELECT MessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND + ( + ( + _Checkpoint IS NOT NULL + AND MessageId <= _Checkpoint + AND CreatedOn < DATE_SUB(_Now, INTERVAL _RetentionPeriodSeconds SECOND) + ) + OR + ( + _MaximumRetentionPeriodSeconds IS NOT NULL + AND CreatedOn < DATE_SUB(_Now, INTERVAL _MaximumRetentionPeriodSeconds SECOND) + ) + ) + ORDER BY MessageId + LIMIT _CleanupBatchSize + FOR UPDATE SKIP LOCKED; + + SELECT + COUNT(*), + MAX(MessageId), + COALESCE(SUM(_Checkpoint IS NULL OR MessageId > _Checkpoint), 0), + MIN(CASE WHEN _Checkpoint IS NULL OR MessageId > _Checkpoint THEN MessageId END), + MAX(CASE WHEN _Checkpoint IS NULL OR MessageId > _Checkpoint THEN MessageId END) + INTO + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId + FROM OrleansStreamCleanupBatch; + + DELETE M + FROM OrleansStreamMessage AS M + INNER JOIN OrleansStreamCleanupBatch AS B + ON B.MessageId = M.MessageId + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + DROP TEMPORARY TABLE OrleansStreamCleanupBatch; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + TRUE AS Ran, + _DeletedCount AS DeletedCount, + _DeletedThroughMessageId AS DeletedThroughMessageId, + _HardDeletedCount AS HardDeletedCount, + _HardDeletedFromMessageId AS HardDeletedFromMessageId, + _HardDeletedThroughMessageId AS HardDeletedThroughMessageId, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; + END IF; +END$$ -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'CALL EvictStreamDeadLetters(@ServiceId, @ProviderId, @QueueId, @BatchSize)' -; +DELIMITER ; -DELIMITER $$ \ No newline at end of file +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'CALL AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload, TRUE)'), + ('AcquireStreamPartitionKey', 'CALL AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow, TRUE)'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), + ('AdvanceStreamCheckpointKey', 'CALL AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint, TRUE)'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint'), + ('CleanupStreamMessagesKey', 'CALL CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize, TRUE)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 817742dab8f..576bb868eb1 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -1,707 +1,475 @@ -CREATE SEQUENCE OrleansStreamMessageSequence -AS BIGINT -START WITH 1 -INCREMENT BY 1 -NO MAXVALUE -NO CYCLE; +/* +ADO.NET streaming schema version 2. -CREATE TABLE OrleansStreamMessage -( - ServiceId VARCHAR(150) NOT NULL, - ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - Payload BYTEA NOT NULL, +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. +*/ - CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY - ( - ServiceId, - ProviderId, - QueueId, - MessageId - ) -); +DO $$ +BEGIN + IF to_regclass('orleansstreampartition') IS NOT NULL + OR to_regclass('orleansstreammessage') IS NOT NULL + OR to_regclass('orleansstreamdeadletter') IS NOT NULL + OR to_regclass('orleansstreamcontrol') IS NOT NULL + OR to_regclass('orleansstreammessagesequence') IS NOT NULL + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) + THEN + RAISE EXCEPTION 'Incompatible alpha ADO.NET streaming schema. Drop old streaming tables, sequence, routines, and OrleansQuery rows before applying version 2; no in-place migration is supported.'; + END IF; +END; +$$; -CREATE TABLE OrleansStreamDeadLetter +CREATE TABLE OrleansStreamPartition ( - ServiceId VARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) NOT NULL, ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - DeadOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - Payload BYTEA, - - CONSTRAINT PK_OrleansStreamDeadLetter PRIMARY KEY + QueueId VARCHAR(150) NOT NULL, + NextMessageId BIGINT NOT NULL, + Checkpoint BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + + CONSTRAINT PK_OrleansStreamPartition PRIMARY KEY ( ServiceId, ProviderId, - QueueId, - MessageId + QueueId ) ); -CREATE TABLE OrleansStreamControl +CREATE TABLE OrleansStreamMessage ( - ServiceId VARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) NOT NULL, ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - EvictOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - - CONSTRAINT PK_OrleansStreamControl PRIMARY KEY + QueueId VARCHAR(150) NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes BYTEA NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + Payload BYTEA NOT NULL, + + CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY ( ServiceId, ProviderId, - QueueId + QueueId, + MessageId ) ); -CREATE OR REPLACE FUNCTION QueueStreamMessage -( - _ServiceId VARCHAR(150), - _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _Payload BYTEA, - _ExpiryTimeout INT -) -RETURNS TABLE -( - ServiceId VARCHAR(150), - ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT -) -LANGUAGE plpgsql -AS $$ -#VARIABLE_CONFLICT USE_COLUMN -DECLARE - _MessageId BIGINT := nextval('OrleansStreamMessageSequence'); - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _ExpiryTimeout; -BEGIN - -RETURN QUERY -INSERT INTO OrleansStreamMessage -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -VALUES -( - _ServiceId, - _ProviderId, - _QueueId, - _MessageId, - 0, - _Now, - _ExpiresOn, - _Now, - _Now, - _Payload -) -RETURNING - ServiceId, - ProviderId, - QueueId, - MessageId; - -END; -$$; - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'SELECT * FROM QueueStreamMessage(@ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout)' -; - -CREATE OR REPLACE FUNCTION GetStreamMessages +CREATE OR REPLACE FUNCTION AppendStreamMessage ( - _ServiceId VARCHAR(150), + _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _MaxCount INT, - _MaxAttempts INT, - _VisibilityTimeout INT, - _RemovalTimeout INT, - _EvictionInterval INT, - _EvictionBatchSize INT + _QueueId VARCHAR(150), + _StreamIdBytes BYTEA, + _StreamNamespaceLength INT, + _Payload BYTEA ) RETURNS TABLE ( - ServiceId VARCHAR(150), + ServiceId VARCHAR(150), ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT, - Dequeued INT, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE, - Payload BYTEA + QueueId VARCHAR(150), + MessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _VisibilityTimeout; - _EvictOn TIMESTAMP(6) WITHOUT TIME ZONE; - _NextEvictOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _EvictionInterval; + _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _MessageId BIGINT; BEGIN - -/* get the next eviction schedule */ -SELECT EvictOn -INTO _EvictOn -FROM OrleansStreamControl -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId; - -/* initialize the control row if necessary */ -IF _EvictOn IS NULL THEN - - /* initialize with a past date so eviction runs immediately */ - INSERT INTO OrleansStreamControl + INSERT INTO OrleansStreamPartition ( ServiceId, ProviderId, QueueId, - EvictOn + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn ) VALUES ( _ServiceId, _ProviderId, _QueueId, - _Now - INTERVAL '1 SECOND' + 1, + NULL, + 0, + _Now, + _Now, + _Now ) - ON CONFLICT (ServiceId, ProviderId, QueueId) - DO NOTHING; - - /* get the next eviction schedule again */ - SELECT EvictOn - INTO _EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId; - -END IF; - -/* evict messages if necessary */ -IF _EvictOn <= _Now THEN - - /* race to set the next schedule */ - UPDATE OrleansStreamControl - SET EvictOn = _NextEvictOn - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND EvictOn <= _Now; - - /* if we won the race then we also run the due eviction */ - IF (FOUND) THEN - CALL EvictStreamMessages(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize, _MaxAttempts, _RemovalTimeout); - CALL EvictStreamDeadLetters(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize); - END IF; + ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; -END IF; - -RETURN QUERY -WITH Batch AS -( - /* elect the next batch of visible messages */ - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND Dequeued < _MaxAttempts - AND VisibleOn <= _Now - AND ExpiresOn > _Now + UPDATE OrleansStreamPartition AS P + SET + NextMessageId = P.NextMessageId + 1, + ModifiedOn = _Now + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + RETURNING P.NextMessageId - 1 INTO _MessageId; - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _MaxCount -) -UPDATE OrleansStreamMessage AS M -SET - Dequeued = Dequeued + 1, - VisibleOn = _VisibleOn, - ModifiedOn = _Now -FROM - Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload; + INSERT INTO OrleansStreamMessage + ( + ServiceId, + ProviderId, + QueueId, + MessageId, + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload + ) + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + _MessageId, + _StreamIdBytes, + _StreamNamespaceLength, + _Now, + _Payload + ); + RETURN QUERY + SELECT _ServiceId, _ProviderId, _QueueId, _MessageId; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'SELECT * FROM GetStreamMessages(@ServiceId, @ProviderId, @QueueId, @MaxCount, @MaxAttempts, @VisibilityTimeout, @RemovalTimeout, @EvictionInterval, @EvictionBatchSize)' -; - -CREATE OR REPLACE FUNCTION ConfirmStreamMessages +CREATE OR REPLACE FUNCTION AcquireStreamPartition ( - _ServiceId VARCHAR(150), + _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _Items TEXT + _QueueId VARCHAR(150), + _StartFromNow BOOLEAN ) RETURNS TABLE ( - ServiceId VARCHAR(150), + ServiceId VARCHAR(150), ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT + QueueId VARCHAR(150), + OwnerEpoch BIGINT, + Checkpoint BIGINT, + EarliestMessageId BIGINT, + TailMessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Count INT; _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _NextMessageId BIGINT; + _Checkpoint BIGINT; + _OwnerEpoch BIGINT; + _EarliestMessageId BIGINT; + _TailMessageId BIGINT; BEGIN - -CREATE TEMP TABLE _ItemsTable -( - MessageId BIGINT PRIMARY KEY NOT NULL, - Dequeued INT NOT NULL -) ON COMMIT DROP; - -INSERT INTO _ItemsTable -( - MessageId, - Dequeued -) -SELECT - CAST(split_part(Value, ':', 1) AS BIGINT) AS MessageId, - CAST(split_part(Value, ':', 2) AS INT) AS Dequeued -FROM - UNNEST(string_to_array(_Items, '|')) AS Value; - -/* negative dequeue receipts release messages for immediate redelivery */ -IF EXISTS (SELECT 1 FROM _ItemsTable WHERE Dequeued < 0) THEN - RETURN QUERY - WITH Batch AS + INSERT INTO OrleansStreamPartition ( - SELECT - M.* - FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON I.MessageId = M.MessageId - AND -I.Dequeued = M.Dequeued - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE + ServiceId, + ProviderId, + QueueId, + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + 1, + NULL, + 0, + _Now, + _Now, + _Now ) - UPDATE OrleansStreamMessage AS M + ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; + + SELECT P.NextMessageId, P.Checkpoint + INTO _NextMessageId, _Checkpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + FOR UPDATE; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + IF _Checkpoint IS NULL THEN + _Checkpoint := CASE + WHEN _StartFromNow THEN _NextMessageId - 1 + ELSE COALESCE(_EarliestMessageId - 1, _NextMessageId - 1) + END; + END IF; + + UPDATE OrleansStreamPartition AS P SET - VisibleOn = _Now, + Checkpoint = _Checkpoint, + OwnerEpoch = P.OwnerEpoch + 1, ModifiedOn = _Now - FROM - Batch AS B - WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId; - RETURN; -END IF; - -RETURN QUERY -WITH Batch AS -( - SELECT - M.* - FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON I.MessageId = M.MessageId - AND I.Dequeued = M.Dequeued - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - - /* the criteria below helps prevent deadlocks */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE -) -DELETE FROM OrleansStreamMessage AS M -USING Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId; + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + RETURNING P.OwnerEpoch INTO _OwnerEpoch; + RETURN QUERY + SELECT + _ServiceId, + _ProviderId, + _QueueId, + _OwnerEpoch, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'SELECT * FROM ConfirmStreamMessages(@ServiceId, @ProviderId, @QueueId, @Items)' -; - -CREATE OR REPLACE PROCEDURE FailStreamMessage +CREATE OR REPLACE FUNCTION AdvanceStreamCheckpoint ( _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), _QueueId VARCHAR(150), - _MessageId BIGINT, - _MaxAttempts INT, - _RemovalTimeout INT -) -LANGUAGE plpgsql -AS $$ -#VARIABLE_CONFLICT USE_COLUMN -DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _RemovalTimeout; -BEGIN - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = _Now, - ModifiedOn = _Now -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - AND Dequeued < _MaxAttempts; - -IF FOUND THEN - RETURN; -END IF; - -/* otherwise attempt to move the message to dead letters */ -WITH Deleted AS -( - DELETE FROM OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - RETURNING - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now AS DeadOn, - _RemoveOn AS RemoveOn, - Payload -FROM - Deleted; - -END; -$$; - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText + _OwnerEpoch BIGINT, + _Checkpoint BIGINT ) -SELECT - 'FailStreamMessageKey', - 'CALL FailStreamMessage(@ServiceId, @ProviderId, @QueueId, @MessageId, @MaxAttempts, @RemovalTimeout)' -; - -CREATE OR REPLACE PROCEDURE EvictStreamMessages +RETURNS TABLE ( - _ServiceId VARCHAR(150), - _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _BatchSize INT, - _MaxAttempts INT, - _RemovalTimeout INT + ServiceId VARCHAR(150), + ProviderId VARCHAR(150), + QueueId VARCHAR(150), + OwnerEpoch BIGINT, + Checkpoint BIGINT, + Updated BOOLEAN ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 second' * _RemovalTimeout; + _CurrentOwnerEpoch BIGINT; + _CurrentCheckpoint BIGINT; + _Updated BOOLEAN := FALSE; BEGIN - -/* elect the next batch of messages to evict */ -WITH Batch AS -( - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - - -- the message was given the opportunity to complete - AND VisibleOn <= _Now - AND - ( - -- the message was dequeued too many times - Dequeued >= _MaxAttempts - OR - -- the message expired - ExpiresOn <= _Now - ) - - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _BatchSize -), - -/* delete the messages locked in the batch */ -Deleted AS -( - DELETE FROM OrleansStreamMessage AS M - USING Batch AS B - WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload -) - -/* copy the deleted messages to the dead-letter table */ -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now, - _RemoveOn, - Payload -FROM - Deleted AS D; - + UPDATE OrleansStreamPartition AS P + SET + Checkpoint = _Checkpoint, + ModifiedOn = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + AND P.OwnerEpoch = _OwnerEpoch + AND (P.Checkpoint IS NULL OR P.Checkpoint < _Checkpoint) + AND _Checkpoint < P.NextMessageId + RETURNING P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint; + + IF FOUND THEN + _Updated := TRUE; + ELSE + SELECT P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId; + END IF; + + IF _CurrentOwnerEpoch IS NOT NULL THEN + RETURN QUERY + SELECT + _ServiceId, + _ProviderId, + _QueueId, + _CurrentOwnerEpoch, + _CurrentCheckpoint, + _Updated; + END IF; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'CALL EvictStreamMessages(@ServiceId, @ProviderId, @QueueId, @BatchSize, @MaxAttempts, @RemovalTimeout)' -; - -CREATE OR REPLACE PROCEDURE EvictStreamDeadLetters +CREATE OR REPLACE FUNCTION CleanupStreamMessages ( _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), _QueueId VARCHAR(150), - _BatchSize INT + _RetentionPeriodSeconds INT, + _MaximumRetentionPeriodSeconds INT, + _CleanupIntervalSeconds INT, + _CleanupBatchSize INT +) +RETURNS TABLE +( + Ran BOOLEAN, + DeletedCount INT, + DeletedThroughMessageId BIGINT, + HardDeletedCount INT, + HardDeletedFromMessageId BIGINT, + HardDeletedThroughMessageId BIGINT, + Checkpoint BIGINT, + EarliestMessageId BIGINT, + TailMessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _Checkpoint BIGINT; + _DeletedCount INT := 0; + _DeletedThroughMessageId BIGINT; + _HardDeletedCount INT := 0; + _HardDeletedFromMessageId BIGINT; + _HardDeletedThroughMessageId BIGINT; + _EarliestMessageId BIGINT; + _TailMessageId BIGINT; BEGIN - -/* elect the next batch of dead letters to evict */ -WITH Batch AS -( + UPDATE OrleansStreamPartition AS P + SET + CleanupOn = _Now + make_interval(secs => _CleanupIntervalSeconds), + ModifiedOn = _Now + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + AND P.CleanupOn <= _Now + RETURNING P.Checkpoint INTO _Checkpoint; + + IF NOT FOUND THEN + SELECT P.Checkpoint + INTO _Checkpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + RETURN QUERY + SELECT + FALSE, + 0, + NULL::BIGINT, + 0, + NULL::BIGINT, + NULL::BIGINT, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; + RETURN; + END IF; + + WITH Candidate AS + ( + SELECT M.MessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND + ( + ( + _Checkpoint IS NOT NULL + AND M.MessageId <= _Checkpoint + AND M.CreatedOn < _Now - make_interval(secs => _RetentionPeriodSeconds) + ) + OR + ( + _MaximumRetentionPeriodSeconds IS NOT NULL + AND M.CreatedOn < _Now - make_interval(secs => _MaximumRetentionPeriodSeconds) + ) + ) + ORDER BY M.MessageId + FOR UPDATE SKIP LOCKED + LIMIT _CleanupBatchSize + ), + Deleted AS + ( + DELETE FROM OrleansStreamMessage AS M + USING Candidate AS C + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND M.MessageId = C.MessageId + RETURNING M.MessageId + ) SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamDeadLetter - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND RemoveOn <= _Now - - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _BatchSize -) -DELETE FROM OrleansStreamDeadLetter AS M -USING Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; + COUNT(*)::INT, + MAX(D.MessageId), + COUNT(*) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint)::INT, + MIN(D.MessageId) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint), + MAX(D.MessageId) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint) + INTO + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId + FROM Deleted AS D; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + RETURN QUERY + SELECT + TRUE, + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'CALL EvictStreamDeadLetters(@ServiceId, @ProviderId, @QueueId, @BatchSize)' -; \ No newline at end of file +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'SELECT * FROM AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload)'), + ('AcquireStreamPartitionKey', 'SELECT * FROM AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow)'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), + ('AdvanceStreamCheckpointKey', 'SELECT * FROM AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint)'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint'), + ('CleanupStreamMessagesKey', 'SELECT * FROM CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index f8ef68c7066..e7040c22b51 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -48,6 +48,21 @@ var builder = Host.CreateApplicationBuilder(args) await builder.RunAsync(); ``` +The provider stores each queue as an immutable partition log. Configure retained-log behavior with: + +- `StartFromNow`: initialize a new checkpoint at the retained tail instead of before the earliest retained message. +- `MaxMessagesPerRead`: bound each ordered storage read. +- `CheckpointPersistInterval`: throttle durable checkpoint updates. +- `RetentionPeriod`: retain checkpointed messages for at least this period (one day by default). +- `MaximumRetentionPeriod`: optionally delete older messages even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed delivery gap. +- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. + +## Alpha schema upgrade + +The current streaming scripts use schema version 2 and are intentionally incompatible with the former queue, visibility-timeout, confirmation, and dead-letter schema. The provider fails during initialization when it detects old or mixed streaming query keys. + +There is no in-place migration for this alpha package. Stop producers and consumers, drop `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, `OrleansStreamMessageSequence`, the old streaming routines, and their `OrleansQuery` rows. Drop `OrleansStreamPartition` too after a partial version 2 installation. Then apply the current SQL Server, PostgreSQL, or MySQL streaming script. Existing alpha rows are not read or silently converted, so export payloads first if they must be retained. + ## Example - Using ADO.NET Streams in a Grain ```csharp // Producer grain diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index af4461624a6..aac7578a96f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -1,792 +1,607 @@ /* -Orleans Stream Message Sequence. -This sequence reduces contention on generation of [MessageId] values vs an identity column. -The CACHE parameter can be increased to further reduce contention. -*/ -CREATE SEQUENCE OrleansStreamMessageSequence -AS BIGINT -START WITH 1 -INCREMENT BY 1 -NO MAXVALUE -NO CYCLE -CACHE 1000; -GO - -/* -Orleans Streaming Message Queue. - -This table stores queued messages awaiting processing by Orleans. - -The demands for this table are as follows: - -1. The table will see inserts only at the tail, as new rows are added. -2. The table will be polled with high frequency to reserve the first batch of rows that matches a well-known criteria ("visible" and "not expired" and "under max attempts"). -3. The table will see rows being removed at the head as messages are confirmed. -4. The table will see rows being removed at the head as expired messages are moved to dead letters. -5. Due to the above queries touching more than one row at a time, there is a possibility of deadlocks. -6. A few faulted or poisoned messages can linger for some time at the head before being moved to dead letters. -7. The table will occasionaly become empty or at least sparse as the cluster succeeds to catch up to all messages. - -While [1-6] all cause page fragmentation over time, [7] self resolves this degradation by allowing sql server to eventually remove all pages. -Therefore the design attempts to optimise for [2] while assuming the resulting degradation eventually resolves itself. - -The design also attempts to minimize the possibility of deadlocks at the expense of higher locking contention. -This happens by forcing all queries to touch data in the exact same order of the clustered index. -This induces ordered resource lock acquisition while avoiding the cost of ordering itself. +ADO.NET streaming schema version 2. +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. */ -CREATE TABLE OrleansStreamMessage -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME2(7) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload VARBINARY(MAX) NOT NULL, - - /* This Clustered PK supports the various ordered scanning queries. */ - CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC, - MessageId ASC - ) -); +IF OBJECT_ID(N'OrleansStreamPartition', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamMessage', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamDeadLetter', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamControl', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamMessageSequence', N'SO') IS NOT NULL + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) +BEGIN + THROW 51001, 'Incompatible alpha ADO.NET streaming schema. Drop old streaming tables, sequence, routines, and OrleansQuery rows before applying version 2; no in-place migration is supported.', 1; +END; GO -/* -Orleans Streaming Dead Letters. - -This table holds events that could not be processed within the allowed number of attempts or that have expired. -*/ -CREATE TABLE OrleansStreamDeadLetter +CREATE TABLE OrleansStreamPartition ( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ + ServiceId NVARCHAR(150) NOT NULL, ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was given up on - troubleshooting only */ - DeadOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event is scheduled to be removed from dead letters */ - RemoveOn DATETIME2(7) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload VARBINARY(MAX) NULL, - - /* This Clustered PK supports the various ordered scanning queries. */ - /* Its main purpose is to help partition the update row locks as to minimize dequeing contention. */ - CONSTRAINT PK_OrleansStreamDeadLetter PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC, - MessageId ASC - ) + QueueId NVARCHAR(150) NOT NULL, + NextMessageId BIGINT NOT NULL, + [Checkpoint] BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn DATETIME2(7) NOT NULL, + CreatedOn DATETIME2(7) NOT NULL, + ModifiedOn DATETIME2(7) NOT NULL, + + CONSTRAINT PK_OrleansStreamPartition PRIMARY KEY CLUSTERED + ( + ServiceId, + ProviderId, + QueueId + ) ); GO -/* -Orleans Streaming Control Table. -This table holds schedule variables to help providers self manage their own work. -*/ -CREATE TABLE OrleansStreamControl +CREATE TABLE OrleansStreamMessage ( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ + ServiceId NVARCHAR(150) NOT NULL, ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider */ - QueueId NVARCHAR(150) NOT NULL, - - /* The next due schedule for messages to be evicted */ - EvictOn DATETIME2(7) NOT NULL, - - /* Each row represents a flat configuration object for an individual queue */ - CONSTRAINT PK_OrleansStreamControl PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC - ) + QueueId NVARCHAR(150) NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes VARBINARY(MAX) NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn DATETIME2(7) NOT NULL, + Payload VARBINARY(MAX) NOT NULL, + + CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY CLUSTERED + ( + ServiceId, + ProviderId, + QueueId, + MessageId + ) ); GO -/* Queues a message to the Orleans Streaming Message Queue */ -CREATE PROCEDURE QueueStreamMessage - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AppendStreamMessage + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @Payload VARBINARY(MAX), - @ExpiryTimeout INT + @QueueId NVARCHAR(150), + @StreamIdBytes VARBINARY(MAX), + @StreamNamespaceLength INT, + @Payload VARBINARY(MAX) AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Allocated TABLE (MessageId BIGINT NOT NULL); -/* -MessageIds must become visible in allocation order within each queue. Otherwise, -concurrent transactions can expose a later MessageId while an earlier insert is -still uncommitted, violating the stream cache's monotonic sequence invariant. -*/ -DECLARE @LockResource NVARCHAR(255) = CONCAT -( - N'OrleansStreamMessage:', - CONVERT - ( - VARCHAR(64), - HASHBYTES + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + IF NOT EXISTS ( - 'SHA2_256', - CONCAT + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + DECLARE @InitializationLockResource NVARCHAR(255) = CONCAT ( - DATALENGTH(@ServiceId), N':', @ServiceId, - DATALENGTH(@ProviderId), N':', @ProviderId, - DATALENGTH(@QueueId), N':', @QueueId + N'OrleansStreamPartition:', + CONVERT + ( + VARCHAR(64), + HASHBYTES + ( + 'SHA2_256', + CONCAT + ( + DATALENGTH(@ServiceId), N':', @ServiceId, + DATALENGTH(@ProviderId), N':', @ProviderId, + DATALENGTH(@QueueId), N':', @QueueId + ) + ), + 2 + ) + ); + DECLARE @InitializationLockResult INT; + + EXECUTE @InitializationLockResult = sys.sp_getapplock + @Resource = @InitializationLockResource, + @LockMode = 'Exclusive', + @LockOwner = 'Transaction'; + + IF @InitializationLockResult < 0 + BEGIN + THROW 51000, 'Failed to acquire the stream partition initialization lock.', 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId ) - ), - 2 - ) -); -DECLARE @LockResult INT; -DECLARE @StartedTransaction BIT = 0; - -BEGIN TRY - IF @@TRANCOUNT = 0 - BEGIN - BEGIN TRANSACTION; - SET @StartedTransaction = 1; - END; - - EXECUTE @LockResult = sys.sp_getapplock - @Resource = @LockResource, - @LockMode = 'Exclusive', - @LockOwner = 'Transaction'; - - IF @LockResult < 0 - BEGIN - THROW 51000, 'Failed to acquire the stream message queue lock.', 1; - END; - - DECLARE @MessageId BIGINT = NEXT VALUE FOR OrleansStreamMessageSequence; - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - DECLARE @ExpiresOn DATETIME2(7) = DATEADD(SECOND, @ExpiryTimeout, @Now); - - INSERT INTO OrleansStreamMessage - ( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload - ) - OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.MessageId - VALUES - ( - @ServiceId, - @ProviderId, - @QueueId, - @MessageId, - 0, - @Now, - @ExpiresOn, - @Now, - @Now, - @Payload - ); - - IF @StartedTransaction = 1 - BEGIN - COMMIT TRANSACTION; - END; -END TRY -BEGIN CATCH - IF @StartedTransaction = 1 AND XACT_STATE() <> 0 - BEGIN - ROLLBACK TRANSACTION; - END; - - THROW; -END CATCH; + BEGIN + INSERT INTO OrleansStreamPartition + ( + ServiceId, + ProviderId, + QueueId, + NextMessageId, + [Checkpoint], + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + @ServiceId, + @ProviderId, + @QueueId, + 1, + NULL, + 0, + @Now, + @Now, + @Now + ); + END; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + NextMessageId = NextMessageId + 1, + ModifiedOn = @Now + OUTPUT Inserted.NextMessageId - 1 INTO @Allocated (MessageId) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; -END -GO + INSERT INTO OrleansStreamMessage + ( + ServiceId, + ProviderId, + QueueId, + MessageId, + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload + ) + SELECT + @ServiceId, + @ProviderId, + @QueueId, + MessageId, + @StreamIdBytes, + @StreamNamespaceLength, + @Now, + @Payload + FROM @Allocated; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'EXECUTE QueueStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @Payload = @Payload, @ExpiryTimeout = @ExpiryTimeout' + SELECT + @ServiceId AS ServiceId, + @ProviderId AS ProviderId, + @QueueId AS QueueId, + MessageId + FROM @Allocated; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; +END; GO -/* Gets message batches from the Orleans Streaming Message Queue */ -/* Also opportunistically performs eviction activities when they are due */ -CREATE PROCEDURE GetStreamMessages - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AcquireStreamPartition + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @MaxCount INT, - @MaxAttempts INT, - @VisibilityTimeout INT, - @RemovalTimeout INT, - @EvictionInterval INT, - @EvictionBatchSize INT + @QueueId NVARCHAR(150), + @StartFromNow BIT AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @VisibleOn DATETIME2(7) = DATEADD(SECOND, @VisibilityTimeout, @Now); - -/* lightweight check to see if an eviction activity is due */ -DECLARE @EvictOn DATETIME2(7) = -( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId -); + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @NextMessageId BIGINT; + DECLARE @Checkpoint BIGINT; + DECLARE @OwnerEpoch BIGINT; + DECLARE @EarliestMessageId BIGINT; + DECLARE @TailMessageId BIGINT; + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + DECLARE @InitializationLockResource NVARCHAR(255) = CONCAT + ( + N'OrleansStreamPartition:', + CONVERT + ( + VARCHAR(64), + HASHBYTES + ( + 'SHA2_256', + CONCAT + ( + DATALENGTH(@ServiceId), N':', @ServiceId, + DATALENGTH(@ProviderId), N':', @ProviderId, + DATALENGTH(@QueueId), N':', @QueueId + ) + ), + 2 + ) + ); + DECLARE @InitializationLockResult INT; + + EXECUTE @InitializationLockResult = sys.sp_getapplock + @Resource = @InitializationLockResource, + @LockMode = 'Exclusive', + @LockOwner = 'Transaction'; + + IF @InitializationLockResult < 0 + BEGIN + THROW 51000, 'Failed to acquire the stream partition initialization lock.', 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + INSERT INTO OrleansStreamPartition + ( + ServiceId, + ProviderId, + QueueId, + NextMessageId, + [Checkpoint], + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + @ServiceId, + @ProviderId, + @QueueId, + 1, + NULL, + 0, + @Now, + @Now, + @Now + ); + END; + END; -/* escalate to a eviction attempt only if an activity is due */ -IF @EvictOn IS NULL OR @EvictOn < @Now -BEGIN + SELECT + @NextMessageId = NextMessageId, + @Checkpoint = [Checkpoint] + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; - /* attempt to win a race to update the schedule */ - /* this will also initialize the table if necessary */ - WITH Candidate AS - ( SELECT - ServiceId = @ServiceId, - ProviderId = @ProviderId, - QueueId = @QueueId, - Now = @Now, - EvictOn = DATEADD(SECOND, @EvictionInterval, @Now) - ) - MERGE OrleansStreamControl WITH (UPDLOCK, HOLDLOCK) AS T - USING Candidate AS S - ON T.ServiceId = S.ServiceId - AND T.ProviderId = S.ProviderId - AND T.QueueId = S.QueueId - WHEN MATCHED AND T.EvictOn < S.Now THEN - UPDATE SET T.EvictOn = S.EvictOn - WHEN NOT MATCHED BY TARGET THEN - INSERT - ( - ServiceId, - ProviderId, - QueueId, - EvictOn - ) - VALUES - ( - ServiceId, - ProviderId, - QueueId, - EvictOn - ); + @EarliestMessageId = MIN(MessageId), + @TailMessageId = MAX(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + IF @Checkpoint IS NULL + BEGIN + SET @Checkpoint = CASE + WHEN @StartFromNow = 1 THEN @NextMessageId - 1 + ELSE COALESCE(@EarliestMessageId - 1, @NextMessageId - 1) + END; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + [Checkpoint] = @Checkpoint, + OwnerEpoch = OwnerEpoch + 1, + ModifiedOn = @Now + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; - /* if the above statement won the race then we also get to run the eviction */ - /* other concurrent queries will continue running as normal until the next due time */ - IF (@@ROWCOUNT > 0) - BEGIN + SELECT @OwnerEpoch = OwnerEpoch + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; - /* evict messages */ - EXECUTE EvictStreamMessages - @ServiceId = @ServiceId, - @ProviderId = @ProviderId, - @QueueId = @QueueId, - @MaxAttempts = @MaxAttempts, - @RemovalTimeout = @RemovalTimeout, - @BatchSize = @EvictionBatchSize - - /* evict dead letters */ - EXECUTE EvictStreamDeadLetters - @ServiceId = @ServiceId, - @ProviderId = @ProviderId, - @QueueId = @QueueId, - @BatchSize = @EvictionBatchSize; - - END; + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + SELECT + @ServiceId AS ServiceId, + @ProviderId AS ProviderId, + @QueueId AS QueueId, + @OwnerEpoch AS OwnerEpoch, + @Checkpoint AS [Checkpoint], + @EarliestMessageId AS EarliestMessageId, + @TailMessageId AS TailMessageId; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; END; - -/* update messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@MaxCount) - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload - FROM - OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND Dequeued < @MaxAttempts - AND VisibleOn <= @Now - AND ExpiresOn > @Now - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -UPDATE Batch -SET - Dequeued += 1, - VisibleOn = @VisibleOn, - ModifiedOn = @Now -OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.MessageId, - Inserted.Dequeued, - Inserted.VisibleOn, - Inserted.ExpiresOn, - Inserted.CreatedOn, - Inserted.ModifiedOn, - Inserted.Payload -FROM - Batch; - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'EXECUTE GetStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @MaxCount = @MaxCount, @MaxAttempts = @MaxAttempts, @VisibilityTimeout = @VisibilityTimeout, @RemovalTimeout = @RemovalTimeout, @EvictionInterval = @EvictionInterval, @EvictionBatchSize = @EvictionBatchSize' GO -/* Confirms delivery of a stream message. */ -CREATE PROCEDURE ConfirmStreamMessages - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AdvanceStreamCheckpoint + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @Items NVARCHAR(MAX) + @QueueId NVARCHAR(150), + @OwnerEpoch BIGINT, + @Checkpoint BIGINT AS BEGIN + SET NOCOUNT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -/* parse the message identifiers to be deleted */ -DECLARE @ItemsTable TABLE -( - MessageId BIGINT PRIMARY KEY NOT NULL, - Dequeued INT NOT NULL -); -WITH Items AS -( - SELECT Value FROM STRING_SPLIT(@Items, '|') -) -INSERT INTO @ItemsTable -( - MessageId, - Dequeued -) -SELECT - CAST(SUBSTRING(Value, 1, CHARINDEX(':', Value, 1) - 1) AS BIGINT) AS MessageId, - CAST(SUBSTRING(Value, CHARINDEX(':', Value, 1) + 1, LEN(Value)) AS INT) AS Dequeued -FROM - Items; - -/* count the number of messages to delete so we can use order by in the next query */ -DECLARE @Count INT = (SELECT COUNT(*) FROM @ItemsTable); - -/* negative dequeue receipts release messages for immediate redelivery */ -IF EXISTS (SELECT 1 FROM @ItemsTable WHERE Dequeued < 0) -BEGIN - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - - WITH Batch AS + DECLARE @Result TABLE ( - SELECT TOP (@Count) - M.* - FROM - OrleansStreamMessage AS M WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND EXISTS - ( - SELECT * - FROM @ItemsTable AS I - WHERE I.MessageId = M.MessageId - AND -I.Dequeued = M.Dequeued - ) - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - ) - UPDATE Batch + ServiceId NVARCHAR(150) NOT NULL, + ProviderId NVARCHAR(150) NOT NULL, + QueueId NVARCHAR(150) NOT NULL, + OwnerEpoch BIGINT NOT NULL, + [Checkpoint] BIGINT NULL, + Updated BIT NOT NULL + ); + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) SET - VisibleOn = @Now, - ModifiedOn = @Now + [Checkpoint] = @Checkpoint, + ModifiedOn = SYSUTCDATETIME() OUTPUT Inserted.ServiceId, Inserted.ProviderId, Inserted.QueueId, - Inserted.MessageId; - - RETURN; -END; - -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -/* skip messages being changed concurrently since their dequeue receipt may no longer match */ -WITH Batch AS -( - SELECT TOP (@Count) - * - FROM - OrleansStreamMessage AS M WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) - WHERE - ServiceId = @ServiceId + Inserted.OwnerEpoch, + Inserted.[Checkpoint], + CAST(1 AS BIT) + INTO @Result + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND EXISTS - ( - SELECT * - FROM @ItemsTable AS I - WHERE I.MessageId = M.MessageId - AND I.Dequeued = M.Dequeued - ) - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -DELETE FROM Batch -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId; - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'EXECUTE ConfirmStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @Items = @Items' -GO - -/* Applies delivery failure rules to the specified message. */ -/* If the message has been dequeued too many times, we move it to the dead letter table. */ -/* If the message has expired, we move to the dead letter table. */ -/* If the message is still eligible for delivery, it is made visible again. */ -CREATE PROCEDURE FailStreamMessage - @ServiceId NVARCHAR(150), - @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @MessageId BIGINT, - @MaxAttempts INT, - @RemovalTimeout INT -AS -BEGIN - -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @RemoveOn DATETIME2(7) = DATEADD(SECOND, @RemovalTimeout, @Now); - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = @Now, - ModifiedOn = @Now -WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND MessageId = @MessageId - AND Dequeued < @MaxAttempts; - -IF @@ROWCOUNT > 0 RETURN; - -/* otherwise attempt to move the message to dead letters */ -DELETE FROM OrleansStreamMessage -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId, - Deleted.Dequeued, - Deleted.VisibleOn, - Deleted.ExpiresOn, - Deleted.CreatedOn, - Deleted.ModifiedOn, - @Now AS DeadOn, - @RemoveOn AS RemoveOn, - Deleted.Payload -INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND MessageId = @MessageId; - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'FailStreamMessageKey', - 'EXECUTE FailStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @MessageId = @MessageId, @MaxAttempts = @MaxAttempts, @RemovalTimeout = @RemovalTimeout' -GO - -/* Moves non-delivered messages from the message table to the dead letter table for human troubleshooting. */ -CREATE PROCEDURE EvictStreamMessages - @ServiceId NVARCHAR(150), - @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @BatchSize INT, - @MaxAttempts INT, - @RemovalTimeout INT -AS -BEGIN - -SET NOCOUNT ON; -SET XACT_ABORT ON; + AND QueueId = @QueueId + AND OwnerEpoch = @OwnerEpoch + AND ([Checkpoint] IS NULL OR [Checkpoint] < @Checkpoint) + AND @Checkpoint < NextMessageId; -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @RemoveOn DATETIME2(7) = DATEADD(SECOND, @RemovalTimeout, @Now); + IF EXISTS (SELECT 1 FROM @Result) + BEGIN + SELECT ServiceId, ProviderId, QueueId, OwnerEpoch, [Checkpoint], Updated + FROM @Result; + RETURN; + END; -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@BatchSize) - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn = @Now, - RemoveOn = @RemoveOn, - Payload - FROM - OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - - -- the message was given the opportunity to complete - AND VisibleOn <= @Now - AND - ( - -- the message was dequeued too many times - Dequeued >= @MaxAttempts - OR - -- the message expired - ExpiresOn <= @Now - ) - ORDER BY + SELECT ServiceId, ProviderId, QueueId, - MessageId -) -DELETE FROM Batch -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId, - Deleted.Dequeued, - Deleted.VisibleOn, - Deleted.ExpiresOn, - Deleted.CreatedOn, - Deleted.ModifiedOn, - Deleted.DeadOn, - Deleted.RemoveOn, - Deleted.Payload -INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -); - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'EXECUTE EvictStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @BatchSize = @BatchSize, @MaxAttempts = @MaxAttempts, @RemovalTimeout = @RemovalTimeout' + OwnerEpoch, + [Checkpoint], + CAST(0 AS BIT) AS Updated + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; +END; GO -/* Removes messages from the dead letters table. */ -CREATE PROCEDURE EvictStreamDeadLetters - @ServiceId NVARCHAR(150), +CREATE PROCEDURE CleanupStreamMessages + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @BatchSize INT + @QueueId NVARCHAR(150), + @RetentionPeriodSeconds INT, + @MaximumRetentionPeriodSeconds INT = NULL, + @CleanupIntervalSeconds INT, + @CleanupBatchSize INT AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@BatchSize) - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamDeadLetter WITH (UPDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND RemoveOn <= @Now - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -DELETE FROM Batch; + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Checkpoint BIGINT; + DECLARE @Deleted TABLE (MessageId BIGINT NOT NULL, HardDeleted BIT NOT NULL); + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + CleanupOn = DATEADD(SECOND, @CleanupIntervalSeconds, @Now), + ModifiedOn = @Now, + @Checkpoint = [Checkpoint] + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND CleanupOn <= @Now; + + IF @@ROWCOUNT = 0 + BEGIN + SELECT @Checkpoint = [Checkpoint] + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + SELECT + CAST(0 AS BIT) AS Ran, + 0 AS DeletedCount, + CAST(NULL AS BIGINT) AS DeletedThroughMessageId, + 0 AS HardDeletedCount, + CAST(NULL AS BIGINT) AS HardDeletedFromMessageId, + CAST(NULL AS BIGINT) AS HardDeletedThroughMessageId, + @Checkpoint AS [Checkpoint], + MIN(MessageId) AS EarliestMessageId, + MAX(MessageId) AS TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + + RETURN; + END; + + ;WITH Candidate AS + ( + SELECT TOP (@CleanupBatchSize) + MessageId + FROM OrleansStreamMessage WITH (UPDLOCK, READPAST, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND + ( + ( + @Checkpoint IS NOT NULL + AND MessageId <= @Checkpoint + AND CreatedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) + ) + OR + ( + @MaximumRetentionPeriodSeconds IS NOT NULL + AND CreatedOn < DATEADD(SECOND, -@MaximumRetentionPeriodSeconds, @Now) + ) + ) + ORDER BY MessageId + ) + DELETE Message + OUTPUT + Deleted.MessageId, + CASE + WHEN @Checkpoint IS NULL OR Deleted.MessageId > @Checkpoint THEN CAST(1 AS BIT) + ELSE CAST(0 AS BIT) + END + INTO @Deleted (MessageId, HardDeleted) + FROM OrleansStreamMessage AS Message + INNER JOIN Candidate + ON Candidate.MessageId = Message.MessageId + WHERE Message.ServiceId = @ServiceId + AND Message.ProviderId = @ProviderId + AND Message.QueueId = @QueueId; -END + SELECT + CAST(1 AS BIT) AS Ran, + COUNT(*) AS DeletedCount, + MAX(MessageId) AS DeletedThroughMessageId, + COALESCE(SUM(CASE WHEN HardDeleted = 1 THEN 1 ELSE 0 END), 0) AS HardDeletedCount, + MIN(CASE WHEN HardDeleted = 1 THEN MessageId END) AS HardDeletedFromMessageId, + MAX(CASE WHEN HardDeleted = 1 THEN MessageId END) AS HardDeletedThroughMessageId, + @Checkpoint AS [Checkpoint], + ( + SELECT MIN(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) AS EarliestMessageId, + ( + SELECT MAX(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) AS TailMessageId + FROM @Deleted; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; +END; GO -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'EXECUTE EvictStreamDeadLetters @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @BatchSize = @BatchSize' +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'EXECUTE AppendStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @StreamIdBytes = @StreamIdBytes, @StreamNamespaceLength = @StreamNamespaceLength, @Payload = @Payload'), + ('AcquireStreamPartitionKey', 'EXECUTE AcquireStreamPartition @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @StartFromNow = @StartFromNow'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId OFFSET 0 ROWS FETCH NEXT @MaxCount ROWS ONLY'), + ('AdvanceStreamCheckpointKey', 'EXECUTE AdvanceStreamCheckpoint @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @OwnerEpoch = @OwnerEpoch, @Checkpoint = @Checkpoint'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.[Checkpoint] AS [Checkpoint], MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.[Checkpoint]'), + ('CleanupStreamMessagesKey', 'EXECUTE CleanupStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @RetentionPeriodSeconds = @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds = @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds = @CleanupIntervalSeconds, @CleanupBatchSize = @CleanupBatchSize'); GO diff --git a/src/AdoNet/Shared/Storage/DbStoredQueries.cs b/src/AdoNet/Shared/Storage/DbStoredQueries.cs index 57fcaa52f1b..ec2d08c80ca 100644 --- a/src/AdoNet/Shared/Storage/DbStoredQueries.cs +++ b/src/AdoNet/Shared/Storage/DbStoredQueries.cs @@ -31,10 +31,39 @@ namespace Orleans.Tests.SqlUtils /// internal class DbStoredQueries { +#if STREAMING_ADONET || TESTER_SQLUTILS + private const string CurrentStreamingSchemaVersion = "2"; + + private static readonly string[] RequiredStreamingQueryKeys = + [ + nameof(StreamSchemaVersionKey), + nameof(AppendStreamMessageKey), + nameof(AcquireStreamPartitionKey), + nameof(ReadStreamMessagesKey), + nameof(AdvanceStreamCheckpointKey), + nameof(GetStreamPartitionBoundsKey), + nameof(CleanupStreamMessagesKey) + ]; + + private static readonly string[] LegacyStreamingQueryKeys = + [ + "QueueStreamMessageKey", + "GetStreamMessagesKey", + "ConfirmStreamMessagesKey", + "FailStreamMessageKey", + "EvictStreamMessagesKey", + "EvictStreamDeadLettersKey" + ]; +#endif + private readonly Dictionary queries; internal DbStoredQueries(Dictionary queries) { +#if STREAMING_ADONET || TESTER_SQLUTILS + ValidateStreamingSchema(queries); +#endif + var fields = typeof(DbStoredQueries).GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) .Select(p => p.Name); var missingQueryKeys = fields.Except(queries.Keys).ToArray(); @@ -46,6 +75,34 @@ internal DbStoredQueries(Dictionary queries) this.queries = queries; } +#if STREAMING_ADONET || TESTER_SQLUTILS + private static void ValidateStreamingSchema(Dictionary queries) + { + var hasLegacyKeys = LegacyStreamingQueryKeys.Any(queries.ContainsKey); + var missingKeys = RequiredStreamingQueryKeys.Where(key => !queries.ContainsKey(key)).ToArray(); + var hasExpectedVersion = queries.TryGetValue(nameof(StreamSchemaVersionKey), out var version) + && string.Equals(version, CurrentStreamingSchemaVersion, StringComparison.Ordinal); + + if (hasExpectedVersion && !hasLegacyKeys && missingKeys.Length == 0) + { + return; + } + + var detectedVersion = version ?? "legacy or missing"; + var missingDescription = missingKeys.Length == 0 ? "none" : string.Join(", ", missingKeys); + var legacyDescription = LegacyStreamingQueryKeys.Where(queries.ContainsKey).ToArray() is { Length: > 0 } legacyKeys + ? string.Join(", ", legacyKeys) + : "none"; + throw new InvalidOperationException( + $"The ADO.NET streaming schema is incompatible. Expected immutable partition-log schema version {CurrentStreamingSchemaVersion}, " + + $"but detected '{detectedVersion}'. Missing query keys: {missingDescription}. Legacy query keys: {legacyDescription}. " + + "This alpha schema has no in-place migration. Drop the legacy OrleansStreamMessage, " + + "OrleansStreamDeadLetter, OrleansStreamControl, and OrleansStreamMessageSequence objects, drop OrleansStreamPartition if it " + + "exists, remove the streaming routines and OrleansQuery rows, and then apply the current SQL Server, PostgreSQL, or MySQL " + + "streaming script. Existing alpha queue rows are not compatible and will not be migrated."); + } +#endif + /// /// The query that's used to get all the stored queries. /// this will probably be the same for all relational dbs. @@ -148,34 +205,39 @@ internal DbStoredQueries(Dictionary queries) #if STREAMING_ADONET || TESTER_SQLUTILS /// - /// A query template to enqueue a message into the stream table. + /// The immutable partition-log schema version marker. /// - internal string QueueStreamMessageKey => queries[nameof(QueueStreamMessageKey)]; + internal string StreamSchemaVersionKey => queries[nameof(StreamSchemaVersionKey)]; /// - /// A query template to dequeue messages from the stream table. + /// A query template to append a message to a stream partition. /// - internal string GetStreamMessagesKey => queries[nameof(GetStreamMessagesKey)]; + internal string AppendStreamMessageKey => queries[nameof(AppendStreamMessageKey)]; /// - /// A query template to confirm message delivery from the stream table. + /// A query template to acquire stream partition ownership. /// - internal string ConfirmStreamMessagesKey => queries[nameof(ConfirmStreamMessagesKey)]; + internal string AcquireStreamPartitionKey => queries[nameof(AcquireStreamPartitionKey)]; /// - /// A query template to evict a single message (by moving it to dead letters). + /// A query template to read ordered messages after an exclusive position. /// - internal string FailStreamMessageKey => queries[nameof(FailStreamMessageKey)]; + internal string ReadStreamMessagesKey => queries[nameof(ReadStreamMessagesKey)]; /// - /// A query template to batch evict messages (by moving them to dead letters). + /// A query template to advance an epoch-fenced checkpoint. /// - internal string EvictStreamMessagesKey => queries[nameof(EvictStreamMessagesKey)]; + internal string AdvanceStreamCheckpointKey => queries[nameof(AdvanceStreamCheckpointKey)]; /// - /// A query template to evict expired dead letters (by deleting them). + /// A query template to read partition checkpoint and retained bounds. /// - internal string EvictStreamDeadLettersKey => queries[nameof(EvictStreamDeadLettersKey)]; + internal string GetStreamPartitionBoundsKey => queries[nameof(GetStreamPartitionBoundsKey)]; + + /// + /// A query template to perform bounded retained-log cleanup. + /// + internal string CleanupStreamMessagesKey => queries[nameof(CleanupStreamMessagesKey)]; #endif @@ -509,44 +571,64 @@ internal long MessageId set => Add(nameof(MessageId), value); } + internal long AfterMessageId + { + set => Add(nameof(AfterMessageId), value); + } + internal byte[] Payload { set => Add(nameof(Payload), value); } - internal int ExpiryTimeout + internal byte[] StreamIdBytes { - set => Add(nameof(ExpiryTimeout), value); + set => Add(nameof(StreamIdBytes), value); } - internal int MaxCount + internal int StreamNamespaceLength { - set => Add(nameof(MaxCount), value); + set => Add(nameof(StreamNamespaceLength), value); } - internal int MaxAttempts + internal bool StartFromNow { - set => Add(nameof(MaxAttempts), value); + set => Add(nameof(StartFromNow), value); + } + + internal long OwnerEpoch + { + set => Add(nameof(OwnerEpoch), value); + } + + internal long Checkpoint + { + set => Add(nameof(Checkpoint), value); + } + + internal int MaxCount + { + set => Add(nameof(MaxCount), value); } - internal int RemovalTimeout + internal int RetentionPeriodSeconds { - set => Add(nameof(RemovalTimeout), value); + set => Add(nameof(RetentionPeriodSeconds), value); } - internal int VisibilityTimeout + internal int? MaximumRetentionPeriodSeconds { - set => Add(nameof(VisibilityTimeout), value); + set => Add(nameof(MaximumRetentionPeriodSeconds), value, DbType.Int32); } - internal int EvictionInterval + internal int CleanupIntervalSeconds { - set => Add(nameof(EvictionInterval), value); + set => Add(nameof(CleanupIntervalSeconds), value); } - internal int EvictionBatchSize + internal int CleanupBatchSize { - set => Add(nameof(EvictionBatchSize), value); + set => Add(nameof(CleanupBatchSize), value); } internal string EventIds diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index f7279761e71..805dace9a94 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -4,7 +4,6 @@ using System.Data.Common; using System.Globalization; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Orleans.Runtime; @@ -453,22 +452,35 @@ private static MembershipTableData ConvertToMembershipTableData(IEnumerable - /// Queues a stream message to the stream message table. + /// Appends an immutable message to a stream partition. /// /// The service identifier. /// The provider identifier. /// The queue identifier. + /// The canonical bytes. + /// The namespace boundary within . /// The serialized event payload. - /// The expiry timeout for this event batch. - /// An acknowledgement that the message was queued. - internal Task QueueStreamMessageAsync(string serviceId, string providerId, string queueId, byte[] payload, int expiryTimeout) + /// An acknowledgement containing the allocated partition-local message identifier. + internal Task AppendStreamMessageAsync( + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); + ArgumentNullException.ThrowIfNull(streamIdBytes); + ArgumentNullException.ThrowIfNull(payload); + if (streamIdBytes.Length == 0 || streamNamespaceLength < 0 || streamNamespaceLength >= streamIdBytes.Length) + { + throw new ArgumentOutOfRangeException(nameof(streamNamespaceLength), "The stream namespace boundary must leave a non-empty stream key."); + } return ReadAsync( - dbStoredQueries.QueueStreamMessageKey, + dbStoredQueries.AppendStreamMessageKey, record => new AdoNetStreamMessageAck( (string)record[nameof(AdoNetStreamMessageAck.ServiceId)], (string)record[nameof(AdoNetStreamMessageAck.ProviderId)], @@ -479,234 +491,203 @@ internal Task QueueStreamMessageAsync(string serviceId, ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Payload = payload, - ExpiryTimeout = expiryTimeout, + StreamIdBytes = streamIdBytes, + StreamNamespaceLength = streamNamespaceLength, + Payload = payload }, result => result.Single()); } /// - /// Gets stream messages from the stream message table. + /// Acquires partition ownership and initializes its checkpoint when necessary. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The maximum count of event batches to get. - /// The maximum attempts to lock an unprocessed event batch. - /// The visibility timeout for the retrieved event batches. - /// The timeout before the message is to be deleted from dead letters. - /// The interval between opportunistic data eviction. - /// The number of messages to evict in each batch. - /// A list of dequeued payloads. - internal Task> GetStreamMessagesAsync(string serviceId, string providerId, string queueId, int maxCount, int maxAttempts, int visibilityTimeout, int removalTimeout, int evictionInterval, int evictionBatchSize) + internal Task AcquireStreamPartitionAsync( + string serviceId, + string providerId, + string queueId, + bool startFromNow) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - return ReadAsync>( - dbStoredQueries.GetStreamMessagesKey, - record => new AdoNetStreamMessage( - (string)record[nameof(AdoNetStreamMessage.ServiceId)], - (string)record[nameof(AdoNetStreamMessage.ProviderId)], - (string)record[nameof(AdoNetStreamMessage.QueueId)], - (long)record[nameof(AdoNetStreamMessage.MessageId)], - (int)record[nameof(AdoNetStreamMessage.Dequeued)], - (DateTime)record[nameof(AdoNetStreamMessage.VisibleOn)], - (DateTime)record[nameof(AdoNetStreamMessage.ExpiresOn)], - (DateTime)record[nameof(AdoNetStreamMessage.CreatedOn)], - (DateTime)record[nameof(AdoNetStreamMessage.ModifiedOn)], - (byte[])record[nameof(AdoNetStreamMessage.Payload)]), + return ReadAsync( + dbStoredQueries.AcquireStreamPartitionKey, + record => new AdoNetStreamPartitionState( + (string)record[nameof(AdoNetStreamPartitionState.ServiceId)], + (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], + (string)record[nameof(AdoNetStreamPartitionState.QueueId)], + (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - MaxCount = maxCount, - MaxAttempts = maxAttempts, - VisibilityTimeout = visibilityTimeout, - RemovalTimeout = removalTimeout, - EvictionInterval = evictionInterval, - EvictionBatchSize = evictionBatchSize + StartFromNow = startFromNow }, - result => - { - var messages = result.ToList(); - messages.Sort(static (left, right) => left.MessageId.CompareTo(right.MessageId)); - return messages; - }); + result => result.Single()); } /// - /// Confirms delivery of messages from the stream message table. + /// Reads retained messages with identifiers strictly greater than . /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The messages to confirm. - /// A list of confirmations. - /// - /// If is empty then an empty confirmation list is returned. - /// - internal Task> ConfirmStreamMessagesAsync(string serviceId, string providerId, string queueId, IList messages) + internal Task> ReadStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + long afterMessageId, + int maxCount) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - ArgumentNullException.ThrowIfNull(messages); + ArgumentOutOfRangeException.ThrowIfNegative(afterMessageId); + ArgumentOutOfRangeException.ThrowIfLessThan(maxCount, 1); - if (messages.Count == 0) - { - return Task.FromResult>([]); - } - - return ReadAsync>( - dbStoredQueries.ConfirmStreamMessagesKey, - record => new AdoNetStreamConfirmationAck( - (string)record[nameof(AdoNetStreamConfirmationAck.ServiceId)], - (string)record[nameof(AdoNetStreamConfirmationAck.ProviderId)], - (string)record[nameof(AdoNetStreamConfirmationAck.QueueId)], - (long)record[nameof(AdoNetStreamConfirmationAck.MessageId)]), + return ReadAsync>( + dbStoredQueries.ReadStreamMessagesKey, + record => new AdoNetStreamMessage( + (string)record[nameof(AdoNetStreamMessage.ServiceId)], + (string)record[nameof(AdoNetStreamMessage.ProviderId)], + (string)record[nameof(AdoNetStreamMessage.QueueId)], + (long)record[nameof(AdoNetStreamMessage.MessageId)], + (byte[])record[nameof(AdoNetStreamMessage.StreamIdBytes)], + (int)record[nameof(AdoNetStreamMessage.StreamNamespaceLength)], + record.GetDateTimeValue(nameof(AdoNetStreamMessage.CreatedOn)), + (byte[])record[nameof(AdoNetStreamMessage.Payload)]), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Items = FormatStreamConfirmations(messages, release: false) + AfterMessageId = afterMessageId, + MaxCount = maxCount }, result => result.ToList()); } /// - /// Makes unconfirmed stream messages immediately available for redelivery. + /// Advances a checkpoint only when the caller owns the current epoch and the value moves forward. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The messages to release. - /// A list of released messages. - /// - /// The dequeue counter acts as a receipt so that a stale receiver cannot release a message - /// which has already been dequeued by a new receiver. - /// - internal Task> ReleaseStreamMessagesAsync(string serviceId, string providerId, string queueId, IList messages) + internal Task AdvanceStreamCheckpointAsync( + string serviceId, + string providerId, + string queueId, + long ownerEpoch, + long checkpoint) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - ArgumentNullException.ThrowIfNull(messages); - - if (messages.Count == 0) - { - return Task.FromResult>([]); - } - - return ReadAsync>( - dbStoredQueries.ConfirmStreamMessagesKey, - record => new AdoNetStreamConfirmationAck( - (string)record[nameof(AdoNetStreamConfirmationAck.ServiceId)], - (string)record[nameof(AdoNetStreamConfirmationAck.ProviderId)], - (string)record[nameof(AdoNetStreamConfirmationAck.QueueId)], - (long)record[nameof(AdoNetStreamConfirmationAck.MessageId)]), + ArgumentOutOfRangeException.ThrowIfLessThan(ownerEpoch, 1); + ArgumentOutOfRangeException.ThrowIfNegative(checkpoint); + + return ReadAsync( + dbStoredQueries.AdvanceStreamCheckpointKey, + record => new AdoNetStreamCheckpointUpdate( + (string)record[nameof(AdoNetStreamCheckpointUpdate.ServiceId)], + (string)record[nameof(AdoNetStreamCheckpointUpdate.ProviderId)], + (string)record[nameof(AdoNetStreamCheckpointUpdate.QueueId)], + (long)record[nameof(AdoNetStreamCheckpointUpdate.OwnerEpoch)], + GetNullableInt64(record, nameof(AdoNetStreamCheckpointUpdate.Checkpoint)), + Convert.ToBoolean(record[nameof(AdoNetStreamCheckpointUpdate.Updated)])), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Items = FormatStreamConfirmations(messages, release: true) + OwnerEpoch = ownerEpoch, + Checkpoint = checkpoint }, - result => result.ToList()); + result => result.SingleOrDefault()); } - // Builds a provider-neutral receipt list in the form "1:2|3:4|5:6". - private static string FormatStreamConfirmations(IList messages, bool release) => - messages.Aggregate( - new StringBuilder(), - (builder, message) => builder - .Append(builder.Length > 0 ? "|" : "") - .Append(message.MessageId) - .Append(':') - .Append(release ? -message.Dequeued : message.Dequeued), - static builder => builder.ToString()); - /// - /// Applies delivery failure logic to a stream message, such as making the message visible again or moving it to dead letters. + /// Reads the current checkpoint, ownership epoch, and retained message bounds. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The message identifier. - /// The maximum number of delivery attempts before moving the message to dead letters. - /// The number of seconds to retain the message in dead letters. - internal Task FailStreamMessageAsync(string serviceId, string providerId, string queueId, long messageId, int maxAttempts, int removalTimeout) + internal Task GetStreamPartitionBoundsAsync( + string serviceId, + string providerId, + string queueId) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - return ExecuteAsync( - dbStoredQueries.FailStreamMessageKey, + return ReadAsync( + dbStoredQueries.GetStreamPartitionBoundsKey, + record => new AdoNetStreamPartitionState( + (string)record[nameof(AdoNetStreamPartitionState.ServiceId)], + (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], + (string)record[nameof(AdoNetStreamPartitionState.QueueId)], + (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, - QueueId = queueId, - MessageId = messageId, - MaxAttempts = maxAttempts, - RemovalTimeout = removalTimeout - }); + QueueId = queueId + }, + result => result.SingleOrDefault()); } /// - /// Moves eligible messages from the stream message table to the dead letter table. + /// Deletes an ordered, bounded batch of retained messages. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The max number of messages to move in this batch. - /// The max number of times a message can be dequeued. - /// The timeout before the message is to be deleted from dead letters. - internal Task EvictStreamMessagesAsync(string serviceId, string providerId, string queueId, int maxCount, int maxAttempts, int removalTimeout) + internal Task CleanupStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + int retentionPeriodSeconds, + int? maximumRetentionPeriodSeconds, + int cleanupIntervalSeconds, + int cleanupBatchSize) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); + ArgumentOutOfRangeException.ThrowIfLessThan(retentionPeriodSeconds, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(cleanupIntervalSeconds, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(cleanupBatchSize, 1); + if (maximumRetentionPeriodSeconds is { } maximum && maximum < retentionPeriodSeconds) + { + throw new ArgumentOutOfRangeException(nameof(maximumRetentionPeriodSeconds), "The maximum retention period must be greater than or equal to the normal retention period."); + } - return ExecuteAsync( - dbStoredQueries.EvictStreamMessagesKey, + return ReadAsync( + dbStoredQueries.CleanupStreamMessagesKey, + record => new AdoNetStreamCleanupResult( + Convert.ToBoolean(record[nameof(AdoNetStreamCleanupResult.Ran)]), + Convert.ToInt32(record[nameof(AdoNetStreamCleanupResult.DeletedCount)]), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId)), + Convert.ToInt32(record[nameof(AdoNetStreamCleanupResult.HardDeletedCount)]), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - MaxCount = maxCount, - MaxAttempts = maxAttempts, - RemovalTimeout = removalTimeout - }); + RetentionPeriodSeconds = retentionPeriodSeconds, + MaximumRetentionPeriodSeconds = maximumRetentionPeriodSeconds, + CleanupIntervalSeconds = cleanupIntervalSeconds, + CleanupBatchSize = cleanupBatchSize + }, + result => result.Single()); } - /// - /// Removes messages from the dead letter after their removal timeout expires. - /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The max number of messages to move in this batch. - internal Task EvictStreamDeadLettersAsync(string serviceId, string providerId, string queueId, int maxCount) + private static long? GetNullableInt64(IDataRecord record, string fieldName) { - ArgumentNullException.ThrowIfNull(serviceId); - ArgumentNullException.ThrowIfNull(providerId); - ArgumentNullException.ThrowIfNull(queueId); - - return ExecuteAsync( - dbStoredQueries.EvictStreamDeadLettersKey, - command => new DbStoredQueries.Columns(command) - { - ServiceId = serviceId, - ProviderId = providerId, - QueueId = queueId, - MaxCount = maxCount - }); + var ordinal = record.GetOrdinal(fieldName); + return record.IsDBNull(ordinal) ? null : Convert.ToInt64(record.GetValue(ordinal)); } #endif diff --git a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs index 632663fc594..1450aa4eef9 100644 --- a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs +++ b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs @@ -10,27 +10,29 @@ namespace Orleans.Configuration { public partial class AdoNetStreamOptions { + public System.TimeSpan CheckpointPersistInterval { get { throw null; } set { } } + + public int CleanupBatchSize { get { throw null; } set { } } + + public System.TimeSpan CleanupInterval { get { throw null; } set { } } + [Redact] public string? ConnectionString { get { throw null; } set { } } [Redact] public System.Data.Common.DbDataSource? DataSource { get { throw null; } set { } } - public System.TimeSpan DeadLetterEvictionTimeout { get { throw null; } set { } } - - public int EvictionBatchSize { get { throw null; } set { } } - - public System.TimeSpan EvictionInterval { get { throw null; } set { } } - - public System.TimeSpan ExpiryTimeout { get { throw null; } set { } } - public System.TimeSpan InitializationTimeout { get { throw null; } set { } } public string Invariant { get { throw null; } set { } } - public int MaxAttempts { get { throw null; } set { } } + public System.TimeSpan? MaximumRetentionPeriod { get { throw null; } set { } } + + public int MaxMessagesPerRead { get { throw null; } set { } } + + public System.TimeSpan RetentionPeriod { get { throw null; } set { } } - public System.TimeSpan VisibilityTimeout { get { throw null; } set { } } + public bool StartFromNow { get { throw null; } set { } } } public partial class AdoNetStreamOptionsValidator : IConfigurationValidator diff --git a/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs deleted file mode 100644 index 0f6de197c9c..00000000000 --- a/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs +++ /dev/null @@ -1,127 +0,0 @@ -using Microsoft.Data.SqlClient; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using Orleans.Streaming.AdoNet; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; - -namespace Benchmarks.AdoNet.Streaming; - -public class SqlServerMessageDequeuingBenchmark() : MessageDequeuingBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") -{ - public override void GlobalSetup() - { - base.GlobalSetup(); - - SqlConnection.ClearAllPools(); - } -} - -/// -/// This benchmark measures the performance of message queueing. -/// -[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class MessageDequeuingBenchmark(string invariant, string database) -{ - private const int OperationsPerInvoke = 1000; - - private readonly Consumer _consumer = new(); - // BenchmarkDotNet invokes GlobalSetup before any of these fields are read. - private IRelationalStorage _storage = default!; - private RelationalOrleansQueries _queries = default!; - private byte[] _payload = []; - private string[] _queueIds = default!; - private AdoNetStreamMessageAck[] _acks = []; - - /// - /// This highlights degradation from queue concurrency. - /// - [Params(1, 4, 8)] - public int QueueCount { get; set; } - - /// - /// This highlights degradation from payload size. - /// - [Params(10000)] - public int PayloadSize { get; set; } - - /// - /// This highlights variation according to batch size. - /// - [Params(1, 16, 32)] - public int BatchSize { get; set; } - - /// - /// This highlights variation from how full the table is. - /// - [Params(0, 0.5, 1)] - public double FullnessRatio { get; set; } - - [GlobalSetup] - public virtual void GlobalSetup() - { - Async().GetAwaiter().GetResult(); - - async Task Async() - { - // create an appropriate size payload - _payload = new byte[PayloadSize]; - Array.Fill(_payload, 0xFF); - - // define the set queues - _queueIds = Enumerable.Range(0, QueueCount).Select(i => $"QueueId-{i}").ToArray(); - - // setup the test database - var testing = await RelationalStorageForTesting.SetupInstance(invariant, database); - if (IsNullOrEmpty(testing.CurrentConnectionString)) - { - throw new InvalidOperationException($"Database '{database}' not initialized"); - } - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); - } - } - - [IterationSetup] - public void IterationSetup() - { - Async().GetAwaiter().GetResult(); - - async Task Async() - { - await _storage.ExecuteAsync("TRUNCATE TABLE OrleansStreamMessage"); - - // generate test data to dequeue - var count = (int)Math.Ceiling(OperationsPerInvoke * QueueCount * BatchSize * FullnessRatio); - _acks = new AdoNetStreamMessageAck[count]; - await Parallel.ForAsync(0, count, async (i, ct) => - { - // generate messages in round robin queue order to help simulate multiple agents - var queueId = _queueIds[i % _queueIds.Length]; - var ack = await _queries.QueueStreamMessageAsync("ServiceId-0", "ProviderId-0", queueId, _payload, 1000); - - _acks[i] = ack; - }); - } - } - - [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] - public async Task GetStreamMessages() - { - var count = OperationsPerInvoke * QueueCount; - - await Parallel.ForAsync(0, count, new ParallelOptions { MaxDegreeOfParallelism = QueueCount }, async (i, ct) => - { - // get a queue id in round robin order to help simulate multiple agents - var queueId = _queueIds[i % _queueIds.Length]; - - // get messages for the queue of the ack - // the queue may or may not have data to dequeue depending on the fullness and batch size parameters - // we dequeue regardless in order to measure overhead - var messages = await _queries.GetStreamMessagesAsync("ServiceId-0", "ProviderId-0", queueId, BatchSize, 1, 1000, 1000, 1000, 1000); - - _consumer.Consume(messages); - }); - } -} diff --git a/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs deleted file mode 100644 index c160f612539..00000000000 --- a/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Microsoft.Data.SqlClient; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; - -namespace Benchmarks.AdoNet.Streaming; - -public class SqlServerMessageQueueingBenchmark() : MessageQueueingBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") -{ - public override void GlobalSetup() - { - base.GlobalSetup(); - - SqlConnection.ClearAllPools(); - } -} - -/// -/// This benchmark measures the performance of message queueing. -/// -[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class MessageQueueingBenchmark(string invariant, string database) -{ - private const int OperationsPerInvoke = 1000; - - private readonly Consumer _consumer = new(); - // BenchmarkDotNet invokes GlobalSetup before any of these fields are read. - private IRelationalStorage _storage = default!; - private RelationalOrleansQueries _queries = default!; - private byte[] _payload = []; - private string[] _queueIds = default!; - - /// - /// This highlights degradation from database locking. - /// - [Params(1, 4, 8)] - public int QueueCount { get; set; } - - /// - /// This highlights degradation from payload size. - /// - [Params(1000, 10000, 100000)] - public int PayloadSize { get; set; } - - /// - /// This highlights degradation from concurrency. - /// - [Params(1, 4, 8)] - public int Concurrency { get; set; } - - [GlobalSetup] - public virtual void GlobalSetup() - { - _payload = new byte[PayloadSize]; - Array.Fill(_payload, 0xFF); - - _queueIds = Enumerable.Range(0, QueueCount).Select(i => $"QueueId-{i}").ToArray(); - - var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); - - if (IsNullOrEmpty(testing.CurrentConnectionString)) - { - throw new InvalidOperationException($"Database '{database}' not initialized"); - } - - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - - _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); - } - - [IterationSetup] - public void IterationSetup() => _storage.ExecuteAsync("TRUNCATE TABLE OrleansStreamMessage").GetAwaiter().GetResult(); - - [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] - public Task QueueStreamMessage() - { - var count = OperationsPerInvoke * Concurrency; - - return Parallel.ForAsync(0, count, new ParallelOptions { MaxDegreeOfParallelism = Concurrency }, async (i, ct) => - { - var queueId = _queueIds[Random.Shared.Next(_queueIds.Length)]; - var ack = await _queries.QueueStreamMessageAsync("ServiceId-0", "ProviderId-0", queueId, _payload, 1000); - - _consumer.Consume(ack); - }); - } -} diff --git a/test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs new file mode 100644 index 00000000000..a3249fe6de0 --- /dev/null +++ b/test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs @@ -0,0 +1,84 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Data.SqlClient; +using Orleans.Tests.SqlUtils; +using UnitTests.General; +using static System.String; + +namespace Benchmarks.AdoNet.Streaming; + +public class SqlServerRetainedLogAppendBenchmark() : RetainedLogAppendBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures immutable log append throughput while varying payload size, concurrency, and partition contention. +/// A partition count of one is the lock-wait baseline for database-side wait telemetry. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class RetainedLogAppendBenchmark(string invariant, string database) +{ + private const int OperationsPerInvoke = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(1_000, 100_000)] + public int PayloadSize { get; set; } + + [Params(1, 8)] + public int Concurrency { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() => _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("Append", "LockWait")] + public Task AppendWithPartitionContention() => + Parallel.ForAsync( + 0, + OperationsPerInvoke, + new ParallelOptions { MaxDegreeOfParallelism = Concurrency }, + async (i, cancellationToken) => + { + var result = await _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % _partitionIds.Length], + _streamIdBytes, + 9, + _payload); + _consumer.Consume(result); + }); +} diff --git a/test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs new file mode 100644 index 00000000000..553fe4655a8 --- /dev/null +++ b/test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs @@ -0,0 +1,257 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Data.SqlClient; +using Orleans.Streaming.AdoNet; +using Orleans.Tests.SqlUtils; +using UnitTests.General; +using static System.String; + +namespace Benchmarks.AdoNet.Streaming; + +public class SqlServerRetainedLogReadCheckpointBenchmark() : RetainedLogReadCheckpointBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures exclusive ordered reads and epoch-fenced checkpoint updates across partitions. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class RetainedLogReadCheckpointBenchmark(string invariant, string database) +{ + private const int OperationsPerInvoke = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + private long[] _ownerEpochs = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(1, 32, 256)] + public int BatchSize { get; set; } + + [Params(1_000)] + public int PayloadSize { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + _ownerEpochs = new long[PartitionCount]; + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() + { + _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + for (var i = 0; i < OperationsPerInvoke * PartitionCount; i++) + { + _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % PartitionCount], + _streamIdBytes, + 9, + _payload).GetAwaiter().GetResult(); + } + + for (var i = 0; i < PartitionCount; i++) + { + _ownerEpochs[i] = _queries.AcquireStreamPartitionAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i], + startFromNow: false).GetAwaiter().GetResult().OwnerEpoch; + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("OrderedRead")] + public async Task ReadExclusiveOrderedBatches() + { + await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => + { + var messages = await _queries.ReadStreamMessagesAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % PartitionCount], + afterMessageId: 0, + BatchSize); + _consumer.Consume(messages); + }); + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("Checkpoint")] + public async Task AdvanceEpochFencedCheckpoints() + { + await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => + { + var partition = i % PartitionCount; + var checkpoint = (i / PartitionCount) + 1L; + var result = await _queries.AdvanceStreamCheckpointAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + _ownerEpochs[partition], + checkpoint); + if (result is not null) + { + _consumer.Consume(result); + } + }); + } +} + +public class SqlServerRetainedLogCleanupBenchmark() : RetainedLogCleanupBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures bounded cleanup sweeps while varying partition count, batch size, and eligible-row density. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class RetainedLogCleanupBenchmark(string invariant, string database) +{ + private const int MessagesPerPartition = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(16, 256)] + public int CleanupBatchSize { get; set; } + + [Params(0.0, 0.5, 1.0)] + public double CleanupImpactRatio { get; set; } + + [Params(1_000)] + public int PayloadSize { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() + { + _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + for (var partition = 0; partition < PartitionCount; partition++) + { + for (var message = 0; message < MessagesPerPartition; message++) + { + _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + _streamIdBytes, + 9, + _payload).GetAwaiter().GetResult(); + } + + var state = _queries.AcquireStreamPartitionAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + startFromNow: false).GetAwaiter().GetResult(); + _queries.AdvanceStreamCheckpointAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + state.OwnerEpoch, + MessagesPerPartition).GetAwaiter().GetResult(); + } + + var eligiblePerPartition = (int)(MessagesPerPartition * CleanupImpactRatio); + _storage.ExecuteAsync( + $""" + WITH Ranked AS + ( + SELECT + ServiceId, + ProviderId, + QueueId, + MessageId, + ROW_NUMBER() OVER (PARTITION BY QueueId ORDER BY MessageId) AS RowNumber + FROM OrleansStreamMessage + ) + UPDATE Message + SET CreatedOn = DATEADD(DAY, -2, SYSUTCDATETIME()) + FROM OrleansStreamMessage AS Message + INNER JOIN Ranked + ON Ranked.ServiceId = Message.ServiceId + AND Ranked.ProviderId = Message.ProviderId + AND Ranked.QueueId = Message.QueueId + AND Ranked.MessageId = Message.MessageId + WHERE Ranked.RowNumber <= {eligiblePerPartition}; + + UPDATE OrleansStreamPartition SET CleanupOn = DATEADD(SECOND, -1, SYSUTCDATETIME()); + """).GetAwaiter().GetResult(); + } + + [Benchmark] + [BenchmarkCategory("Cleanup", "CleanupImpact")] + public async Task CleanupRetainedPartitions() + { + var results = await Task.WhenAll(_partitionIds.Select(partitionId => + _queries.CleanupStreamMessagesAsync( + "ServiceId-0", + "ProviderId-0", + partitionId, + retentionPeriodSeconds: 86_400, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + CleanupBatchSize))); + _consumer.Consume(results); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs index 80c2cdaba35..a788a51604f 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs @@ -205,6 +205,145 @@ public void Streaming_RequiresInvariant() AssertInvariantRequired(new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration); } + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesMaxMessagesPerRead(int maxMessagesPerRead, bool valid) + { + var options = ValidStreamOptions(); + options.MaxMessagesPerRead = maxMessagesPerRead; + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.MaxMessagesPerRead)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCheckpointPersistInterval(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.CheckpointPersistInterval = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CheckpointPersistInterval)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesRetentionPeriod(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + + [Fact] + public void Streaming_AllowsNullMaximumRetentionPeriod() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(30); + options.MaximumRetentionPeriod = null; + + // Should not throw: a null hard ceiling means no hard-retention diagnostics are ever produced. + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + + [Theory] + [InlineData(30, 30, true)] // equal to the normal retention period is allowed + [InlineData(30, 31, true)] // greater than the normal retention period is allowed + [InlineData(30, 29, false)] // a hard ceiling tighter than normal retention is invalid + public void Streaming_ValidatesMaximumRetentionPeriodAgainstRetentionPeriod(int retentionSeconds, int maximumRetentionSeconds, bool valid) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(retentionSeconds); + options.MaximumRetentionPeriod = TimeSpan.FromSeconds(maximumRetentionSeconds); + + if (valid) + { + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + else + { + var exception = Assert.Throws(() => new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration()); + Assert.Contains(nameof(AdoNetStreamOptions.MaximumRetentionPeriod), exception.Message, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCleanupInterval(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.CleanupInterval = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CleanupInterval)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCleanupBatchSize(int cleanupBatchSize, bool valid) + { + var options = ValidStreamOptions(); + options.CleanupBatchSize = cleanupBatchSize; + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CleanupBatchSize)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesInitializationTimeout(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.InitializationTimeout = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.InitializationTimeout)); + } + + [Fact] + public void Streaming_Options_HaveRetentionSafeDefaults() + { + var options = new AdoNetStreamOptions(); + + Assert.False(options.StartFromNow); + Assert.Equal(TimeSpan.FromDays(1), options.RetentionPeriod); + Assert.Null(options.MaximumRetentionPeriod); + } + + private static AdoNetStreamOptions ValidStreamOptions() => new() + { + Invariant = AdoNetInvariants.InvariantNameSqlLite, + ConnectionString = "Data Source=:memory:", + MaxMessagesPerRead = 100, + CheckpointPersistInterval = TimeSpan.FromSeconds(5), + RetentionPeriod = TimeSpan.FromMinutes(1), + MaximumRetentionPeriod = TimeSpan.FromMinutes(5), + CleanupInterval = TimeSpan.FromMinutes(1), + CleanupBatchSize = 1000, + }; + + private static void AssertStreamingValidation(bool valid, AdoNetStreamOptions options, string propertyName) + { + if (valid) + { + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + else + { + var exception = Assert.Throws(() => new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration()); + Assert.Contains(propertyName, exception.Message, StringComparison.Ordinal); + } + } + private static void AssertValidation(bool valid, Action validate) { if (valid) diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs index a61364c3175..9c0e6af48ce 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs @@ -44,7 +44,15 @@ public void AdoNetBatchContainer_FromMessage_CreatesContainer() var temp = new AdoNetBatchContainer(streamId, events, requestContext); var serializer = fixture.Serializer.GetSerializer(); var payload = serializer.SerializeToArray(temp); - var message = new AdoNetStreamMessage("MyServiceId", "MyProviderId", "MyQueueId", 123, 234, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, payload); + var message = new AdoNetStreamMessage( + "MyServiceId", + "MyProviderId", + "MyQueueId", + 123, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); // act var container = AdoNetBatchContainer.FromMessage(serializer, message); @@ -54,7 +62,7 @@ public void AdoNetBatchContainer_FromMessage_CreatesContainer() Assert.Equal(events, container.Events); Assert.Equal(requestContext, container.RequestContext); Assert.Equal(new EventSequenceTokenV2(123), container.SequenceToken); - Assert.Equal(234, container.Dequeued); + Assert.Equal(0, container.Dequeued); } [Fact] @@ -104,7 +112,15 @@ public void AdoNetBatchContainer_GetEvents_FiltersEvents() var temp = new AdoNetBatchContainer(streamId, events, requestContext); var serializer = fixture.Serializer.GetSerializer(); var payload = serializer.SerializeToArray(temp); - var message = new AdoNetStreamMessage("MyServiceId", "MyProviderId", "MyQueueId", 123, 234, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, payload); + var message = new AdoNetStreamMessage( + "MyServiceId", + "MyProviderId", + "MyQueueId", + 123, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); // act var container = AdoNetBatchContainer.FromMessage(serializer, message); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs deleted file mode 100644 index 51bc6b258f1..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs +++ /dev/null @@ -1,552 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using MySql.Data.MySqlClient; -using Orleans.Configuration; -using Orleans.Streaming.AdoNet; -using Orleans.Tests.SqlUtils; -using System.Runtime.CompilerServices; -using TestExtensions; -using UnitTests.General; -using static System.String; -using RelationalOrleansQueries = Orleans.Streaming.AdoNet.Storage.RelationalOrleansQueries; - -namespace Tester.AdoNet.Streaming; - -/// -/// Provider-independent lifecycle tests for . -/// -[Collection(TestEnvironmentFixture.DefaultCollection)] -[TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("None")] -[TestSuite("BVT")] -[TestArea("Streaming")] -public class AdoNetQueueAdapterReceiverLifecycleTests(TestEnvironmentFixture fixture) -{ - [Fact] - public void AdoNetQueueAdapterReceiver_CanBeCreatedByAdapterFactory() => - RuntimeHelpers.RunClassConstructor(typeof(AdoNetQueueAdapter).TypeHandle); - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForDequeueBookkeepingBeforeRelease() - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - var getTask = receiver.GetQueueMessagesAsync(1); - Assert.Equal(1, queries.DequeueCalls); - - Task? shutdownTask = null; - queries.Dequeue.SetResult(new CallbackList([message], () => - { - shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - Assert.Equal(0, queries.ReleaseCalls); - })); - var dequeued = Assert.IsType(Assert.Single(await getTask.WaitAsync(cancellationToken))); - Assert.NotNull(shutdownTask); - await shutdownTask.WaitAsync(cancellationToken); - - Assert.Equal(message.MessageId, dequeued.SequenceToken.SequenceNumber); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForConcurrentDequeueAndConfirmation(bool confirmationFirst) - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - queries.Dequeue.SetResult([message]); - var dequeued = await receiver.GetQueueMessagesAsync(1); - queries.Dequeue = new(TaskCreationOptions.RunContinuationsAsynchronously); - var confirmationTask = receiver.MessagesDeliveredAsync(dequeued); - var getTask = receiver.GetQueueMessagesAsync(1); - Assert.Equal(2, queries.DequeueCalls); - Assert.Equal(1, queries.ConfirmationCalls); - - var shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - Assert.Empty(await receiver.GetQueueMessagesAsync(1)); - await receiver.MessagesDeliveredAsync(dequeued); - Assert.Equal(2, queries.DequeueCalls); - Assert.Equal(1, queries.ConfirmationCalls); - - var redelivered = message with { Dequeued = message.Dequeued + 1 }; - if (confirmationFirst) - { - queries.Confirmation.SetResult([new(message.ServiceId, message.ProviderId, message.QueueId, message.MessageId)]); - await confirmationTask.WaitAsync(cancellationToken); - Assert.False(shutdownTask.IsCompleted); - Assert.Equal(0, queries.ReleaseCalls); - queries.Dequeue.SetResult([redelivered]); - } - else - { - queries.Dequeue.SetResult([redelivered]); - await getTask.WaitAsync(cancellationToken); - Assert.False(shutdownTask.IsCompleted); - Assert.Equal(0, queries.ReleaseCalls); - queries.Confirmation.SetResult([new(message.ServiceId, message.ProviderId, message.QueueId, message.MessageId)]); - } - - await Task.WhenAll(getTask, confirmationTask, shutdownTask).WaitAsync(cancellationToken); - Assert.Equal(1, queries.ReleaseCalls); - var released = Assert.Single(queries.Released); - Assert.Equal(redelivered.MessageId, released.MessageId); - Assert.Equal(redelivered.Dequeued, released.Dequeued); - } - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForConfirmationBookkeeping() - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - queries.Dequeue.SetResult([message]); - var dequeued = await receiver.GetQueueMessagesAsync(1); - Task? shutdownTask = null; - var acknowledgements = new CallbackList( - [new(message.ServiceId, message.ProviderId, message.QueueId, message.MessageId)], () => - { - shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - Assert.Equal(0, queries.ReleaseCalls); - }); - var confirmationTask = receiver.MessagesDeliveredAsync(dequeued); - queries.Confirmation.SetResult(acknowledgements); - - await confirmationTask.WaitAsync(cancellationToken); - Assert.NotNull(shutdownTask); - await shutdownTask.WaitAsync(cancellationToken); - Assert.Equal(0, queries.ReleaseCalls); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ReleasesPendingAfterQueryFault(bool confirmation) - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - queries.Dequeue.SetResult([message]); - var dequeued = await receiver.GetQueueMessagesAsync(1); - queries.Dequeue = new(TaskCreationOptions.RunContinuationsAsynchronously); - Task operation = confirmation - ? receiver.MessagesDeliveredAsync(dequeued) - : receiver.GetQueueMessagesAsync(1); - var shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - - var exception = new InvalidOperationException("Query failed"); - if (confirmation) - { - queries.Confirmation.SetException(exception); - } - else - { - queries.Dequeue.SetException(exception); - } - - Assert.Same(exception, await Assert.ThrowsAsync(() => operation.WaitAsync(cancellationToken))); - await shutdownTask.WaitAsync(cancellationToken); - Assert.Equal(1, queries.ReleaseCalls); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ReleasesPendingAfterConversionFault() - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - var getTask = receiver.GetQueueMessagesAsync(1); - var shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - var serializer = fixture.Serializer.GetSerializer(); - queries.Dequeue.SetResult([message with { Payload = serializer.SerializeToArray(null!) }]); - - await Assert.ThrowsAsync(() => getTask.WaitAsync(cancellationToken)); - await shutdownTask.WaitAsync(cancellationToken); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ReleasesPendingAfterConfirmationPreparationFault() - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - queries.Dequeue.SetResult([message]); - await receiver.GetQueueMessagesAsync(1); - var incompleteBatch = new AdoNetBatchContainer(StreamId.Create("MyNamespace", "MyKey"), [], null); - - await Assert.ThrowsAsync(() => receiver.MessagesDeliveredAsync([incompleteBatch])); - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - - Assert.Equal(0, queries.ConfirmationCalls); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_TimeoutKeepsAdmissionClosedUntilActualDrain() - { - var cancellationToken = TestContext.Current.CancellationToken; - var (receiver, queries, message) = CreateReceiver(); - var getTask = receiver.GetQueueMessagesAsync(1); - - await receiver.Shutdown(TimeSpan.Zero).WaitAsync(cancellationToken); - Assert.False(getTask.IsCompleted); - Assert.Equal(0, queries.ReleaseCalls); - await receiver.Initialize(TimeSpan.FromSeconds(10)); - Assert.Empty(await receiver.GetQueueMessagesAsync(1)); - Assert.Equal(1, queries.DequeueCalls); - - var shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - queries.Dequeue.SetResult([message]); - var dequeued = await getTask.WaitAsync(cancellationToken); - await shutdownTask.WaitAsync(cancellationToken); - await receiver.MessagesDeliveredAsync(dequeued); - - Assert.Equal(0, queries.ConfirmationCalls); - Assert.Equal(1, queries.ReleaseCalls); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ClosesIdleReceiver() - { - var (receiver, queries, message) = CreateReceiver(); - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(TestContext.Current.CancellationToken); - await receiver.Initialize(TimeSpan.FromSeconds(10)); - Assert.Empty(await receiver.GetQueueMessagesAsync(1)); - var serializer = fixture.Serializer.GetSerializer(); - await receiver.MessagesDeliveredAsync([AdoNetBatchContainer.FromMessage(serializer, message)]); - - Assert.Equal(0, queries.DequeueCalls); - Assert.Equal(0, queries.ConfirmationCalls); - Assert.Equal(0, queries.ReleaseCalls); - } - - private (AdoNetQueueAdapterReceiver Receiver, BlockingStreamMessageQueries Queries, AdoNetStreamMessage Message) CreateReceiver() - { - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(StreamId.Create("MyNamespace", "MyKey"), [new TestModel(1)], null!)); - var now = DateTime.UtcNow; - var message = new AdoNetStreamMessage(serviceId, providerId, queueId, 42, 1, now.AddMinutes(5), now.AddHours(1), now, now, payload); - var queries = new BlockingStreamMessageQueries(TestContext.Current.CancellationToken); - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, queries, serializer, logger); - return (receiver, queries, message); - } - - [GenerateSerializer] - [Alias("Tester.AdoNet.Streaming.AdoNetQueueAdapterReceiverLifecycleTests.TestModel")] - public record TestModel( - [property: Id(0)] int Value); - - private sealed class CallbackList(IEnumerable items, Action beforeEnumerating) : List(items), IEnumerable - { - IEnumerator IEnumerable.GetEnumerator() - { - beforeEnumerating(); - return GetEnumerator(); - } - } - - private sealed class BlockingStreamMessageQueries(CancellationToken cancellationToken) : IStreamMessageQueries - { - public TaskCompletionSource> Dequeue { get; set; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public TaskCompletionSource> Confirmation { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public IList Released { get; private set; } = []; - public int DequeueCalls { get; private set; } - public int ConfirmationCalls { get; private set; } - public int ReleaseCalls { get; private set; } - - public async Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize) - { - DequeueCalls++; - return await Dequeue.Task.WaitAsync(cancellationToken); - } - - public async Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) - { - ConfirmationCalls++; - return await Confirmation.Task.WaitAsync(cancellationToken); - } - - public Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) - { - ReleaseCalls++; - Released = messages.ToList(); - return Task.FromResult>( - messages.Select(message => new AdoNetStreamConfirmationAck(serviceId, providerId, queueId, message.MessageId)).ToList()); - } - } -} - -/// -/// Tests for against SQL Server. -/// -[TestCategory("SqlServer"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : AdoNetQueueAdapterReceiverTests(AdoNetInvariants.InvariantNameSqlServer, fixture) -{ -} - -/// -/// Tests for against MySQL. -/// -[TestCategory("MySql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlAdoNetQueueAdapterReceiverTests : AdoNetQueueAdapterReceiverTests -{ - public MySqlAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : base(AdoNetInvariants.InvariantNameMySql, fixture) - { - MySqlConnection.ClearAllPools(); - } -} - -/// -/// Tests for against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : AdoNetQueueAdapterReceiverTests(AdoNetInvariants.InvariantNamePostgreSql, fixture) -{ -} - -/// -/// Tests for . -/// -[Collection(TestEnvironmentFixture.DefaultCollection)] -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class AdoNetQueueAdapterReceiverTests(string invariant, TestEnvironmentFixture fixture) : IAsyncLifetime -{ - private readonly TestEnvironmentFixture _fixture = fixture; - private RelationalStorageForTesting _testing = null!; - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - private const string TestDatabaseName = "OrleansStreamTest"; - - public async ValueTask InitializeAsync() - { - _testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - Assert.SkipWhen(IsNullOrEmpty(_testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = _testing.Storage; - _queries = await RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString); - } - - /// - /// Tests that the can get and confirm messages. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_GetsMessages_ConfirmsMessages() - { - // arrange - receiver - var serviceId = "MyServiceId"; - var clusterOptions = new ClusterOptions - { - ServiceId = serviceId - }; - var providerId = "MyProviderId"; - var queueId = "MyQueueId"; - var maxCount = 10; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - - // disable eviction for this test - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - await receiver.Initialize(TimeSpan.FromSeconds(10)); - - // arrange - data - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var events = new List { new TestModel(1), new TestModel(2), new TestModel(3) }; - var context = new Dictionary { { "MyKey", "MyValue" } }; - var container = new AdoNetBatchContainer(streamId, events, context); - var payload = serializer.SerializeToArray(container); - - // arrange - enqueue (via storage) some invalid messages followed by a valid message - var ackExpired = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 0); - var ackOtherQueueId = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId + "X", payload, 100); - var ackOtherProviderId = await _queries.QueueStreamMessageAsync(serviceId, providerId + "X", queueId, payload, 100); - var ackOtherServiceId = await _queries.QueueStreamMessageAsync(serviceId + "X", providerId, queueId, payload, 100); - var ackValid = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - // act - dequeue messages via receiver - var dequeued = await receiver.GetQueueMessagesAsync(maxCount); - Assert.NotNull(dequeued); - var storedDequeued = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToDictionary(x => x.MessageId); - - // act - confirm messages via receiver - await receiver.MessagesDeliveredAsync(dequeued); - var storedConfirmed = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToDictionary(x => x.MessageId); - - // assert - dequeued messages are as expected - var single = Assert.IsType(Assert.Single(dequeued)); - Assert.NotNull(single.RequestContext); - Assert.Equal(streamId, single.StreamId); - Assert.Equal(events, single.Events); - Assert.Equal(context.Select(x => (x.Key, x.Value)), single.RequestContext.Select(x => (x.Key, x.Value))); - Assert.Equal(ackValid.MessageId, single.SequenceToken.SequenceNumber); - Assert.Equal(1, single.Dequeued); - - // assert - storage is as expected after dequeuing - Assert.Equal(5, storedDequeued.Count); - Assert.Equal(0, storedDequeued[ackExpired.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherQueueId.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherProviderId.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherServiceId.MessageId].Dequeued); - Assert.Equal(1, storedDequeued[ackValid.MessageId].Dequeued); - - // assert - stored confirmed messages - Assert.Equal(4, storedConfirmed.Count); - Assert.True(storedConfirmed.ContainsKey(ackExpired.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherQueueId.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherProviderId.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherServiceId.MessageId)); - Assert.False(storedConfirmed.ContainsKey(ackValid.MessageId)); - } - - /// - /// Tests that shutting down a receiver immediately releases its unconfirmed messages. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ReleasesUnconfirmedMessages() - { - var cancellationToken = TestContext.Current.CancellationToken; - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(streamId, [new TestModel(1)], null!)); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var first = Assert.IsType( - Assert.Single(await receiver.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(1, first.Dequeued); - - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - - var replacement = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var redelivered = Assert.IsType( - Assert.Single(await replacement.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(ack.MessageId, redelivered.SequenceToken.SequenceNumber); - Assert.Equal(2, redelivered.Dequeued); - await replacement.MessagesDeliveredAsync([redelivered]).WaitAsync(cancellationToken); - await replacement.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - } - - /// - /// Tests that waits for the outstanding task. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForOutstandingTask() - { - var cancellationToken = TestContext.Current.CancellationToken; - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(StreamId.Create("MyNamespace", "MyKey"), [new TestModel(1)], null!)); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - var getTask = receiver.GetQueueMessagesAsync(1); - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - - Assert.True(getTask.IsCompleted); - var first = Assert.IsType(Assert.Single(await getTask.WaitAsync(cancellationToken))); - Assert.Equal(1, first.Dequeued); - - var replacement = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var redelivered = Assert.IsType( - Assert.Single(await replacement.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(ack.MessageId, redelivered.SequenceToken.SequenceNumber); - Assert.Equal(2, redelivered.Dequeued); - await replacement.MessagesDeliveredAsync([redelivered]).WaitAsync(cancellationToken); - await replacement.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - - [GenerateSerializer] - [Alias("Tester.AdoNet.Streaming.AdoNetQueueAdapterReceiverTests.TestModel")] - public record TestModel( - [property: Id(0)] int Value); -} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index a58641a97bf..fe61c8c61c5 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -123,8 +123,7 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString, - ExpiryTimeout = TimeSpan.FromSeconds(100) + ConnectionString = _storage.ConnectionString }; var serializer = _fixture.Serializer.GetSerializer(); var logger = NullLogger.Instance; @@ -146,95 +145,9 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() // assert - stored messages are as expected var stored = (await _storage.ReadAsync( "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToList(); - for (var i = 0; i < stored.Count; i++) - { - var item = stored[i]; - - Assert.Equal(serviceId, item.ServiceId); - Assert.Equal(providerId, item.ProviderId); - Assert.Equal(adoNetQueueId, item.QueueId); - Assert.NotEqual(0, item.MessageId); - Assert.Equal(0, item.Dequeued); - Assert.True(item.VisibleOn >= beforeEnqueued); - Assert.True(item.VisibleOn <= afterEnqueued); - Assert.True(item.ExpiresOn >= beforeEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.True(item.ExpiresOn <= afterEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.Equal(item.VisibleOn, item.CreatedOn); - Assert.Equal(item.VisibleOn, item.ModifiedOn); - - var serializedContainer = serializer.Deserialize(item.Payload); - Assert.NotNull(serializedContainer); - Assert.NotNull(serializedContainer.RequestContext); - Assert.Equal(streamId, serializedContainer.StreamId); - Assert.Null(serializedContainer.SequenceToken); - Assert.Equal(new[] { new TestModel(i + 1) }, serializedContainer.Events); - Assert.Single(serializedContainer.RequestContext); - Assert.Equal("MyValue", serializedContainer.RequestContext["MyKey"]); - Assert.Equal(0, serializedContainer.Dequeued); - } - } - - /// - /// Tests that the can enqueue messages that are visible to its receivers. - /// - [Fact] - public async Task AdoNetQueueAdapter_WiresUpReceiver() - { - // arrange - var serviceId = "MyServiceId"; - var clusterOptions = new ClusterOptions - { - ServiceId = serviceId - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var providerId = "MyProviderId"; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var hashOptions = new HashRingStreamQueueMapperOptions { TotalQueueCount = 8 }; - var hashMapper = new HashRingBasedStreamQueueMapper(hashOptions, "MyQueue"); - var queueId = hashMapper.GetQueueForStream(streamId); - var adoMapper = new AdoNetStreamQueueMapper(hashMapper); - var adoNetQueueId = adoMapper.GetAdoNetQueueId(streamId); - var adapter = new AdoNetQueueAdapter(providerId, streamOptions, clusterOptions, cacheOptions, adoMapper, _queries, serializer, logger, _fixture.Services); - - // act - enqueue (via adapter) some messages - var beforeEnqueued = DateTime.UtcNow.AddSeconds(-1); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(1) }, null!, new Dictionary { { "MyKey", 1 } }); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(2) }, null!, new Dictionary { { "MyKey", 2 } }); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(3) }, null!, new Dictionary { { "MyKey", 3 } }); - var afterEnqueued = DateTime.UtcNow.AddSeconds(1); - - // act - grab receiver and dequeue messages - var receiver = adapter.CreateReceiver(queueId); - await receiver.Initialize(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - var beforeDequeued = DateTime.UtcNow.AddSeconds(-1); - var messages = await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); - var afterDequeued = DateTime.UtcNow.AddSeconds(1); - - // assert - dequeued messages are as expected - Assert.NotNull(messages); - Assert.Equal(3, messages.Count); - for (var i = 0; i < messages.Count; i++) - { - var message = messages[i]; - - Assert.Equal(streamId, message.StreamId); - Assert.Equal([new TestModel(i + 1)], message.GetEvents().Select(x => x.Item1)); - Assert.True(message.ImportRequestContext()); - Assert.Equal(i + 1, RequestContext.Get("MyKey")); - } - - // assert - stored messages are as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToList(); + TestContext.Current.CancellationToken)) + .OrderBy(static message => message.MessageId) + .ToList(); for (var i = 0; i < stored.Count; i++) { var item = stored[i]; @@ -243,15 +156,11 @@ public async Task AdoNetQueueAdapter_WiresUpReceiver() Assert.Equal(providerId, item.ProviderId); Assert.Equal(adoNetQueueId, item.QueueId); Assert.NotEqual(0, item.MessageId); - Assert.Equal(1, item.Dequeued); - Assert.True(item.VisibleOn >= beforeDequeued.Add(streamOptions.VisibilityTimeout)); - Assert.True(item.VisibleOn <= afterDequeued.Add(streamOptions.VisibilityTimeout)); - Assert.True(item.ExpiresOn >= beforeEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.True(item.ExpiresOn <= afterEnqueued.Add(streamOptions.ExpiryTimeout)); + Assert.Equal(streamId.FullKey.ToArray(), item.StreamIdBytes); + Assert.Equal(streamId.Namespace.Length, item.StreamNamespaceLength); + Assert.Equal(streamId, item.StreamId); Assert.True(item.CreatedOn >= beforeEnqueued); Assert.True(item.CreatedOn <= afterEnqueued); - Assert.True(item.ModifiedOn >= beforeDequeued); - Assert.True(item.ModifiedOn <= afterDequeued); var serializedContainer = serializer.Deserialize(item.Payload); Assert.NotNull(serializedContainer); @@ -260,7 +169,7 @@ public async Task AdoNetQueueAdapter_WiresUpReceiver() Assert.Null(serializedContainer.SequenceToken); Assert.Equal(new[] { new TestModel(i + 1) }, serializedContainer.Events); Assert.Single(serializedContainer.RequestContext); - Assert.Equal(i + 1, serializedContainer.RequestContext["MyKey"]); + Assert.Equal("MyValue", serializedContainer.RequestContext["MyKey"]); Assert.Equal(0, serializedContainer.Dequeued); } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs deleted file mode 100644 index 206821554ad..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs +++ /dev/null @@ -1,222 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using MySql.Data.MySqlClient; -using Orleans.Configuration; -using Orleans.Providers.Streams.Common; -using Orleans.Streaming.AdoNet; -using Orleans.Streams; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; -using RelationalOrleansQueries = Orleans.Streaming.AdoNet.Storage.RelationalOrleansQueries; - -namespace Tester.AdoNet.Streaming; - -/// -/// Tests for against SQL Server. -/// -[TestCategory("SqlServer"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerAdoNetStreamFailureHandlerTests() : AdoNetStreamFailureHandlerTests(AdoNetInvariants.InvariantNameSqlServer) -{ -} - -/// -/// Tests for against MySQL. -/// -[TestCategory("MySql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlAdoNetStreamFailureHandlerTests : AdoNetStreamFailureHandlerTests -{ - public MySqlAdoNetStreamFailureHandlerTests() : base(AdoNetInvariants.InvariantNameMySql) - { - MySqlConnection.ClearAllPools(); - } -} - -/// -/// Tests for against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlAdoNetStreamFailureHandlerTests() : AdoNetStreamFailureHandlerTests(AdoNetInvariants.InvariantNamePostgreSql) -{ -} - -/// -/// Tests for . -/// -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class AdoNetStreamFailureHandlerTests(string invariant) : IAsyncLifetime -{ - private RelationalStorageForTesting _testing = null!; - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - private const string TestDatabaseName = "OrleansStreamTest"; - - public async ValueTask InitializeAsync() - { - _testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - - Assert.SkipWhen(IsNullOrEmpty(_testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = _testing.Storage; - _queries = await RelationalOrleansQueries.CreateInstance(invariant, _testing.CurrentConnectionString); - } - - /// - /// Tests that a can be constructed. - /// - [Fact] - public void AdoNetStreamFailureHandler_Constructs() - { - // arrange - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - - // act - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // assert - Assert.Equal(faultOnFailure, handler.ShouldFaultSubsriptionOnError); - } - - /// - /// Tests that a can move a poisoned message to dead letters. - /// - [Fact] - public async Task AdoNetStreamFailureHandler_OnDeliveryFailure_MovesPoisonedMessageToDeadLetters() - { - // arrange - handler - var providerId = "MyProviderId"; - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - MaxAttempts = 1 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var agentOptions = new StreamPullingAgentOptions - { - MaxEventDeliveryTime = TimeSpan.FromSeconds(0) - }; - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // arrange - queue an expired message - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var queueId = mapper.GetAdoNetQueueId(streamId); - var payload = new byte[] { 0xFF }; - - var ack = await _queries.QueueStreamMessageAsync(clusterOptions.ServiceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync( - ack.ServiceId, - ack.ProviderId, - ack.QueueId, - cacheOptions.CacheSize, - streamOptions.MaxAttempts, - agentOptions.MaxEventDeliveryTime.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - - // act - clean up with max attempts of one so the message above is flagged - await handler.OnDeliveryFailure(GuidId.GetNewGuidId(), providerId, streamId, new EventSequenceTokenV2(ack.MessageId)); - - // assert - var dead = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - Assert.Equal(clusterOptions.ServiceId, dead.ServiceId); - Assert.Equal(providerId, dead.ProviderId); - Assert.Equal(queueId, dead.QueueId); - Assert.Equal(ack.MessageId, dead.MessageId); - Assert.Equal(1, dead.Dequeued); - Assert.Equal(dead.CreatedOn.Add(streamOptions.ExpiryTimeout.SecondsCeiling()), dead.ExpiresOn); - Assert.Equal(dead.ModifiedOn, dead.VisibleOn); - Assert.Equal(dead.DeadOn.Add(streamOptions.DeadLetterEvictionTimeout.SecondsCeiling()), dead.RemoveOn); - Assert.Equal(payload, dead.Payload); - } - - /// - /// Tests that a can move a poisoned message to dead letters. - /// - [Fact] - public async Task AdoNetStreamFailureHandler_OnDeliveryFailure_DoesNotMoveHealthyMessageToDeadLetters() - { - // arrange - handler - var providerId = "MyProviderId"; - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var agentOptions = new StreamPullingAgentOptions(); - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // arrange - queue an expired message - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var queueId = mapper.GetAdoNetQueueId(streamId); - var payload = new byte[] { 0xFF }; - var ack = await _queries.QueueStreamMessageAsync(clusterOptions.ServiceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync( - ack.ServiceId, - ack.ProviderId, - ack.QueueId, - cacheOptions.CacheSize, - streamOptions.MaxAttempts, - agentOptions.MaxEventDeliveryTime.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - - // act - clean up with max attempts of one so the message above is flagged - await handler.OnDeliveryFailure(GuidId.GetNewGuidId(), providerId, streamId, new EventSequenceTokenV2(ack.MessageId)); - - // assert - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; -} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs new file mode 100644 index 00000000000..7d1ff206b7b --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs @@ -0,0 +1,131 @@ +using System.Text; +using Orleans.Runtime; +using Orleans.Streaming.AdoNet; + +namespace Tester.AdoNet.Streaming; + +/// +/// Tests that reconstructs the canonical +/// bytes and namespace boundary exactly as they would be +/// persisted in the StreamIdBytes/StreamNamespaceLength columns of +/// OrleansStreamMessage. These tests require no database and always run. +/// +[TestCategory("AdoNet"), TestCategory("Streaming"), TestCategory("BVT")] +[TestProvider("None")] +[TestSuite("BVT")] +[TestArea("Streaming")] +public sealed class AdoNetStreamMessageStreamIdTests +{ + private static AdoNetStreamMessage CreateMessage(StreamId streamId, long messageId = 1) => + new( + ServiceId: "service", + ProviderId: "provider", + QueueId: "queue", + MessageId: messageId, + StreamIdBytes: streamId.FullKey.ToArray(), + StreamNamespaceLength: streamId.Namespace.Length, + CreatedOn: DateTime.UtcNow, + Payload: [1, 2, 3]); + + [Fact] + public void StreamId_RoundTrips_WithNamespace() + { + var original = StreamId.Create("orders-namespace", Guid.NewGuid()); + + var message = CreateMessage(original); + + Assert.Equal(original, message.StreamId); + Assert.True(original.FullKey.Span.SequenceEqual(message.StreamIdBytes)); + Assert.Equal(original.Namespace.Length, message.StreamNamespaceLength); + Assert.True(original.Namespace.Span.SequenceEqual(message.StreamId.Namespace.Span)); + Assert.True(original.Key.Span.SequenceEqual(message.StreamId.Key.Span)); + } + + [Fact] + public void StreamId_RoundTrips_WithoutNamespace() + { + var original = StreamId.Create(ns: null, key: Guid.NewGuid()); + + var message = CreateMessage(original); + + Assert.Equal(0, original.Namespace.Length); + Assert.Equal(0, message.StreamNamespaceLength); + Assert.Equal(original, message.StreamId); + Assert.True(original.Key.Span.SequenceEqual(message.StreamId.Key.Span)); + Assert.True(original.FullKey.Span.SequenceEqual(message.StreamId.FullKey.Span)); + } + + [Fact] + public void StreamId_RoundTrips_WithStringKeyAndNamespace() + { + var original = StreamId.Create("tenant/42", "order-key-123"); + + var message = CreateMessage(original); + + Assert.Equal(original, message.StreamId); + Assert.Equal(Encoding.UTF8.GetByteCount("tenant/42"), message.StreamNamespaceLength); + Assert.Equal("order-key-123", Encoding.UTF8.GetString(message.StreamId.Key.Span)); + } + + [Theory] + [InlineData(null, "just-a-key")] + [InlineData("", "key-with-empty-namespace")] + [InlineData("ns", "k")] + [InlineData("namespace-with-unicode-\u00e9\u00e8", "key-with-unicode-\u00fc")] + public void StreamId_NamespaceBoundary_SeparatesNamespaceAndKeyExactly(string? ns, string key) + { + var original = StreamId.Create(ns, key); + var message = CreateMessage(original); + + // Re-derive the namespace/key split purely from the stored bytes and boundary, + // exactly as a storage-layer reader would, rather than trusting StreamId.Equals + // (which compares only the full key bytes and ignores the namespace boundary). + var namespaceBytes = message.StreamIdBytes.AsSpan(0, message.StreamNamespaceLength).ToArray(); + var keyBytes = message.StreamIdBytes.AsSpan(message.StreamNamespaceLength).ToArray(); + + Assert.True(original.Namespace.Span.SequenceEqual(namespaceBytes)); + Assert.True(original.Key.Span.SequenceEqual(keyBytes)); + Assert.Equal(key, Encoding.UTF8.GetString(keyBytes)); + } + + [Fact] + public void StreamId_DifferentNamespaceBoundary_ProducesDifferentNamespaceAndKeySplit() + { + // StreamId.Equals compares only the full key bytes, so an off-by-one boundary mutation + // would NOT be caught by an equality assertion alone. Guard the boundary explicitly by + // asserting the derived Namespace/Key spans themselves differ when the stored + // StreamNamespaceLength is shifted by one byte. + var original = StreamId.Create("ns", "key"); + var shiftedMessage = new AdoNetStreamMessage( + "service", "provider", "queue", 1, + original.FullKey.ToArray(), + original.Namespace.Length + 1, + DateTime.UtcNow, + [1]); + + var shifted = shiftedMessage.StreamId; + + // Sanity: the underlying full key bytes are unchanged. + Assert.True(original.FullKey.Span.SequenceEqual(shifted.FullKey.Span)); + + // But the namespace/key split has moved, and must be observably different. + Assert.NotEqual(original.Namespace.Length, shifted.Namespace.Length); + Assert.False(original.Namespace.Span.SequenceEqual(shifted.Namespace.Span)); + Assert.False(original.Key.Span.SequenceEqual(shifted.Key.Span)); + } + + [Fact] + public void StreamId_FullKey_IsExactByteConcatenationOfNamespaceAndKey() + { + var original = StreamId.Create("orders", "order-42"); + var message = CreateMessage(original); + + var reconstructedFullKey = message.StreamIdBytes; + var expectedFullKey = new byte[original.Namespace.Length + original.Key.Length]; + original.Namespace.Span.CopyTo(expectedFullKey.AsSpan(0, original.Namespace.Length)); + original.Key.Span.CopyTo(expectedFullKey.AsSpan(original.Namespace.Length)); + + Assert.Equal(expectedFullKey, reconstructedFullKey); + Assert.Equal(original.Namespace.Length + original.Key.Length, reconstructedFullKey.Length); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs new file mode 100644 index 00000000000..b1d8f88a533 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs @@ -0,0 +1,979 @@ +using System.Data; +using System.Data.Common; +using Microsoft.Data.SqlClient; +using MySql.Data.MySqlClient; +using Npgsql; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Streaming.AdoNet; +using Orleans.Streaming.AdoNet.Storage; +using UnitTests.General; +using static System.String; + +namespace Tester.AdoNet.Streaming; + +/// +/// Tests the immutable partition-log storage layer via against Sql Server. +/// +[TestCategory("SqlServer"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("SqlServer")] +[TestSuite("Functional")] +public class SqlServerAdoNetStreamPartitionLogTests() : AdoNetStreamPartitionLogTests(AdoNetInvariants.InvariantNameSqlServer) +{ + /// + /// Concurrent appends to the same partition must be serialized by the partition-row lock, and a + /// rolled-back append must not permanently burn its allocated message identifier. + /// + [SkippableFact] + public Task AppendStreamMessage_ConcurrentAppends_AreSerializedAndRollbackDoesNotBurnIds() => + VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoesNotBurnIds(); + + /// + /// A reader must never observe a message allocated by a transaction that has not yet committed. + /// + [SkippableFact] + public Task ReadStreamMessages_ExcludesUncommittedInFlightAppend() => + VerifySqlServerReadExcludesUncommittedInFlightAppend(); + + [SkippableFact] + public Task AppendStreamMessage_DifferentPartitionsDoNotShareTheAllocationLock() => + VerifySqlServerPartitionsAreIndependent(); +} + +/// +/// Tests the immutable partition-log storage layer via against MySQL. +/// +[TestCategory("MySql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("MySql")] +[TestSuite("Functional")] +public class MySqlAdoNetStreamPartitionLogTests : AdoNetStreamPartitionLogTests +{ + public MySqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.InvariantNameMySql) + { + MySqlConnection.ClearAllPools(); + } + + [SkippableFact] + public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyMySqlRollbackRestoresAllocation(); +} + +/// +/// Tests the immutable partition-log storage layer via against PostgreSQL. +/// +[TestCategory("PostgreSql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("PostgreSql")] +[TestSuite("Functional")] +public class PostgreSqlAdoNetStreamPartitionLogTests : AdoNetStreamPartitionLogTests +{ + public PostgreSqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.InvariantNamePostgreSql) + { + NpgsqlConnection.ClearAllPools(); + } + + [SkippableFact] + public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyPostgreSqlRollbackRestoresAllocation(); +} + +/// +/// Tests the immutable partition-log storage layer via : transactional +/// append with rollback-safe allocation, exclusive ordered reads, epoch-fenced monotonic checkpoints, +/// bounded retention/cleanup with hard-ceiling diagnostics, partition independence, and explicit schema +/// version/query-key enforcement. +/// +[TestCategory("AdoNet"), TestCategory("Streaming")] +[TestSuite("Functional")] +[TestArea("Streaming")] +public abstract class AdoNetStreamPartitionLogTests(string invariant) : IAsyncLifetime +{ + private const string TestDatabaseName = "OrleansStreamTest"; + + private IRelationalStorage _storage = null!; + private RelationalOrleansQueries _queries = null!; + + public async Task InitializeAsync() + { + var testing = await RelationalStorageForTesting.SetupInstance(invariant, TestDatabaseName); + Skip.If(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); + } + + public Task DisposeAsync() => Task.CompletedTask; + + #region Helpers + + // A large range keeps identifiers effectively unique across the partition-independence tests, + // which (unlike the single-partition tests) need two distinct queue ids within the same test. + private static string RandomServiceId(int max = 1_000_000) => $"ServiceId{Random.Shared.Next(max)}"; + + private static string RandomProviderId(int max = 1_000_000) => $"ProviderId{Random.Shared.Next(max)}"; + + private static string RandomQueueId(int max = 1_000_000) => $"QueueId{Random.Shared.Next(max)}"; + + private static byte[] RandomPayload(int size = 128) + { + var payload = new byte[size]; + Random.Shared.NextBytes(payload); + return payload; + } + + private static (byte[] StreamIdBytes, int StreamNamespaceLength) RandomStreamKey() + { + var streamId = StreamId.Create($"ns-{Guid.NewGuid():N}", Guid.NewGuid()); + return (streamId.FullKey.ToArray(), streamId.Namespace.Length); + } + + private Task AppendAsync(string serviceId, string providerId, string queueId, byte[]? payload = null) + { + var (streamIdBytes, nsLength) = RandomStreamKey(); + return _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload ?? RandomPayload()); + } + + private Task AgePartitionMessagesAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamMessage + SET CreatedOn = @CreatedOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CreatedOn", DateTime.UtcNow.AddDays(-2)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + + private Task MakeCleanupDueAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamPartition + SET CleanupOn = @CleanupOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CleanupOn", DateTime.UtcNow.AddSeconds(-1)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + + private static void AddParameter(IDbCommand command, string name, object value) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.Value = value; + command.Parameters.Add(parameter); + } + + #endregion Helpers + + #region Append: sequential allocation and partition independence + + [SkippableFact] + public async Task AppendStreamMessage_AllocatesSequentialMessageIdsPerPartition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var first = await AppendAsync(serviceId, providerId, queueId); + var second = await AppendAsync(serviceId, providerId, queueId); + var third = await AppendAsync(serviceId, providerId, queueId); + + Assert.Equal(1, first.MessageId); + Assert.Equal(2, second.MessageId); + Assert.Equal(3, third.MessageId); + } + + [SkippableFact] + public async Task AppendStreamMessage_ConcurrentAppendsAreGapFree() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + const int count = 32; + + var results = await Task.WhenAll(Enumerable.Range(0, count).Select(_ => AppendAsync(serviceId, providerId, queueId))); + + Assert.Equal(Enumerable.Range(1, count).Select(static value => (long)value), results.Select(static result => result.MessageId).Order()); + } + + [SkippableFact] + public async Task AppendStreamMessage_PartitionsAreIndependent() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueIdA = RandomQueueId(); + var queueIdB = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueIdA); + var secondA = await AppendAsync(serviceId, providerId, queueIdA); + var firstB = await AppendAsync(serviceId, providerId, queueIdB); + + Assert.Equal(2, secondA.MessageId); + Assert.Equal(1, firstB.MessageId); + + var boundsA = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdA); + var boundsB = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdB); + + Assert.Equal(2, boundsA!.TailMessageId); + Assert.Equal(1, boundsB!.TailMessageId); + Assert.Equal(1, boundsA.EarliestMessageId); + Assert.Equal(1, boundsB.EarliestMessageId); + } + + [SkippableFact] + public async Task AppendStreamMessage_ValidatesStreamNamespaceBoundary() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var payload = RandomPayload(); + var (streamIdBytes, _) = RandomStreamKey(); + + // An empty stream key (no bytes at all) is never a valid canonical StreamId. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, [], 0, payload)); + + // A negative boundary cannot separate namespace from key. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, -1, payload)); + + // A boundary consuming the entire key would leave an empty stream key, which is invalid. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, streamIdBytes.Length, payload)); + + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, null!, 0, payload)); + + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, 0, null!)); + + // Valid boundaries must NOT throw: a zero-length namespace (entire key, no namespace) and a + // namespace that consumes every byte except the last (a single-byte key) are both legal, and + // each still allocates the next sequential message identifier as normal. + var zeroNamespaceAck = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, 0, payload); + Assert.Equal(1, zeroNamespaceAck.MessageId); + var maxNamespaceAck = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, streamIdBytes.Length - 1, payload); + Assert.Equal(2, maxNamespaceAck.MessageId); + } + + [SkippableFact] + public async Task AppendStreamMessage_PersistsCanonicalStreamIdFullKeyAndNamespaceBoundary() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var streamId = StreamId.Create("orders", Guid.NewGuid()); + var payload = RandomPayload(); + + var ack = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamId.FullKey.ToArray(), streamId.Namespace.Length, payload); + var messages = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 10); + var stored = Assert.Single(messages); + + Assert.Equal(ack.MessageId, stored.MessageId); + Assert.True(streamId.FullKey.Span.SequenceEqual(stored.StreamIdBytes)); + Assert.Equal(streamId.Namespace.Length, stored.StreamNamespaceLength); + Assert.Equal(streamId, stored.StreamId); + Assert.True(streamId.Namespace.Span.SequenceEqual(stored.StreamId.Namespace.Span)); + Assert.True(streamId.Key.Span.SequenceEqual(stored.StreamId.Key.Span)); + Assert.Equal(payload, stored.Payload); + } + + #endregion Append + + #region Acquire: ownership, epoch, and checkpoint initialization + + [SkippableFact] + public async Task AcquireStreamPartition_InitializesCheckpointBeforeEarliestWhenNotStartingFromNow() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(1, state.OwnerEpoch); + Assert.Equal(0, state.Checkpoint); // one before the earliest retained message (id 1) + Assert.Equal(1, state.EarliestMessageId); + Assert.Equal(2, state.TailMessageId); + } + + [SkippableFact] + public async Task AcquireStreamPartition_InitializesCheckpointAtTailWhenStartingFromNow() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + var last = await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + + Assert.Equal(last.MessageId, state.Checkpoint); + Assert.Equal(last.MessageId, state.TailMessageId); + } + + [SkippableFact] + public async Task AcquireStreamPartition_OnNeverAppendedPartitionHasNoBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(1, state.OwnerEpoch); + Assert.Equal(0, state.Checkpoint); + Assert.Null(state.EarliestMessageId); + Assert.Null(state.TailMessageId); + } + + [SkippableFact] + public async Task AcquireStreamPartition_ReacquisitionIncrementsOwnerEpochAndPreservesCheckpoint() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + + var first = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, first.OwnerEpoch, 1); + + var second = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(first.OwnerEpoch + 1, second.OwnerEpoch); + Assert.Equal(1, second.Checkpoint); // preserved from the prior owner, not re-initialized + } + + [SkippableFact] + public async Task AcquireStreamPartition_OwnerEpochAndCheckpointArePerPartition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueIdA = RandomQueueId(); + var queueIdB = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueIdA); + await AppendAsync(serviceId, providerId, queueIdB); + + var stateA1 = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdA, startFromNow: false); + await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdA, startFromNow: false); + var stateB1 = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdB, startFromNow: false); + + var boundsA = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdA); + var boundsB = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdB); + + Assert.Equal(stateA1.OwnerEpoch + 1, boundsA!.OwnerEpoch); + Assert.Equal(stateB1.OwnerEpoch, boundsB!.OwnerEpoch); + } + + #endregion Acquire + + #region Read: exclusive ordered ranges + + [SkippableFact] + public async Task ReadStreamMessages_ReturnsExclusiveOrderedRangeRespectingMaxCount() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 5; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var page = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: acks[1].MessageId, maxCount: 2); + + Assert.Equal([acks[2].MessageId, acks[3].MessageId], page.Select(m => m.MessageId)); + Assert.Equal(page.OrderBy(m => m.MessageId).Select(m => m.MessageId), page.Select(m => m.MessageId)); + } + + [SkippableFact] + public async Task ReadStreamMessages_EmptyWhenAfterMessageIdIsAtOrBeyondTail() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var only = await AppendAsync(serviceId, providerId, queueId); + + var atTail = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: only.MessageId, maxCount: 10); + var beyondTail = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: only.MessageId + 100, maxCount: 10); + + Assert.Empty(atTail); + Assert.Empty(beyondTail); + } + + [SkippableFact] + public async Task ReadStreamMessages_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: -1, maxCount: 1)); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 0)); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: -5)); + + // Valid boundaries must NOT throw: afterMessageId = 0 (read from the very start) and + // maxCount = 1 (the smallest legal page size) are both legal and return the expected message. + var appended = await AppendAsync(serviceId, providerId, queueId); + var page = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 1); + var onlyMessage = Assert.Single(page); + Assert.Equal(appended.MessageId, onlyMessage.MessageId); + } + + #endregion Read + + #region Checkpoint: monotonicity, non-regression, and epoch fencing + + [SkippableFact] + public async Task AdvanceStreamCheckpoint_AdvancesMonotonicallyAndRejectsRegression() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + var advanceTo2 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 2); + Assert.NotNull(advanceTo2); + Assert.True(advanceTo2!.Updated); + Assert.Equal(2, advanceTo2.Checkpoint); + + var regressTo1 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 1); + Assert.NotNull(regressTo1); + Assert.False(regressTo1!.Updated); + Assert.Equal(2, regressTo1.Checkpoint); // unchanged by the rejected regression + + var sameValue = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 2); + Assert.NotNull(sameValue); + Assert.False(sameValue!.Updated); // strictly-forward only: re-affirming the same value is not an advance + Assert.Equal(2, sameValue.Checkpoint); + + var advanceTo3 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 3); + Assert.True(advanceTo3!.Updated); + Assert.Equal(3, advanceTo3.Checkpoint); + } + + [SkippableFact] + public async Task AdvanceStreamCheckpoint_RejectsCheckpointAtOrBeyondTail() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var only = await AppendAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + // The checkpoint marks the last fully-processed message; it cannot reach or exceed the + // not-yet-allocated next message identifier. + var result = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, only.MessageId + 1); + + Assert.NotNull(result); + Assert.False(result!.Updated); + } + + [SkippableFact] + public async Task AdvanceStreamCheckpoint_FencesStaleOwnerEpochAfterReacquisition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var firstOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + var secondOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + Assert.True(secondOwner.OwnerEpoch > firstOwner.OwnerEpoch); + + // The former owner's epoch must be fenced: its checkpoint attempt must not apply. + var staleAttempt = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, firstOwner.OwnerEpoch, 2); + Assert.NotNull(staleAttempt); + Assert.False(staleAttempt!.Updated); + + // The current owner's epoch must still be able to advance the checkpoint. + var currentAttempt = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, secondOwner.OwnerEpoch, 2); + Assert.NotNull(currentAttempt); + Assert.True(currentAttempt!.Updated); + Assert.Equal(2, currentAttempt.Checkpoint); + } + + [SkippableFact] + public async Task AdvanceStreamCheckpoint_ReturnsNullForUnknownPartition() + { + var result = await _queries.AdvanceStreamCheckpointAsync(RandomServiceId(), RandomProviderId(), RandomQueueId(), ownerEpoch: 1, checkpoint: 1); + + Assert.Null(result); + } + + [SkippableFact] + public async Task AdvanceStreamCheckpoint_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 0, checkpoint: 1)); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: -1, checkpoint: 1)); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 1, checkpoint: -1)); + + // Valid boundaries must NOT throw: ownerEpoch = 1 (the smallest legal epoch) and + // checkpoint = 0 (the smallest legal checkpoint) are both legal argument values, even + // though there is no matching partition row here (which is reported via a null result, + // not an exception). + var result = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 1, checkpoint: 0); + Assert.Null(result); + } + + #endregion Checkpoint + + #region Bounds: partition state reporting + + [SkippableFact] + public async Task GetStreamPartitionBounds_ReflectsCurrentState() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + Assert.Null(await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId)); + + await AppendAsync(serviceId, providerId, queueId); + var second = await AppendAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, second.MessageId); + + var bounds = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + + Assert.NotNull(bounds); + Assert.Equal(state.OwnerEpoch, bounds!.OwnerEpoch); + Assert.Equal(second.MessageId, bounds.Checkpoint); + Assert.Equal(1, bounds.EarliestMessageId); + Assert.Equal(second.MessageId, bounds.TailMessageId); + } + + #endregion Bounds + + #region Cleanup: retention, hard ceiling diagnostics, batching, and throttling + + [SkippableFact] + public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentionElapses() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 3; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); + + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + + var result = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(result.Ran); + Assert.Equal(3, result.DeletedCount); + Assert.Equal(acks[^1].MessageId, result.DeletedThroughMessageId); + Assert.Equal(0, result.HardDeletedCount); + Assert.Null(result.HardDeletedFromMessageId); + Assert.Null(result.HardDeletedThroughMessageId); + Assert.Null(result.EarliestMessageId); + Assert.Null(result.TailMessageId); + + var remaining = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Empty(remaining); + } + + [SkippableFact] + public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 3; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + // No checkpoint is ever established, so these messages are ahead of the (null) checkpoint; + // only the hard retention ceiling can force their removal, and that removal must be + // reported distinctly from a normal, checkpoint-driven cleanup. + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + + var result = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: 120, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(result.Ran); + Assert.Equal(3, result.DeletedCount); + Assert.Equal(3, result.HardDeletedCount); + Assert.Equal(acks[0].MessageId, result.HardDeletedFromMessageId); + Assert.Equal(acks[^1].MessageId, result.HardDeletedThroughMessageId); + Assert.Null(result.Checkpoint); + } + + [SkippableFact] + public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 5; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); + + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + + var first = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(first.Ran); + Assert.Equal(2, first.DeletedCount); + + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + var second = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(second.Ran); + Assert.Equal(2, second.DeletedCount); + + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + var third = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(third.Ran); + Assert.Equal(1, third.DeletedCount); // the final, partial batch + + var remaining = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Empty(remaining); + } + + [SkippableFact] + public async Task CleanupStreamMessages_ThrottlesRepeatedRunsWithinInterval() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + await AppendAsync(serviceId, providerId, queueId); + + var first = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 60, cleanupBatchSize: 100); + Assert.True(first.Ran); + + // Immediately repeating the call must be throttled by CleanupInterval and not run again. + var second = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 60, cleanupBatchSize: 100); + Assert.False(second.Ran); + Assert.Equal(0, second.DeletedCount); + } + + [SkippableFact] + public async Task CleanupStreamMessages_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 0, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 1, cleanupBatchSize: 1)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 0, cleanupBatchSize: 1)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 1, cleanupBatchSize: 0)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 10, maximumRetentionPeriodSeconds: 5, cleanupIntervalSeconds: 1, cleanupBatchSize: 1)); + + // Valid boundaries must NOT throw: retentionPeriodSeconds/cleanupIntervalSeconds/cleanupBatchSize + // of exactly 1 (the smallest legal values), and a maximumRetentionPeriodSeconds exactly equal to + // retentionPeriodSeconds (the ceiling may legitimately coincide with the normal retention period). + await AppendAsync(serviceId, providerId, queueId); + var result = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: 1, cleanupIntervalSeconds: 1, cleanupBatchSize: 1); + Assert.True(result.Ran); + } + + #endregion Cleanup + + #region Explicit schema mismatch + + /// + /// An old or partially-migrated schema that is missing partition-log query keys must fail + /// explicitly and name the missing keys, rather than fail lazily or silently on first use. + /// + [SkippableFact] + public async Task CreateInstance_MissingPartitionLogQueryKeys_FailsExplicitly() + { + await _storage.ExecuteAsync( + "DELETE FROM OrleansQuery WHERE QueryKey IN ('AppendStreamMessageKey', 'StreamSchemaVersionKey')", + command => { }); + + var exception = await Assert.ThrowsAsync(() => + RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); + + Assert.Contains("AppendStreamMessageKey", exception.Message, StringComparison.Ordinal); + Assert.Contains("StreamSchemaVersionKey", exception.Message, StringComparison.Ordinal); + } + + [SkippableFact] + public async Task CreateInstance_MixedLegacyAndPartitionLogQueryKeys_FailsExplicitly() + { + await _storage.ExecuteAsync( + "INSERT INTO OrleansQuery (QueryKey, QueryText) VALUES ('QueueStreamMessageKey', 'legacy')", + command => { }); + + var exception = await Assert.ThrowsAsync(() => + RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); + + Assert.Contains("QueueStreamMessageKey", exception.Message, StringComparison.Ordinal); + Assert.Contains("no in-place migration", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + #endregion Explicit schema mismatch + + #region SQL Server: raw-connection concurrency and rollback semantics + + /// + /// Holds an append transaction open on one connection, proves a concurrent append on the same + /// partition cannot proceed while it is in-flight (serialization/ordering), then rolls the first + /// transaction back and proves the allocated identifier was not burned: the next successful + /// append reuses it and no message row was left behind. + /// + protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoesNotBurnIds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var firstConnection = new SqlConnection(_storage.ConnectionString); + await firstConnection.OpenAsync(); + await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(); + await using var firstCommand = CreateAppendCommand(firstConnection, firstTransaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + var firstMessageId = await ReadMessageId(firstCommand); + Assert.Equal(1, firstMessageId); + + // A concurrent append on the same partition must not be able to proceed while the first + // transaction still holds the partition-row lock. + await using (var secondConnection = new SqlConnection(_storage.ConnectionString)) + { + await secondConnection.OpenAsync(); + await using var secondCommand = secondConnection.CreateCommand(); + secondCommand.CommandType = CommandType.Text; + secondCommand.CommandText = "SET LOCK_TIMEOUT 0; EXECUTE AppendStreamMessage @ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload;"; + secondCommand.Parameters.AddWithValue("ServiceId", serviceId); + secondCommand.Parameters.AddWithValue("ProviderId", providerId); + secondCommand.Parameters.AddWithValue("QueueId", queueId); + secondCommand.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + secondCommand.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + secondCommand.Parameters.AddWithValue("Payload", payload); + + var exception = await Assert.ThrowsAsync(() => secondCommand.ExecuteReaderAsync()); + Assert.Equal(1222, exception.Number); // "Lock request time out period exceeded." + } + + // Rolling back must not burn the allocated identifier: it is reused by the next append, + // and no message row for it was left behind. + await firstTransaction.RollbackAsync(); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + + var rows = await _storage.ReadAsync("SELECT * FROM OrleansStreamMessage"); + var stored = Assert.Single(rows); + Assert.Equal(1, stored.MessageId); + } + + /// + /// While an append transaction is in-flight and uncommitted, a reader must observe only + /// previously-committed messages; once the transaction commits, the message becomes visible. + /// + protected async Task VerifySqlServerReadExcludesUncommittedInFlightAppend() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var committed = await AppendAsync(serviceId, providerId, queueId); + + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new SqlConnection(_storage.ConnectionString); + await connection.OpenAsync(); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(); + await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + var inFlightMessageId = await ReadMessageId(command); + Assert.Equal(committed.MessageId + 1, inFlightMessageId); + + var duringAppend = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Equal([committed.MessageId], duringAppend.Select(m => m.MessageId)); + + await transaction.CommitAsync(); + + var afterCommit = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Equal([committed.MessageId, inFlightMessageId], afterCommit.Select(m => m.MessageId)); + } + + protected async Task VerifySqlServerPartitionsAreIndependent() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var firstQueueId = RandomQueueId(); + var secondQueueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new SqlConnection(_storage.ConnectionString); + await connection.OpenAsync(); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(); + await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, firstQueueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, await ReadMessageId(command)); + + var independent = await _queries.AppendStreamMessageAsync(serviceId, providerId, secondQueueId, streamIdBytes, nsLength, payload) + .WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, independent.MessageId); + + await transaction.RollbackAsync(); + } + + protected async Task VerifyMySqlRollbackRestoresAllocation() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new MySqlConnection(_storage.ConnectionString); + await connection.OpenAsync(); + await using var transaction = await connection.BeginTransactionAsync(); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "CALL AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload, FALSE)"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + command.Parameters.AddWithValue("Payload", payload); + Assert.Equal(1, await ReadMessageId(command)); + } + + await transaction.RollbackAsync(); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + } + + protected async Task VerifyPostgreSqlRollbackRestoresAllocation() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new NpgsqlConnection(_storage.ConnectionString); + await connection.OpenAsync(); + await using var transaction = await connection.BeginTransactionAsync(); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "SELECT * FROM AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload)"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + command.Parameters.AddWithValue("Payload", payload); + Assert.Equal(1, await ReadMessageId(command)); + } + + await transaction.RollbackAsync(); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + } + + private static SqlCommand CreateAppendCommand( + SqlConnection connection, + SqlTransaction? transaction, + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) + { + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "AppendStreamMessage"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", streamNamespaceLength); + command.Parameters.AddWithValue("Payload", payload); + return command; + } + + private static async Task ReadMessageId(DbCommand command) + { + await using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + return reader.GetInt64(reader.GetOrdinal(nameof(AdoNetStreamMessage.MessageId))); + } + + #endregion SQL Server +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs new file mode 100644 index 00000000000..8e50ec736b7 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -0,0 +1,75 @@ +namespace Tester.AdoNet.Streaming; + +[TestCategory("AdoNet"), TestCategory("Streaming"), TestCategory("BVT")] +[TestProvider("None")] +[TestSuite("BVT")] +[TestArea("Streaming")] +public sealed class AdoNetStreamSchemaTests +{ + [Theory] + [InlineData("SQLServer")] + [InlineData("PostgreSQL")] + [InlineData("MySQL")] + public void SchemaDefinesVersionedImmutablePartitionLog(string provider) + { + var script = ReadScript(provider); + + Assert.Contains("CREATE TABLE OrleansStreamPartition", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId BIGINT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("Checkpoint", script, StringComparison.Ordinal); + Assert.Contains("OwnerEpoch BIGINT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("CleanupOn", script, StringComparison.Ordinal); + + Assert.Contains("CREATE TABLE OrleansStreamMessage", script, StringComparison.Ordinal); + Assert.Contains("StreamIdBytes", script, StringComparison.Ordinal); + Assert.Contains("StreamNamespaceLength INT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("Payload", script, StringComparison.Ordinal); + + Assert.Contains("'StreamSchemaVersionKey', '2'", script, StringComparison.Ordinal); + Assert.Contains("'AppendStreamMessageKey'", script, StringComparison.Ordinal); + Assert.Contains("'AcquireStreamPartitionKey'", script, StringComparison.Ordinal); + Assert.Contains("'ReadStreamMessagesKey'", script, StringComparison.Ordinal); + Assert.Contains("'AdvanceStreamCheckpointKey'", script, StringComparison.Ordinal); + Assert.Contains("'GetStreamPartitionBoundsKey'", script, StringComparison.Ordinal); + Assert.Contains("'CleanupStreamMessagesKey'", script, StringComparison.Ordinal); + + Assert.DoesNotContain("CREATE TABLE OrleansStreamDeadLetter", script, StringComparison.Ordinal); + Assert.DoesNotContain("CREATE TABLE OrleansStreamControl", script, StringComparison.Ordinal); + Assert.Contains("no in-place migration", script, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void SqlServerAppendUsesUpdateOutputWhileHoldingPartitionLock() + { + var script = ReadScript("SQLServer"); + + Assert.Contains("UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK)", script, StringComparison.Ordinal); + Assert.Contains("OUTPUT Inserted.NextMessageId - 1", script, StringComparison.Ordinal); + Assert.Contains("@LockOwner = 'Transaction'", script, StringComparison.Ordinal); + } + + [Fact] + public void PostgreSqlAppendUsesUpdateReturning() + { + var script = ReadScript("PostgreSQL"); + + Assert.Contains("UPDATE OrleansStreamPartition AS P", script, StringComparison.Ordinal); + Assert.Contains("RETURNING P.NextMessageId - 1 INTO _MessageId", script, StringComparison.Ordinal); + Assert.Contains("FOR UPDATE", script, StringComparison.Ordinal); + } + + [Fact] + public void MySqlAppendUsesSelectForUpdateThenUpdate() + { + var script = ReadScript("MySQL"); + + Assert.Contains("FOR UPDATE", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId = _MessageId + 1", script, StringComparison.Ordinal); + Assert.Contains("IN _ManageTransaction BOOLEAN", script, StringComparison.Ordinal); + Assert.Contains("@Payload, TRUE)", script, StringComparison.Ordinal); + Assert.DoesNotContain("@@session.in_transaction", script, StringComparison.Ordinal); + } + + private static string ReadScript(string provider) => + File.ReadAllText(Path.Combine(AppContext.BaseDirectory, $"{provider}-Streaming.sql")); +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs deleted file mode 100644 index 388469258de..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs +++ /dev/null @@ -1,1230 +0,0 @@ -using System.Collections.Concurrent; -using System.Data; -using Microsoft.Data.SqlClient; -using MySql.Data.MySqlClient; -using Npgsql; -using Orleans.Configuration; -using Orleans.Streaming.AdoNet; -using Orleans.Streaming.AdoNet.Storage; -using UnitTests.General; -using static System.String; - -namespace Tester.AdoNet.Streaming; - -/// -/// Tests the relational storage layer via against Sql Server. -/// -[TestCategory("SqlServer"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerRelationalOrleansQueriesTests() : RelationalOrleansQueriesTests(AdoNetInvariants.InvariantNameSqlServer, 90) -{ - [Fact] - public Task RelationalOrleansQueries_SerializesQueueMessageCommits() => - VerifySqlServerQueueMessageCommitsAreSerialized(TestContext.Current.CancellationToken); -} - -/// -/// Tests the relational storage layer via against MySQL. -/// -[TestCategory("MySql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlRelationalOrleansQueriesTests : RelationalOrleansQueriesTests -{ - public MySqlRelationalOrleansQueriesTests() : base(AdoNetInvariants.InvariantNameMySql, 20) - { - MySqlConnection.ClearAllPools(); - } - - [Fact] - public Task RelationalOrleansQueries_OrdersProviderResults() => - VerifyProviderResultsAreOrdered(TestContext.Current.CancellationToken); -} - -/// -/// Tests the relational storage layer via against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlRelationalOrleansQueriesTests : RelationalOrleansQueriesTests -{ - public PostgreSqlRelationalOrleansQueriesTests() : base(AdoNetInvariants.InvariantNamePostgreSql, 99) - { - NpgsqlConnection.ClearAllPools(); - } -} - -/// -/// Tests the relational storage layer via . -/// -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class RelationalOrleansQueriesTests(string invariant, int concurrency = 100) : IAsyncLifetime -{ - private const string TestDatabaseName = "OrleansStreamTest"; - - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - public async ValueTask InitializeAsync() - { - var testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - Assert.SkipWhen(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - - _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); - } - - private static string RandomServiceId(int max = 10) => $"ServiceId{Random.Shared.Next(max)}"; - - private static string RandomProviderId(int max = 10) => $"ProviderId{Random.Shared.Next(max)}"; - - private static string RandomQueueId(int max = 10) => $"QueueId{Random.Shared.Next(max)}"; - - private static int RandomExpiryTimeout(int max = 100) => Random.Shared.Next(max); - - private static byte[] RandomPayload(int size = 1_000_000) - { - var payload = new byte[size]; - Random.Shared.NextBytes(payload); - return payload; - } - - private async Task QueueMessagesAsync( - string serviceId, - string providerId, - string queueId, - byte[] payload, - int expiryTimeout, - int count, - CancellationToken cancellationToken) - { - using var semaphore = new SemaphoreSlim(concurrency); - return await Task.WhenAll(Enumerable.Range(0, count).Select(async _ => - { - return await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout), - cancellationToken); - })); - } - - private static async Task ExecuteProviderOperationAsync( - SemaphoreSlim semaphore, - Func> operationFactory, - CancellationToken cancellationToken) - { - await semaphore.WaitAsync(cancellationToken); - Task? operation = null; - try - { - cancellationToken.ThrowIfCancellationRequested(); - operation = operationFactory(); - return await operation.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - if (operation is not null) - { - if (operation.IsCompleted) - { - await ObserveLateCompletionAsync(operation); - } - else - { - _ = ObserveLateCompletionAsync(operation); - } - } - - throw; - } - finally - { - semaphore.Release(); - } - } - - private static async Task ObserveLateCompletionAsync(Task operation) - { - try - { - await operation; - } - catch - { - } - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - - protected async Task VerifyProviderResultsAreOrdered(CancellationToken cancellationToken) - { - const string reverseQuery = """ - SELECT ServiceId, ProviderId, QueueId, MessageId, Dequeued, VisibleOn, ExpiresOn, CreatedOn, ModifiedOn, Payload - FROM OrleansStreamMessage - WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId - ORDER BY MessageId DESC - LIMIT @MaxCount - """; - - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - - await QueueMessagesAsync(serviceId, providerId, queueId, payload, 100, 100, cancellationToken); - - await _storage.ExecuteAsync( - "UPDATE OrleansQuery SET QueryText = @QueryText WHERE QueryKey = 'GetStreamMessagesKey'", - command => - { - var parameter = command.CreateParameter(); - parameter.ParameterName = "QueryText"; - parameter.Value = reverseQuery; - command.Parameters.Add(parameter); - }, - cancellationToken: cancellationToken); - - var queries = await RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString); - var messages = await queries.GetStreamMessagesAsync(serviceId, providerId, queueId, 100, 3, 10, 100, 10, 1000); - - Assert.Equal(messages.OrderBy(message => message.MessageId), messages); - } - - protected async Task VerifySqlServerQueueMessageCommitsAreSerialized(CancellationToken cancellationToken) - { - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - const int expiryTimeout = 100; - - await using var firstConnection = new SqlConnection(_storage.ConnectionString); - await firstConnection.OpenAsync(cancellationToken); - await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(cancellationToken); - await using var firstCommand = CreateQueueCommand(firstConnection, firstTransaction, serviceId, providerId, queueId, payload, expiryTimeout); - var firstMessageId = await ReadMessageId(firstCommand, cancellationToken); - - await using (var secondConnection = new SqlConnection(_storage.ConnectionString)) - { - await secondConnection.OpenAsync(cancellationToken); - await using var secondCommand = CreateQueueCommand(secondConnection, transaction: null, serviceId, providerId, queueId, payload, expiryTimeout); - secondCommand.CommandType = CommandType.Text; - secondCommand.CommandText = "SET LOCK_TIMEOUT 0; EXECUTE QueueStreamMessage @ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout;"; - - var exception = await Assert.ThrowsAsync(() => secondCommand.ExecuteReaderAsync(cancellationToken)); - Assert.Equal(51000, exception.Number); - } - - await firstTransaction.CommitAsync(cancellationToken); - - var secondAck = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, 2, 3, 10, 100, 10, 1000); - - Assert.Equal([firstMessageId, secondAck.MessageId], messages.Select(message => message.MessageId)); - } - - private static SqlCommand CreateQueueCommand( - SqlConnection connection, - SqlTransaction? transaction, - string serviceId, - string providerId, - string queueId, - byte[] payload, - int expiryTimeout) - { - var command = connection.CreateCommand(); - command.Transaction = transaction; - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "QueueStreamMessage"; - command.Parameters.AddWithValue("ServiceId", serviceId); - command.Parameters.AddWithValue("ProviderId", providerId); - command.Parameters.AddWithValue("QueueId", queueId); - command.Parameters.AddWithValue("Payload", payload); - command.Parameters.AddWithValue("ExpiryTimeout", expiryTimeout); - return command; - } - - private static async Task ReadMessageId(SqlCommand command, CancellationToken cancellationToken) - { - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - Assert.True(await reader.ReadAsync(cancellationToken)); - return reader.GetInt64(reader.GetOrdinal(nameof(AdoNetStreamMessage.MessageId))); - } - - /// - /// Tests that a single message is queued. - /// - [Fact] - public async Task RelationalOrleansQueries_QueuesMessage() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var expiryTimeout = RandomExpiryTimeout(); - var payload = RandomPayload(); - - // act - var before = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - ack - Assert.NotNull(ack); - Assert.Equal(serviceId, ack.ServiceId); - Assert.Equal(providerId, ack.ProviderId); - Assert.Equal(queueId, ack.QueueId); - Assert.Equal(1, ack.MessageId); - - // assert - storage - var messages = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - var message = Assert.Single(messages); - Assert.Equal(serviceId, message.ServiceId); - Assert.Equal(providerId, message.ProviderId); - Assert.Equal(queueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(0, message.Dequeued); - Assert.True(message.VisibleOn >= before); - Assert.True(message.VisibleOn <= after); - Assert.True(message.ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(message.VisibleOn, message.CreatedOn); - Assert.Equal(message.VisibleOn, message.ModifiedOn); - Assert.Equal(payload, message.Payload); - } - - /// - /// Tests that many messages are queued in parallel on the same queue. - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_QueuesManyMessagesInParallel() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var expiryTimeout = RandomExpiryTimeout(); - var payload = RandomPayload(1000); - var count = 10000; - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - var before = DateTime.UtcNow.AddSeconds(-1); - var acks = await Task.WhenAll(Enumerable - .Range(0, count) - .Select(i => Task.Run(async () => - { - return await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout), - cancellationToken); - }, cancellationToken)) - .ToList()); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - messages were inserted in sequence. - var ordered = acks - .OrderBy(x => x.ServiceId) - .ThenBy(x => x.ProviderId) - .ThenBy(x => x.QueueId) - .ThenBy(x => x.MessageId) - .ToList(); - var messageId = ordered[0].MessageId; - for (var i = 0; i < count; i++) - { - Assert.Equal(serviceId, ordered[i].ServiceId); - Assert.Equal(providerId, ordered[i].ProviderId); - Assert.Equal(queueId, ordered[i].QueueId); - Assert.Equal(messageId++, ordered[i].MessageId); - } - - // assert - messages were stored as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken)) - .OrderBy(x => x.ServiceId) - .ThenBy(x => x.ProviderId) - .ThenBy(x => x.QueueId) - .ThenBy(x => x.MessageId) - .ToList(); - for (var i = 0; i < count; i++) - { - Assert.Equal(ordered[i].ServiceId, stored[i].ServiceId); - Assert.Equal(ordered[i].ProviderId, stored[i].ProviderId); - Assert.Equal(ordered[i].QueueId, stored[i].QueueId); - Assert.Equal(ordered[i].MessageId, stored[i].MessageId); - Assert.Equal(0, stored[i].Dequeued); - Assert.True(stored[i].VisibleOn >= before); - Assert.True(stored[i].VisibleOn <= after); - Assert.True(stored[i].ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(stored[i].ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(stored[i].VisibleOn, stored[i].CreatedOn); - Assert.Equal(stored[i].VisibleOn, stored[i].ModifiedOn); - Assert.Equal(payload, stored[i].Payload); - } - } - - /// - /// Tests that many messages are queued in parallel on many queues. - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_QueuesManyMessagesInParallelOnManyQueues() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - create up to 27 random partition keys with around 1000 random messages per partition in random order - var expiryTimeout = RandomExpiryTimeout(); - var count = 3 * 3 * 3 * 1000; - var partitions = Enumerable - .Range(0, count) - .Select(i => - ( - ServiceId: RandomServiceId(3), - ProviderId: RandomProviderId(3), - QueueId: RandomQueueId(3), - Payload: RandomPayload(1000) - )) - .ToList(); - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - queue the random messages in parallel - var before = DateTime.UtcNow.AddSeconds(-1); - var results = await Task.WhenAll(partitions - .Select(p => Task.Run(async () => - { - var ack = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(p.ServiceId, p.ProviderId, p.QueueId, p.Payload, expiryTimeout), - cancellationToken); - return (Partition: p, Ack: ack); - }, cancellationToken)) - .ToList()); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - all messages were acknowledged - var messageIds = new SortedSet(); - foreach (var (partition, ack) in results) - { - Assert.Equal(ack.ServiceId, partition.ServiceId); - Assert.Equal(ack.ProviderId, partition.ProviderId); - Assert.Equal(ack.QueueId, partition.QueueId); - Assert.True(messageIds.Add(ack.MessageId), $"Duplicate {ack.MessageId}"); - } - - // assert - generated message ids are consistent - Assert.Equal(count, messageIds.Count); - Assert.Equal(1, messageIds.Min); - Assert.Equal(messageIds.Count, messageIds.Max); - - // assert - messages were stored as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken)) - .ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - - foreach (var (partition, ack) in results) - { - Assert.True(stored.TryGetValue((ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId), out var message), $"Message not found in storage"); - - Assert.Equal(0, message.Dequeued); - Assert.True(message.VisibleOn >= before); - Assert.True(message.VisibleOn <= after); - Assert.True(message.ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(message.VisibleOn, message.CreatedOn); - Assert.Equal(message.VisibleOn, message.ModifiedOn); - Assert.Equal(partition.Payload, message.Payload); - - stored.Remove((ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId)); - } - } - - /// - /// Tests that a single message is dequeued correctly. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesSingleMessage() - { - // arrange - await _storage.ExecuteAsync("DELETE FROM OrleansStreamMessage", TestContext.Current.CancellationToken); - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue a message - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue a message - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var message = Assert.Single(await _queries.GetStreamMessagesAsync( - serviceId, - providerId, - queueId, - maxCount, - maxAttempts, - visibilityTimeout, - removalTimeout, - evictionInterval, - evictionBatchSize)); - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - the message is the same - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - - // assert - the stored message changed - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(message.ServiceId, stored.ServiceId); - Assert.Equal(message.ProviderId, stored.ProviderId); - Assert.Equal(message.QueueId, stored.QueueId); - Assert.Equal(message.MessageId, stored.MessageId); - Assert.Equal(message.Dequeued, stored.Dequeued); - Assert.Equal(message.VisibleOn, stored.VisibleOn); - Assert.Equal(message.ExpiresOn, stored.ExpiresOn); - Assert.Equal(message.CreatedOn, stored.CreatedOn); - Assert.Equal(message.ModifiedOn, stored.ModifiedOn); - Assert.Equal(message.Payload, stored.Payload); - } - - /// - /// Tests that messages are dequeued in a batch. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesMessageBatches() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - var expiryTimeout = 100; - var maxCount = 60; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - var total = 100; - - // arrange - enqueue messages concurrently - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var acks = await QueueMessagesAsync( - serviceId, - providerId, - queueId, - payload, - expiryTimeout, - total, - TestContext.Current.CancellationToken); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue all messages - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var first = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var second = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var third = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - batch counts - Assert.Equal(maxCount, first.Count); - Assert.Equal(total - maxCount, second.Count); - Assert.Empty(third); - - var messages = first.Concat(second).Concat(third).ToList(); - Assert.Equal(messages.OrderBy(message => message.MessageId), messages); - - // assert - dequeued messages are consistent with acks - var ackLookup = acks.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - foreach (var message in messages) - { - Assert.True(ackLookup.TryGetValue((message.ServiceId, message.ProviderId, message.QueueId, message.MessageId), out var ack), "Ack not found"); - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - - ackLookup.Remove((message.ServiceId, message.ProviderId, message.QueueId, message.MessageId)); - } - - // assert - stored messages are consistent with dequeued messages - var messageLookup = messages.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - foreach (var item in stored) - { - Assert.True(messageLookup.TryGetValue((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId), out var message), "Message not found"); - - Assert.Equal(message.ServiceId, item.ServiceId); - Assert.Equal(message.ProviderId, item.ProviderId); - Assert.Equal(message.QueueId, item.QueueId); - Assert.Equal(message.MessageId, item.MessageId); - Assert.Equal(message.Dequeued, item.Dequeued); - Assert.Equal(message.VisibleOn, item.VisibleOn); - Assert.Equal(message.ExpiresOn, item.ExpiresOn); - Assert.Equal(message.CreatedOn, item.CreatedOn); - Assert.Equal(message.ModifiedOn, item.ModifiedOn); - Assert.Equal(message.Payload, item.Payload); - } - } - - /// - /// Tests that a single message is re-dequeued after visibility timeout until max attempts. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesSingleMessageAgainAfterVisibilityTimeout() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 3; - var visibilityTimeout = 0; - var removalTimeout = 100; - var evictionInterval = 100; - var evictionBatchSize = 0; - - // arrange - enqueue a message - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue messages until max attempts plus one - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var results = new List>(); - for (var i = 0; i < maxAttempts + 1; i++) - { - results.Add(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - } - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - batches are as expected - for (var i = 0; i < maxAttempts; i++) - { - var message = Assert.Single(results[i]); - - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(i + 1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - } - - // assert - final batch is empty - Assert.Empty(results[maxAttempts]); - - // assert - final stored message is consistent with final dequeued message - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - var final = Assert.Single(results[maxAttempts - 1]); - Assert.Equal(final.ServiceId, stored.ServiceId); - Assert.Equal(final.ProviderId, stored.ProviderId); - Assert.Equal(final.QueueId, stored.QueueId); - Assert.Equal(final.MessageId, stored.MessageId); - Assert.Equal(final.Dequeued, stored.Dequeued); - Assert.Equal(final.VisibleOn, stored.VisibleOn); - Assert.Equal(final.ExpiresOn, stored.ExpiresOn); - Assert.Equal(final.CreatedOn, stored.CreatedOn); - Assert.Equal(final.ModifiedOn, stored.ModifiedOn); - Assert.Equal(final.Payload, stored.Payload); - } - - /// - /// Tests that a single message is not dequeued again before the visibility timeout. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotDequeueSingleMessageBeforeVisibilityTimeout() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 3; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue a message - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - - // act - dequeue messages - var first = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - var second = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // assert - first dequeued message is consistent with ack - Assert.Equal(ack.ServiceId, first.ServiceId); - Assert.Equal(ack.ProviderId, first.ProviderId); - Assert.Equal(ack.QueueId, first.QueueId); - Assert.Equal(ack.MessageId, first.MessageId); - - // assert - stored message is consistent with first message - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(first.ServiceId, stored.ServiceId); - Assert.Equal(first.ProviderId, stored.ProviderId); - Assert.Equal(first.QueueId, stored.QueueId); - Assert.Equal(first.MessageId, stored.MessageId); - Assert.Equal(first.Dequeued, stored.Dequeued); - Assert.Equal(first.VisibleOn, stored.VisibleOn); - Assert.Equal(first.ExpiresOn, stored.ExpiresOn); - Assert.Equal(first.CreatedOn, stored.CreatedOn); - Assert.Equal(first.ModifiedOn, stored.ModifiedOn); - Assert.Equal(first.Payload, stored.Payload); - - // assert - message not dequeued again - Assert.Empty(second); - } - - /// - /// Tests that a message can be released for immediate redelivery using its dequeue receipt. - /// - [Fact] - public async Task RelationalOrleansQueries_ReleasesMessageUsingDequeueReceipt() - { - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 5; - var visibilityTimeout = 100; - var removalTimeout = 100; - var evictionInterval = 100; - var evictionBatchSize = 0; - - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var first = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - var firstReceipt = new AdoNetStreamConfirmation(first.MessageId, first.Dequeued); - - var released = Assert.Single(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [firstReceipt])); - Assert.Equal(ack.MessageId, released.MessageId); - - var second = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - Assert.Equal(first.Dequeued + 1, second.Dequeued); - - Assert.Empty(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [firstReceipt])); - Assert.Empty(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - - var secondReceipt = new AdoNetStreamConfirmation(second.MessageId, second.Dequeued); - Assert.Single(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [secondReceipt])); - var third = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, [new AdoNetStreamConfirmation(third.MessageId, third.Dequeued)]); - } - - /// - /// Tests that a single message is not dequeued again after expiry - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotDequeueSingleMessageAfterExpiry() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 0; - var maxCount = 3; - var maxAttempts = 3; - var visibilityTimeout = 0; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 0; - - // arrange - enqueue a message - var before = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var after = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // assert - no messages dequeued - Assert.Empty(messages); - - // assert - stored message are as expected - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(ack.ServiceId, stored.ServiceId); - Assert.Equal(ack.ProviderId, stored.ProviderId); - Assert.Equal(ack.QueueId, stored.QueueId); - Assert.Equal(ack.MessageId, stored.MessageId); - Assert.Equal(0, stored.Dequeued); - Assert.True(stored.VisibleOn >= before); - Assert.True(stored.VisibleOn <= after); - Assert.True(stored.ExpiresOn >= before); - Assert.True(stored.ExpiresOn <= after); - Assert.True(stored.CreatedOn >= before); - Assert.True(stored.CreatedOn <= after); - Assert.True(stored.ModifiedOn >= before); - Assert.True(stored.ModifiedOn <= after); - Assert.Equal(payload, stored.Payload); - } - - /// - /// Tests that messages can be confirmed. - /// - [Fact] - public async Task RelationalOrleansQueries_ConfirmsMessages() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm all messages - var items = messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList(); - var results = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, items); - - // assert - confirmations are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Equal(maxCount, results.Count); - - var lookup = acks.Select(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)).ToHashSet(); - foreach (var result in results) - { - Assert.True(lookup.Remove((result.ServiceId, result.ProviderId, result.QueueId, result.MessageId)), "Unexpected Confirmation"); - } - - // assert - no data remains in storage - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - Assert.Empty(stored); - } - - /// - /// Tests that messages are not confirmed if the receipt is incorrect. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotConfirmMessagesWithWrongReceipt() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm all messages in a faulty way - var faulty = messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued - 1)).ToList(); - var results = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, faulty); - - // assert - confirmations are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Empty(results); - - // assert - data remains in storage - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - Assert.Equal(maxCount, stored.Count()); - } - - /// - /// Chaos tests that some messages can be confirmed while others are not. - /// - [Fact] - public async Task RelationalOrleansQueries_ConfirmsSomeMessagesAndNotOthers() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(1000); - var expiryTimeout = 100; - var maxCount = 100; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - var partial = 30; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all the messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm some of the messages at random - var completed = Randomize(messages).Take(partial).Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList(); - var confirmed = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, completed); - - // assert - counts are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Equal(partial, confirmed.Count); - - // assert - confirmed messages are as expected - var lookup = acks.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)) - .ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - foreach (var item in confirmed) - { - Assert.True(lookup.Remove((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId)), "Unexpected Confirmation"); - Assert.False(stored.TryGetValue((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId), out _), "Message still in storage"); - } - - // assert - unconfirmed messages remain in storage - Assert.Equal(maxCount - partial, stored.Count); - Assert.Equal(lookup.Keys.Order(), stored.Keys.Order()); - } - - /// - /// Chaos tests that queuing, dequeuing, confirmation and eviction work in parallel in a complex random scenario. - /// This looks for concurrent brittleness, especially proneness to database deadlocks, rather than a specific condition. - /// If this test faults due to deadlocks then there is likely some issue with the implementation that needs investigation. - /// - /// - /// At early dev time, this test consistently induced deadlocks until the underlying queries were perfected. - /// This is an expensive test to run but can protect against query regression. - /// For MySQL in particular, this test also detected deadlocks with the driver connection pool itself, which required a package upgrade. - /// See: https://bugs.mysql.com/bug.php?id=114272 - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_ChaosTest() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - generate test data - var total = 10000; - var serviceIds = Enumerable.Range(0, 3).Select(x => $"ServiceId{x}").ToList(); - var providerIds = Enumerable.Range(0, 3).Select(x => $"ProviderId{x}").ToList(); - var queueIds = Enumerable.Range(0, 3).Select(x => $"QueueId{x}").ToList(); - var payload = RandomPayload(1000); - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 1; - var removalTimeout = 1; - var evictionInterval = 1; - var evictionBatchSize = 1000; - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - chaos enqueue, dequeue, confirm - // the tasks below are not expected to result in a planned outcome but are expected to result in a consistent one - var acks = new ConcurrentBag(); - var dequeued1 = new ConcurrentBag(); - var dequeued2 = new ConcurrentBag(); - var confirmed = new ConcurrentBag(); - await Task.WhenAll(Enumerable - .Range(0, total) - .Select(async i => - { - // spin up a random enqueuing task - var enqueue = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var ack = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, visibilityTimeout), - cancellationToken); - - acks.Add(ack); - }, cancellationToken); - - // spin up a random dequeuing task that does not confirm - var dequeue = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var messages = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize), - cancellationToken); - - foreach (var item in messages) - { - dequeued1.Add(item); - } - }, cancellationToken); - - // spin a random dequeuing task that also confirms - var confirm = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var messages = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize), - cancellationToken); - - foreach (var item in messages) - { - dequeued2.Add(item); - } - - var confirmation = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.ConfirmStreamMessagesAsync( - serviceId, - providerId, - queueId, - messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList()), - cancellationToken); - - foreach (var item in confirmation) - { - confirmed.Add(item); - } - }, cancellationToken); - - // wait for all to complete - await Task.WhenAll(enqueue, dequeue, confirm); - }) - .ToList()); - - // assert - all messages were enqueued - Assert.Equal(total, acks.Count); - - // assert - some messages were dequeued (rng dependant, remove assert if flaky) - Assert.NotEmpty(dequeued1); - Assert.NotEmpty(dequeued2); - - // assert - some messages were confirmed (rng dependant, remove assert if flaky) - Assert.NotEmpty(confirmed); - - // assert - some messages were left behind (rng dependant, remove assert if flaky) - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken); - Assert.NotEmpty(stored); - - // assert - confirmed messages were not left behind - Assert.Empty(confirmed.IntersectBy(stored.Select(x => x.MessageId), x => x.MessageId)); - - // assert - confirmed messages all match acks - Assert.Empty(confirmed.ExceptBy(acks.Select(x => x.MessageId), x => x.MessageId)); - } - - /// - /// Tests that a poisoned message can be moved to dead letters. - /// - [Fact] - public async Task RelationalOrleansQueries_MovesPoisonedMessageToDeadLetters() - { - // arrange - var serviceId = "ServiceId"; - var providerId = "ProviderId"; - var streamOptions = new AdoNetStreamOptions(); - var cacheOptions = new SimpleQueueCacheOptions(); - - // arrange - queue an expired message - var queueId = "QueueId"; - var payload = new byte[] { 0xFF }; - - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, cacheOptions.CacheSize, streamOptions.MaxAttempts, 0, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), streamOptions.EvictionInterval.TotalSecondsCeiling(), streamOptions.EvictionBatchSize); - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - - // act - clean up with max attempts of one so the message above is flagged - await _queries.FailStreamMessageAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId, 1, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); - - // assert - message no longer in the message table - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - - // assert - message was moved - var dead = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - Assert.Equal(serviceId, dead.ServiceId); - Assert.Equal(providerId, dead.ProviderId); - Assert.Equal(queueId, dead.QueueId); - Assert.Equal(ack.MessageId, dead.MessageId); - Assert.Equal(1, dead.Dequeued); - Assert.Equal(dead.CreatedOn.Add(streamOptions.ExpiryTimeout.SecondsCeiling()), dead.ExpiresOn); - Assert.Equal(dead.ModifiedOn, dead.VisibleOn); - Assert.Equal(dead.DeadOn.Add(streamOptions.DeadLetterEvictionTimeout.SecondsCeiling()), dead.RemoveOn); - Assert.Equal(payload, dead.Payload); - } - - /// - /// Tests that a healthy message is not moved to dead letters. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotMoveHealthyMessageToDeadLetters() - { - // arrange - var serviceId = "ServiceId"; - var providerId = "ProviderId"; - var queueId = "QueueId"; - var streamOptions = new AdoNetStreamOptions(); - var cacheOptions = new SimpleQueueCacheOptions(); - - // arrange - queue a normal message - var payload = new byte[] { 0xFF }; - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message - await _queries.GetStreamMessagesAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, cacheOptions.CacheSize, streamOptions.MaxAttempts, streamOptions.VisibilityTimeout.TotalSecondsCeiling(), streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), streamOptions.EvictionInterval.TotalSecondsCeiling(), streamOptions.EvictionBatchSize); - - // act - fail the message - await _queries.FailStreamMessageAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId, streamOptions.MaxAttempts, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); - - // assert - the message is still in the table and was made visible again - var saved = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(ack.ServiceId, saved.ServiceId); - Assert.Equal(ack.ProviderId, saved.ProviderId); - Assert.Equal(ack.QueueId, saved.QueueId); - Assert.Equal(ack.MessageId, saved.MessageId); - Assert.Equal(1, saved.Dequeued); - Assert.Equal(saved.ModifiedOn, saved.VisibleOn); - - // assert - no message arrived at dead letters - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - } - - private static List Randomize(IEnumerable source) - { - var list = new List(source.TryGetNonEnumeratedCount(out var count) ? count : 0); - - foreach (var item in source) - { - var index = Random.Shared.Next(list.Count + 1); - if (index == list.Count) - { - list.Add(item); - } - else - { - list.Add(list[index]); - list[index] = item; - } - } - - return list; - } -} From ce9296513cbe211862bac9f5f572850f49f744b3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 11:09:37 -0700 Subject: [PATCH 07/90] feat(streaming): migrate Kinesis to recoverable pipeline Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../streams-implementation/index.md | 2 + .../Streams/KinesisAdapterFactory.cs | 32 ++- .../Streams/KinesisBatchContainer.cs | 30 ++- .../Streams/KinesisPooledAdapterReceiver.cs | 106 ++++++++ .../Streams/KinesisRecoverableStream.cs | 255 ++++++++++++++++++ .../ChronologicalEvictionStrategy.cs | 12 +- .../Common/PooledCache/IEvictionStrategy.cs | 8 + .../IRecoverableStreamSource.cs | 13 + .../RecoverableStreamQueueCache.cs | 75 +++++- .../RecoverableStreamReceiver.cs | 25 +- .../Orleans.Streaming/Orleans.Streaming.cs | 11 +- .../KinesisAdapterTests.cs | 8 +- .../KinesisBatchContainerTests.cs | 37 +++ .../RecoverableStreamReceiverTests.cs | 47 ++++ 14 files changed, 637 insertions(+), 24 deletions(-) create mode 100644 src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs create mode 100644 src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 24f3aa0bfa6..1d7c287b758 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -111,6 +111,8 @@ flowchart TB Cache capacity is not durability. The queue remains the durable boundary, subject to the adapter's acknowledgement contract. +Recoverable partition providers can compose with a partition source and data adapter. The coordinator admits immutable raw records into pooled storage, reconstructs batches lazily, reconciles the earliest subscription delivery watermark, and persists a checkpoint which resumes strictly after the last fully delivered record. + ## Pub-sub handshake The agent registers as a producer for each stream and obtains subscription records from stream pub-sub. It holds a pin cursor while subscription handshakes complete so cache cleanup cannot pass the requested start token. New subscription notifications update the agent's local pub-sub cache. diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 772936973fd..b37da5ad668 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -21,13 +21,13 @@ namespace Orleans.Streaming.Kinesis /// /// Queue adapter factory which allows the PersistentStreamProvider to use AWS Kinesis Data Streams as its backend persistent event queue. /// - internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IDisposable + internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueueAdapterCache, IDisposable { private readonly KinesisStreamOptions _options; private readonly Serializer _serializer; private readonly IStreamQueueCheckpointerFactory? _checkpointerFactory; private readonly ILoggerFactory _loggerFactory; - private readonly IQueueAdapterCache _adapterCache; + private readonly SimpleQueueCacheOptions _cacheOptions; private readonly ILogger _logger; private readonly Func _queueMapperFactory; private readonly IAmazonKinesis _client; @@ -35,6 +35,7 @@ internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IDis private HashRingBasedPartitionedStreamQueueMapper _streamQueueMapper = null!; private KinesisShardTopologyMonitor _topologyMonitor = null!; + private QueueAdapterReceiverRegistry _receivers = null!; private int _disposed; public KinesisAdapterFactory( @@ -53,15 +54,10 @@ public KinesisAdapterFactory( _serializer = serializer; _checkpointerFactory = checkpointerFactory; _loggerFactory = loggerFactory; + _cacheOptions = cacheOptions; _logger = loggerFactory.CreateLogger(); _timeProvider = timeProvider ?? TimeProvider.System; - _adapterCache = new SimpleQueueAdapterCache( - cacheOptions, - name, - loggerFactory - ); - _queueMapperFactory = partitions => new HashRingBasedPartitionedStreamQueueMapper(partitions, Name); _client = CreateClient(); } @@ -106,13 +102,14 @@ public async Task CreateAdapter(CancellationToken cancellationTok _options.TopologyCheckInterval, _timeProvider, _loggerFactory.CreateLogger()); + _receivers = new QueueAdapterReceiverRegistry(MakeReceiver); } return this; } public IQueueAdapterCache GetQueueAdapterCache() - => _adapterCache; + => this; public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; @@ -135,6 +132,12 @@ public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable ev } public IQueueAdapterReceiver CreateReceiver(QueueId queueId) + => GetOrCreateReceiver(queueId); + + public IQueueCache CreateQueueCache(QueueId queueId) + => GetOrCreateReceiver(queueId); + + private KinesisPooledAdapterReceiver GetOrCreateReceiver(QueueId queueId) { if (_checkpointerFactory is null) { @@ -142,13 +145,18 @@ public IQueueAdapterReceiver CreateReceiver(QueueId queueId) $"No {nameof(IStreamQueueCheckpointerFactory)} is configured for the Kinesis stream provider '{Name}'."); } - var partition = _streamQueueMapper.QueueToPartition(queueId); + return _receivers.GetOrCreate(queueId); + } - return new KinesisAdapterReceiver( + private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) + { + var partition = _streamQueueMapper.QueueToPartition(queueId); + return new KinesisPooledAdapterReceiver( CreateClient(), _options.StreamName, partition, - _checkpointerFactory, + _checkpointerFactory!, + _cacheOptions, _serializer, _loggerFactory, _topologyMonitor, diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 4d67dbbc569..99be23cfb43 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -21,6 +21,12 @@ internal class KinesisBatchContainer : IBatchContainer, IComparable Serializer { get; set; } = null!; @@ -37,6 +43,20 @@ private KinesisBatchContainer(Record record, Serializer serializer, + StreamId streamId, + string shardSequence, + long sequenceId) + { + Serializer = serializer; + _rawRecord = rawRecord; + _streamId = streamId; + _hasStreamId = true; + Token = new KinesisSequenceToken(shardSequence, sequenceId, 0); + } + [GeneratedActivatorConstructor] internal KinesisBatchContainer(Serializer serializer) { @@ -46,7 +66,7 @@ internal KinesisBatchContainer(Serializer serializer /// /// Stream identifier for the stream this batch is part of. /// - public StreamId StreamId => GetPayload().StreamId; + public StreamId StreamId => _hasStreamId ? _streamId : GetPayload().StreamId; /// /// Stream Sequence Token for the start of this batch. @@ -113,5 +133,13 @@ internal static KinesisBatchContainer FromKinesisRecord(Serializer serializer, + StreamId streamId, + byte[] rawRecord, + string shardSequence, + long sequenceId) + => new(rawRecord, serializer, streamId, shardSequence, sequenceId); } } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs new file mode 100644 index 00000000000..74d292cdcaf --- /dev/null +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -0,0 +1,106 @@ +using System.Diagnostics.CodeAnalysis; +using Amazon.Kinesis; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; + +namespace Orleans.Streaming.Kinesis; + +internal sealed class KinesisPooledAdapterReceiver : IQueueAdapterReceiver, IQueueCache +{ + private const int BufferSize = 1024 * 1024; + private readonly IStreamQueueCheckpointerFactory _checkpointerFactory; + private readonly string _partition; + private readonly KinesisRecoverableStreamSource _source; + private readonly KinesisRecoverableStreamDataAdapter _dataAdapter; + private readonly RecoverableStreamQueueCache _cache; + private RecoverableStreamReceiver? _inner; + + public KinesisPooledAdapterReceiver( + IAmazonKinesis client, + string streamName, + string partition, + IStreamQueueCheckpointerFactory checkpointerFactory, + SimpleQueueCacheOptions cacheOptions, + Serializer serializer, + ILoggerFactory loggerFactory, + KinesisShardTopologyMonitor topologyMonitor, + TimeSpan getRecordsInterval, + TimeProvider timeProvider) + { + _checkpointerFactory = checkpointerFactory; + _partition = partition; + var logger = loggerFactory.CreateLogger(); + _source = new( + client, + streamName, + partition, + topologyMonitor, + getRecordsInterval, + timeProvider); + _dataAdapter = new(serializer); + var evictionStrategy = new ChronologicalEvictionStrategy( + logger, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null); + _cache = new( + Math.Min(1000, cacheOptions.CacheSize), + new ObjectPool(() => new FixedSizeBuffer(BufferSize)), + _dataAdapter, + evictionStrategy, + logger, + maxCacheSize: cacheOptions.CacheSize); + } + + public async Task Initialize(TimeSpan timeout) + { + using var cancellation = new CancellationTokenSource(timeout); + var checkpointer = await _checkpointerFactory.Create(_partition, cancellation.Token); + _inner = new( + _source, + _dataAdapter, + _cache, + checkpointer, + startFromNow: false); + await _inner.Initialize(timeout); + } + + public Task> GetQueueMessagesAsync(int maxCount) + => GetInner().GetQueueMessagesAsync(maxCount, CancellationToken.None); + + public Task> GetQueueMessagesAsync(int maxCount, CancellationToken cancellationToken) + => GetInner().GetQueueMessagesAsync(maxCount, cancellationToken); + + public Task MessagesDeliveredAsync(IList messages) + => Task.CompletedTask; + + public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; + + public Task Shutdown(TimeSpan timeout) + => _inner is null ? _source.Shutdown(CancellationToken.None) : _inner.Shutdown(timeout); + + public int GetMaxAddCount() => _cache.GetMaxAddCount(); + + public void AddToCache(IList messages) + { + } + + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _cache.TryPurgeFromCache(out purgedItems); + + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.GetCacheCursor(streamId, token); + + public bool IsUnderPressure() => _cache.IsUnderPressure(); + + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => GetInner().UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); + + private RecoverableStreamReceiver GetInner() + => _inner ?? throw new InvalidOperationException("The Kinesis receiver has not been initialized."); +} diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs new file mode 100644 index 00000000000..6bf2da85d4f --- /dev/null +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -0,0 +1,255 @@ +using System.Globalization; +using System.Numerics; +using Amazon.Kinesis; +using Amazon.Kinesis.Model; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; + +namespace Orleans.Streaming.Kinesis; + +internal sealed class KinesisCacheRecord( + Record record, + long sequenceNumber) +{ + public Record Record { get; } = record; + + public long SequenceNumber { get; } = sequenceNumber; + + public KinesisBatchContainer.Body? Body { get; set; } +} + +internal sealed class KinesisRecoverableStreamSource( + IAmazonKinesis client, + string streamName, + string partition, + KinesisShardTopologyMonitor topologyMonitor, + TimeSpan getRecordsInterval, + TimeProvider timeProvider) : IRecoverableStreamSource +{ + private string? _shardIterator; + private string? _readOffset; + private long _nextSequenceNumber; + private DateTimeOffset _nextGetRecordsUtc; + private bool _shardExhausted; + private bool _topologyCheckRequired; + private bool _resetRequired; + + public async Task Initialize( + RecoverableStreamStartPosition position, + CancellationToken cancellationToken) + { + _readOffset = position.Checkpoint; + await ResetShardIterator(cancellationToken); + } + + public async Task> Read( + int maxCount, + CancellationToken cancellationToken) + { + if (_resetRequired) + { + await ResetShardIterator(cancellationToken); + _resetRequired = false; + } + + if (!await topologyMonitor.CheckTopology(_topologyCheckRequired, cancellationToken)) + { + return []; + } + + _topologyCheckRequired = false; + if (_shardExhausted) + { + return []; + } + + await WaitForGetRecordsInterval(cancellationToken); + var request = new GetRecordsRequest + { + Limit = maxCount, + ShardIterator = _shardIterator, + }; + + GetRecordsResponse response; + try + { + response = await client.GetRecordsAsync(request, cancellationToken); + } + catch (ExpiredIteratorException) + { + await ResetShardIterator(cancellationToken); + if (_shardExhausted) + { + return []; + } + + await WaitForGetRecordsInterval(cancellationToken); + request.ShardIterator = _shardIterator; + response = await client.GetRecordsAsync(request, cancellationToken); + } + + _shardIterator = response.NextShardIterator; + if (string.IsNullOrEmpty(_shardIterator)) + { + _shardExhausted = true; + _topologyCheckRequired = true; + } + + if (response.Records is not { Count: > 0 } records) + { + return []; + } + + var result = new List(records.Count); + for (var i = 0; i < records.Count; i++) + { + result.Add(new KinesisCacheRecord(records[i], _nextSequenceNumber + i)); + } + + return result; + } + + public void MessagesAdded(IReadOnlyList messages) + { + if (messages.Count == 0) + { + return; + } + + _nextSequenceNumber += messages.Count; + _readOffset = messages[^1].Record.SequenceNumber; + } + + public void MessagesAddFailed(IReadOnlyList messages) + { + _resetRequired = true; + } + + public Task Shutdown(CancellationToken cancellationToken) + { + client.Dispose(); + return cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + } + + private async Task ResetShardIterator(CancellationToken cancellationToken) + { + var request = new GetShardIteratorRequest + { + StreamName = streamName, + ShardId = partition, + ShardIteratorType = string.IsNullOrEmpty(_readOffset) + ? ShardIteratorType.TRIM_HORIZON + : ShardIteratorType.AFTER_SEQUENCE_NUMBER, + StartingSequenceNumber = _readOffset, + }; + var response = await client.GetShardIteratorAsync(request, cancellationToken); + _shardIterator = response.ShardIterator; + _shardExhausted = string.IsNullOrEmpty(_shardIterator); + } + + private async Task WaitForGetRecordsInterval(CancellationToken cancellationToken) + { + var delay = _nextGetRecordsUtc - timeProvider.GetUtcNow(); + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, timeProvider, cancellationToken); + } + + _nextGetRecordsUtc = timeProvider.GetUtcNow() + getRecordsInterval; + } +} + +internal sealed class KinesisRecoverableStreamDataAdapter( + Serializer serializer) : IRecoverableStreamDataAdapter +{ + public StreamPosition GetStreamPosition(KinesisCacheRecord queueMessage) + { + queueMessage.Body ??= serializer.Deserialize(queueMessage.Record.Data.ToArray())!; + return new( + queueMessage.Body.StreamId, + new KinesisSequenceToken( + queueMessage.Record.SequenceNumber, + queueMessage.SequenceNumber, + 0)); + } + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + KinesisCacheRecord queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + var payload = queueMessage.Record.Data.ToArray(); + var size = SegmentBuilder.CalculateAppendSize(queueMessage.Record.SequenceNumber) + + SegmentBuilder.CalculateAppendSize(payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.Record.SequenceNumber); + SegmentBuilder.Append(segment, ref offset, payload); + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.SequenceNumber, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = queueMessage.Record.ApproximateArrivalTimestamp?.ToUniversalTime() ?? dequeueTimeUtc, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + var payload = SegmentBuilder.ReadNextBytes(cachedMessage.Segment, ref offset).ToArray(); + return KinesisBatchContainer.FromCachedRecord( + serializer, + cachedMessage.StreamId, + payload, + shardSequence, + cachedMessage.SequenceNumber); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + { + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + return new KinesisSequenceToken(shardSequence, cachedMessage.SequenceNumber, cachedMessage.EventIndex); + } + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + { + if (token is not KinesisSequenceToken kinesisToken) + { + throw new ArgumentOutOfRangeException(nameof(token)); + } + + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + var difference = BigInteger.Parse(shardSequence, NumberStyles.None, CultureInfo.InvariantCulture) + .CompareTo(BigInteger.Parse(kinesisToken.ShardSequence, NumberStyles.None, CultureInfo.InvariantCulture)); + return difference != 0 ? difference : cachedMessage.EventIndex.CompareTo(kinesisToken.EventIndex); + } + + public string GetOffset(ref CachedMessage cachedMessage) + { + var offset = 0; + return SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + } + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + if (token is KinesisSequenceToken kinesisToken) + { + offset = kinesisToken.ShardSequence; + return true; + } + + offset = string.Empty; + return false; + } +} diff --git a/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs b/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs index 3571e62acf4..b7141602d9a 100644 --- a/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs +++ b/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs @@ -112,7 +112,17 @@ private void PerformPurgeInternal(DateTime nowUtc) if (itemsPurged == 0) return; - //items got purged, time to conduct follow up actions + OnPurgeCompleted(lastMessagePurged, itemsPurged); + } + + /// + public void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) + { + if (itemsPurged <= 0) + { + return; + } + this.cacheMonitor?.TrackMessagesPurged(itemsPurged); OnPurged?.Invoke(lastMessagePurged, this.PurgeObservable.Newest); FreePurgedBuffers(lastMessagePurged, this.PurgeObservable.Oldest); diff --git a/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs b/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs index 711bdab01e3..f2cad39198c 100644 --- a/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs +++ b/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs @@ -28,6 +28,14 @@ public interface IEvictionStrategy /// /// The new block. void OnBlockAllocated(FixedSizeBuffer newBlock); + + /// + /// Performs follow-up accounting after the owner removes messages directly. + /// + /// The last message removed. + /// The number of messages removed. + void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) + => OnPurged?.Invoke(lastMessagePurged, null); } /// diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs index dbae621b76e..97e4383fb8b 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs @@ -23,6 +23,19 @@ public interface IRecoverableStreamSource /// Task> Read(int maxCount, CancellationToken cancellationToken); + /// + /// Notifies the source that records were successfully admitted to the cache. + /// + /// The admitted records. + /// Sources should advance volatile read offsets only in this callback. + void MessagesAdded(IReadOnlyList messages) { } + + /// + /// Notifies the source that records could not be admitted to the cache. + /// + /// The records which were not admitted. + void MessagesAddFailed(IReadOnlyList messages) { } + /// /// Shuts the partition source down. /// diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 5826e36769d..ee2428a2096 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -18,6 +18,7 @@ public sealed class RecoverableStreamQueueCache : IQueueCache, ID private readonly IRecoverableStreamDataAdapter _dataAdapter; private readonly IEvictionStrategy _evictionStrategy; private readonly IQueueFlowController? _flowController; + private readonly int? _maxCacheSize; private readonly PooledQueueCache _cache; private FixedSizeBuffer? _currentBuffer; @@ -33,7 +34,8 @@ public RecoverableStreamQueueCache( IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, TimeSpan? cacheMonitorWriteInterval = null, - TimeSpan? metadataMinTimeInCache = null) + TimeSpan? metadataMinTimeInCache = null, + int? maxCacheSize = null) { if (defaultMaxAddCount <= 0) { @@ -45,6 +47,12 @@ public RecoverableStreamQueueCache( _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); _evictionStrategy = evictionStrategy ?? throw new ArgumentNullException(nameof(evictionStrategy)); _flowController = flowController; + if (maxCacheSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxCacheSize)); + } + + _maxCacheSize = maxCacheSize; _cache = new PooledQueueCache( dataAdapter, logger ?? throw new ArgumentNullException(nameof(logger)), @@ -60,6 +68,30 @@ public RecoverableStreamQueueCache( /// public string? LastPurgedOffset { get; private set; } + /// + /// Gets the number of records currently held in the cache. + /// + public int ItemCount => _cache.ItemCount; + + /// + /// Tries to get the newest cached provider position. + /// + public bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset) + { + if (_cache.Newest is not { } newest) + { + token = null; + offset = null; + return false; + } + + token = _dataAdapter.GetSequenceToken(ref newest); + offset = _dataAdapter.GetOffset(ref newest); + return true; + } + /// /// Packs and adds ordered source records to the cache. /// @@ -83,9 +115,20 @@ public IReadOnlyList Add( /// public int GetMaxAddCount() - => _flowController is null - ? _defaultMaxAddCount - : Math.Min(_defaultMaxAddCount, _flowController.GetMaxAddCount()); + { + var result = _defaultMaxAddCount; + if (_maxCacheSize is { } maxCacheSize) + { + result = Math.Min(result, Math.Max(0, maxCacheSize - _cache.ItemCount)); + } + + if (_flowController is not null) + { + result = Math.Min(result, _flowController.GetMaxAddCount()); + } + + return result; + } /// public void AddToCache(IList messages) @@ -115,6 +158,27 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? /// public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { + if (earliestSubscriptionToken is null) + { + return; + } + + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest + && _dataAdapter.Compare(ref oldest, earliestSubscriptionToken) <= 0) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref oldest); + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); + } + + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + if (_cache.IsEmpty) + { + _currentBuffer = null; + } } /// @@ -142,7 +206,8 @@ private ArraySegment GetSegment(int size) if (!buffer.TryGetSegment(size, out segment)) { buffer.Dispose(); - throw new ArgumentOutOfRangeException(nameof(size), $"Message size is too big. MessageSize: {size}"); + buffer = new FixedSizeBuffer(size); + _ = buffer.TryGetSegment(size, out segment); } _currentBuffer = buffer; diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 1e49f966c8f..f86928d09b1 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -74,7 +74,18 @@ public async Task> GetQueueMessagesAsync( return []; } - var positions = _cache.Add(messages, DateTime.UtcNow); + IReadOnlyList positions; + try + { + positions = _cache.Add(messages, DateTime.UtcNow); + _source.MessagesAdded(messages); + } + catch + { + _source.MessagesAddFailed(messages); + throw; + } + var result = new List(positions.Count); foreach (var position in positions) { @@ -134,8 +145,16 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? /// public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { - if (earliestSubscriptionToken is not null - && _dataAdapter.TryGetOffset(earliestSubscriptionToken, out var offset)) + var progressToken = earliestSubscriptionToken; + string? offset = null; + if (progressToken is null) + { + _ = _cache.TryGetNewestPosition(out progressToken, out offset); + } + + _cache.UpdateDeliveryProgress(progressToken, utcNow); + if (progressToken is not null + && (offset is not null || _dataAdapter.TryGetOffset(progressToken, out offset))) { _checkpointer.Update(offset, utcNow, CancellationToken.None); } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 36607f8fa60..1329086d40f 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -598,6 +598,8 @@ public IPurgeObservable PurgeObservable { set { } } public void OnBlockAllocated(FixedSizeBuffer newBlock) { } + public void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) { } + public void PerformPurge(System.DateTime nowUtc) { } protected virtual bool ShouldPurge(ref CachedMessage cachedMessage, ref CachedMessage newestCachedMessage, System.DateTime nowUtc) { throw null; } @@ -762,6 +764,7 @@ public partial interface IEvictionStrategy IPurgeObservable PurgeObservable { set; } void OnBlockAllocated(FixedSizeBuffer newBlock); + void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged); void PerformPurge(System.DateTime utcNow); } @@ -811,6 +814,8 @@ public partial interface IRecoverableStreamDataAdapter : ICacheDa public partial interface IRecoverableStreamSource { System.Threading.Tasks.Task Initialize(RecoverableStreamStartPosition position, System.Threading.CancellationToken cancellationToken); + void MessagesAdded(System.Collections.Generic.IReadOnlyList messages); + void MessagesAddFailed(System.Collections.Generic.IReadOnlyList messages); System.Threading.Tasks.Task> Read(int maxCount, System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.Task Shutdown(System.Threading.CancellationToken cancellationToken); } @@ -933,7 +938,9 @@ public ReceiverMonitorDimensions(string queueId) { } public sealed partial class RecoverableStreamQueueCache : Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable { - public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null) { } + public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) { } + + public int ItemCount { get { throw null; } } public string? LastPurgedOffset { get { throw null; } } @@ -949,6 +956,8 @@ public void Dispose() { } public bool IsUnderPressure() { throw null; } + public bool TryGetNewestPosition(out Orleans.Streams.StreamSequenceToken? token, out string? offset) { throw null; } + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs index b46bf53aff9..346fb6b099f 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs @@ -123,8 +123,14 @@ await Task.WhenAll(Enumerable.Range(0, NumBatches) foreach (var (queueId, receiver) in receivers) { var messages = (await receiver.GetQueueMessagesAsync(10, cancellationToken)).ToArray(); - foreach (var message in messages.Cast()) + foreach (var notification in messages) { + using var cursor = caches[queueId].GetCacheCursor( + notification.StreamId, + notification.SequenceToken); + Assert.True(cursor.MoveNext()); + var message = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); output.WriteLine($"Queue {queueId} received message on stream {message.StreamId}"); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents().Count()); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents().Count()); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index 3c78b30c2fa..4085bb6763c 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Orleans.Providers.Streams.Common; using Orleans.Runtime; using Orleans.Serialization; using Orleans.Streaming.Kinesis; @@ -124,4 +125,40 @@ public void CompareToOrdersByDurableShardSequenceNotReceiverLocalOrdinal() Assert.True(readFirstButNewer.CompareTo(readSecondButOlder) > 0); Assert.True(readSecondButOlder.CompareTo(readFirstButNewer) < 0); } + + [Fact] + public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering() + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new[] { "event" }, + requestContext: null); + var record = new KinesisRecord + { + Data = new MemoryStream(payload), + SequenceNumber = "123456789012345678901234567890", + }; + var queueMessage = new KinesisCacheRecord(record, sequenceNumber: 7); + var adapter = new KinesisRecoverableStreamDataAdapter(serializer); + + var position = adapter.GetStreamPosition(queueMessage); + var cached = adapter.FromQueueMessage( + position, + queueMessage, + DateTime.UtcNow, + size => new byte[size]); + + Assert.Equal(streamId, cached.StreamId); + Assert.Equal(record.SequenceNumber, adapter.GetOffset(ref cached)); + Assert.True(adapter.Compare( + ref cached, + new KinesisSequenceToken("123456789012345678901234567889", 1000, 0)) > 0); + + var batch = Assert.IsType(adapter.GetBatchContainer(ref cached)); + Assert.Equal(streamId, batch.StreamId); + Assert.Equal(["event"], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(record.SequenceNumber, ((KinesisSequenceToken)batch.SequenceToken).ShardSequence); + } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index af43dcbafc7..3679f328e6a 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -81,6 +81,41 @@ public void Registry_ReturnsSameReceiverAndCacheInstanceForQueue() Assert.Single(registry.Receivers); } + [Fact] + public async Task DeliveryProgress_WithNoSubscribers_AdvancesToNewestCachedRecord() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource([new TestQueueMessage(streamId, 11, "payload")]); + var adapter = new TestDataAdapter(); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + var checkpointer = new TestCheckpointer("10"); + var receiver = new RecoverableStreamReceiver( + source, + adapter, + cache, + checkpointer, + startFromNow: false); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + _ = await receiver.GetQueueMessagesAsync(100, CancellationToken.None); + + receiver.UpdateDeliveryProgress(earliestSubscriptionToken: null, DateTime.UtcNow); + + Assert.Equal("11", checkpointer.LastUpdatedCheckpoint); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(1, bufferPool.FreeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + private sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); private sealed class TestDataAdapter : IRecoverableStreamDataAdapter @@ -245,6 +280,18 @@ public void OnBlockAllocated(FixedSizeBuffer newBlock) } } + private sealed class TrackingBufferPool : IObjectPool + { + public int FreeCount { get; private set; } + + public FixedSizeBuffer Allocate() => new(4 * 1024) { Pool = this }; + + public void Free(FixedSizeBuffer resource) + { + FreeCount++; + } + } + private sealed class TestCombinedReceiver : IQueueAdapterReceiver, IQueueCache { public Task Initialize(TimeSpan timeout) => Task.CompletedTask; From 5a64aa6a7185cb3b9ccdfc47dc058c8b41b4de36 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 11:09:48 -0700 Subject: [PATCH 08/90] feat(streaming)!: wire ADO.NET retained log Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../docs/streaming/stream-providers.md | 4 +- .../AdoNetQueueAdapter.cs | 28 ++- .../AdoNetQueueAdapterFactory.cs | 17 +- .../AdoNetQueueAdapterReceiver.cs | 98 ++++++--- .../AdoNetRecoverableStream.cs | 206 ++++++++++++++++++ .../AdoNetStreamFailureHandler.cs | 65 ++++-- .../AdoNetStreamOptions.cs | 5 + .../AdoNetStreamOptionsValidator.cs | 16 +- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 3 + .../Orleans.Streaming.AdoNet.cs | 2 + .../AdoNetOptionsValidatorTests.cs | 18 ++ .../Streaming/AdoNetBatchContainerTests.cs | 36 +++ .../AdoNetQueueAdapterFactoryTests.cs | 17 +- .../Streaming/AdoNetQueueAdapterTests.cs | 2 +- 14 files changed, 439 insertions(+), 78 deletions(-) create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 95d289ebda3..3f8f5a573da 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -18,7 +18,7 @@ A stream provider connects the Orleans streaming API to a transport and defines | Azure Event Hubs | [`Microsoft.Orleans.Streaming.EventHubs`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.EventHubs) | Stable | Yes, within Event Hubs retention | Yes | Event Hubs namespace, hub, consumer group, and checkpoint storage | | Amazon Kinesis | [`Microsoft.Orleans.Streaming.Kinesis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Kinesis) | Stable | Yes, within Kinesis retention | Yes | Kinesis data stream, AWS credentials, region, and durable checkpoint storage | | Amazon SQS | [`Microsoft.Orleans.Streaming.SQS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.SQS) | Stable | Yes, within SQS retention | No | AWS account, queue permissions, region/endpoint configuration | -| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in a retained relational partition log | No | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | +| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in a retained relational partition log | Yes, while records remain retained | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | | NATS JetStream | [`Microsoft.Orleans.Streaming.NATS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.NATS) | **Alpha** | Configurable; file storage is the default | No | NATS server with JetStream and sufficient storage; subject/stream administration | | Redis Streams | [`Microsoft.Orleans.Streaming.Redis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Redis) | **Alpha** | Configurable through Redis persistence and stream retention | Yes, while entries remain | Redis deployment, persistence/HA policy, and retention sizing | @@ -70,7 +70,7 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S ## ADO.NET streaming (alpha) -Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue is an immutable, ordered partition log with a durable, ownership-fenced checkpoint. retains checkpointed messages for one day by default. can impose a hard storage ceiling, but it can delete unprocessed messages and therefore produces gap diagnostics. Cleanup is bounded by . +Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue is an immutable, ordered partition log with a durable, ownership-fenced checkpoint. The provider is rewindable while records remain retained and resumes strictly after the checkpoint. retains checkpointed messages for one day by default. can impose a hard storage ceiling, but it can delete unprocessed messages and therefore produces gap diagnostics. Cleanup is bounded by . can fault one failing subscription without deleting or mutating the multicast source record. The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index f01b513eda3..4b8c424e2ef 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -3,9 +3,20 @@ namespace Orleans.Streaming.AdoNet; /// /// Stream queue storage adapter for ADO.NET providers. /// -internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, Serializer serializer, ILogger logger, IServiceProvider serviceProvider) : IQueueAdapter +internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, Serializer serializer, ILogger logger, IServiceProvider serviceProvider) : IQueueAdapter, IQueueAdapterCache { private readonly ILogger _logger = logger; + private readonly QueueAdapterReceiverRegistry _receivers = new( + queueId => ReceiverFactory( + serviceProvider, + [ + name, + mapper.GetAdoNetQueueId(queueId), + streamOptions, + clusterOptions, + cacheOptions, + queries, + ])); /// /// Maps to the ProviderId in the database. @@ -13,9 +24,9 @@ internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions strea public string Name { get; } = name; /// - /// The ADO.NET provider is not yet rewindable. + /// The retained-log ADO.NET provider supports replay from cached and durable checkpoints. /// - public bool IsRewindable => false; + public bool IsRewindable => true; /// /// The ADO.NET provider works both ways. @@ -23,17 +34,14 @@ internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions strea public StreamProviderDirection Direction => StreamProviderDirection.ReadWrite; public IQueueAdapterReceiver CreateReceiver(QueueId queueId) - { - // map the queue id - var adoNetQueueId = mapper.GetAdoNetQueueId(queueId); + => _receivers.GetOrCreate(queueId); - // create the receiver - return ReceiverFactory(serviceProvider, [Name, adoNetQueueId, streamOptions, clusterOptions, cacheOptions, queries]); - } + public IQueueCache CreateQueueCache(QueueId queueId) + => _receivers.GetOrCreate(queueId); public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable events, StreamSequenceToken? token, Dictionary? requestContext) { - // the ADO.NET provider is not rewindable so we do not support user supplied tokens + // Producer-supplied tokens are not supported. Replay positions are consumer-side tokens. if (token is not null) { throw new ArgumentException($"{nameof(AdoNetQueueAdapter)} does not support a user supplied {nameof(StreamSequenceToken)}."); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index f681cc5b7bd..c3d5cb3d76c 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -3,7 +3,7 @@ namespace Orleans.Streaming.AdoNet; -internal class AdoNetQueueAdapterFactory : IQueueAdapterFactory +internal class AdoNetQueueAdapterFactory : IQueueAdapterFactory, IQueueAdapterCache { public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, HashRingStreamQueueMapperOptions hashOptions, ILoggerFactory loggerFactory, IHostApplicationLifetime lifetime, IServiceProvider serviceProvider) { @@ -15,7 +15,6 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, _serviceProvider = serviceProvider; _streamQueueMapper = new HashRingBasedStreamQueueMapper(hashOptions, name); - _cache = new SimpleQueueAdapterCache(cacheOptions, name, loggerFactory); _adoNetQueueMapper = new AdoNetStreamQueueMapper(_streamQueueMapper); } @@ -27,10 +26,10 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, private readonly IServiceProvider _serviceProvider; private readonly HashRingBasedStreamQueueMapper _streamQueueMapper; - private readonly SimpleQueueAdapterCache _cache; private readonly AdoNetStreamQueueMapper _adoNetQueueMapper; private RelationalOrleansQueries? _queries; + private AdoNetQueueAdapter? _adapter; /// /// Unfortunate implementation detail to account for lack of async lifetime. @@ -74,17 +73,23 @@ public async Task CreateAdapter() { var queries = await GetQueriesAsync(); - return AdapterFactory(_serviceProvider, [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries]); + return _adapter ??= (AdoNetQueueAdapter)AdapterFactory( + _serviceProvider, + [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries]); } public async Task GetDeliveryFailureHandler(QueueId queueId) { var queries = await GetQueriesAsync(); - return HandlerFactory(_serviceProvider, [false, _streamOptions, _clusterOptions, _adoNetQueueMapper, queries]); + return HandlerFactory(_serviceProvider, [_streamOptions.FaultOnDeliveryFailure, _streamOptions, _clusterOptions, _adoNetQueueMapper, queries]); } - public IQueueAdapterCache GetQueueAdapterCache() => _cache; + public IQueueAdapterCache GetQueueAdapterCache() => this; + + public IQueueCache CreateQueueCache(QueueId queueId) + => (_adapter ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) + .CreateQueueCache(queueId); public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 0d2245bd84a..99bb9afd9bf 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -1,12 +1,14 @@ +using System.Diagnostics.CodeAnalysis; + namespace Orleans.Streaming.AdoNet; /// -/// Receives message batches from an individual queue of an ADO.NET provider. +/// Receives message batches from an individual retained-log partition. /// -internal class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver +internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueueCache { - private readonly string _providerId; - private readonly string _queueId; + private const int BufferSize = 1024 * 1024; + private readonly RecoverableStreamReceiver _inner; public AdoNetQueueAdapterReceiver( string providerId, @@ -18,34 +20,74 @@ public AdoNetQueueAdapterReceiver( Serializer serializer, ILogger logger) { - _providerId = providerId; - _queueId = queueId; - _ = streamOptions; - _ = clusterOptions; - _ = cacheOptions; - _ = queries; - _ = serializer; - _ = logger; + var source = new AdoNetRecoverableStream( + clusterOptions.ServiceId, + providerId, + queueId, + streamOptions, + queries, + logger); + var checkpointer = new StreamQueueCheckpointer( + source, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = streamOptions.CheckpointPersistInterval, + }); + var dataAdapter = new AdoNetRecoverableStreamDataAdapter(serializer); + var bufferPool = new ObjectPool(() => new FixedSizeBuffer(BufferSize)); + var evictionStrategy = new ChronologicalEvictionStrategy( + logger, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null); + var cache = new RecoverableStreamQueueCache( + Math.Min(streamOptions.MaxMessagesPerRead, cacheOptions.CacheSize), + bufferPool, + dataAdapter, + evictionStrategy, + logger, + maxCacheSize: cacheOptions.CacheSize); + _inner = new RecoverableStreamReceiver( + source, + dataAdapter, + cache, + checkpointer, + streamOptions.StartFromNow); } - /// - /// Receiver integration is supplied by the shared recoverable stream pipeline. - /// - public Task Initialize(TimeSpan timeout) => Task.FromException(CreatePipelineUnavailableException()); + public Task Initialize(TimeSpan timeout) => _inner.Initialize(timeout); + + public Task Shutdown(TimeSpan timeout) => _inner.Shutdown(timeout); + + public Task> GetQueueMessagesAsync(int maxCount) + => _inner.GetQueueMessagesAsync(maxCount, CancellationToken.None); + + public Task> GetQueueMessagesAsync( + int maxCount, + CancellationToken cancellationToken) + => _inner.GetQueueMessagesAsync(maxCount, cancellationToken); + + public Task MessagesDeliveredAsync(IList messages) + => _inner.MessagesDeliveredAsync(messages, CancellationToken.None); + + public Task MessagesDeliveredAsync( + IList messages, + CancellationToken cancellationToken) + => _inner.MessagesDeliveredAsync(messages, cancellationToken); + + public int GetMaxAddCount() => _inner.GetMaxAddCount(); + + public void AddToCache(IList messages) => _inner.AddToCache(messages); - /// - /// No receiver work is started by this storage-only implementation. - /// - public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _inner.TryPurgeFromCache(out purgedItems); - /// - public Task> GetQueueMessagesAsync(int maxCount) => - Task.FromException>(CreatePipelineUnavailableException()); + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _inner.GetCacheCursor(streamId, token); - /// - public Task MessagesDeliveredAsync(IList messages) => - Task.FromException(CreatePipelineUnavailableException()); + public bool IsUnderPressure() => _inner.IsUnderPressure(); - private InvalidOperationException CreatePipelineUnavailableException() => - new($"The ADO.NET retained-log receiver for provider '{_providerId}', queue '{_queueId}' requires the shared recoverable stream cache and receiver pipeline."); + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => _inner.UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs new file mode 100644 index 00000000000..e2f65bb3a62 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -0,0 +1,206 @@ +using System.Globalization; +using Orleans.Providers.Streams.Common; + +namespace Orleans.Streaming.AdoNet; + +internal sealed partial class AdoNetRecoverableStream( + string serviceId, + string providerId, + string queueId, + AdoNetStreamOptions options, + RelationalOrleansQueries queries, + ILogger logger) : IRecoverableStreamSource, IStreamCheckpointStore +{ + private AdoNetStreamPartitionState? _partition; + private long _readOffset; + + public async ValueTask Load(CancellationToken cancellationToken) + { + _partition = await queries.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + options.StartFromNow).WaitAsync(cancellationToken); + _readOffset = _partition.Checkpoint ?? 0; + ThrowIfRetentionGap(_partition); + var checkpoint = _partition.Checkpoint?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + return new(checkpoint, _partition.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + var checkpointValue = long.Parse(checkpoint, NumberStyles.None, CultureInfo.InvariantCulture); + var ownerEpoch = long.Parse(expectedVersion, NumberStyles.None, CultureInfo.InvariantCulture); + var update = await queries.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + ownerEpoch, + checkpointValue).WaitAsync(cancellationToken); + if (update is not null) + { + if (update.OwnerEpoch != ownerEpoch) + { + throw new InvalidOperationException( + $"ADO.NET stream partition ownership was lost for '{serviceId}/{providerId}/{queueId}' at epoch {ownerEpoch}. The stale receiver cannot advance its checkpoint."); + } + + return new( + (update.Checkpoint ?? 0).ToString(CultureInfo.InvariantCulture), + update.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); + } + + throw new InvalidOperationException( + $"ADO.NET stream partition ownership was lost for '{serviceId}/{providerId}/{queueId}' at epoch {ownerEpoch}. The stale receiver cannot advance its checkpoint."); + } + + public Task Initialize( + RecoverableStreamStartPosition position, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_partition is null) + { + throw new InvalidOperationException("The ADO.NET stream partition checkpoint must be loaded before initializing its source."); + } + + _readOffset = position.Checkpoint is null + ? 0 + : long.Parse(position.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture); + return Task.CompletedTask; + } + + public async Task> Read( + int maxCount, + CancellationToken cancellationToken) + { + var messages = await queries.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + _readOffset, + Math.Min(maxCount, options.MaxMessagesPerRead)).WaitAsync(cancellationToken); + + var cleanup = await queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + checked((int)options.RetentionPeriod.TotalSeconds), + options.MaximumRetentionPeriod is { } maximum + ? checked((int)maximum.TotalSeconds) + : null, + checked((int)options.CleanupInterval.TotalSeconds), + options.CleanupBatchSize).WaitAsync(cancellationToken); + if (cleanup.HardDeletedCount > 0) + { + LogHardRetentionCrossed( + logger, + serviceId, + providerId, + queueId, + cleanup.HardDeletedCount, + cleanup.HardDeletedFromMessageId, + cleanup.HardDeletedThroughMessageId, + cleanup.Checkpoint); + } + + return messages as IReadOnlyList ?? messages.ToList(); + } + + public void MessagesAdded(IReadOnlyList messages) + { + if (messages.Count > 0) + { + _readOffset = messages[^1].MessageId; + } + } + + public Task Shutdown(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + + private void ThrowIfRetentionGap(AdoNetStreamPartitionState state) + { + if (state.Checkpoint is { } checkpoint + && state.EarliestMessageId is { } earliest + && checkpoint < earliest - 1) + { + throw new DataNotAvailableException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' has a retention gap: " + + $"checkpoint {checkpoint}, earliest retained message {earliest}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); + } + } + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Hard stream retention deleted {DeletedCount} messages for {ServiceId}/{ProviderId}/{QueueId} from {DeletedFrom} through {DeletedThrough}, crossing checkpoint {Checkpoint}.")] + private static partial void LogHardRetentionCrossed( + ILogger logger, + string serviceId, + string providerId, + string queueId, + int deletedCount, + long? deletedFrom, + long? deletedThrough, + long? checkpoint); +} + +internal sealed class AdoNetRecoverableStreamDataAdapter( + Serializer serializer) : IRecoverableStreamDataAdapter +{ + public StreamPosition GetStreamPosition(AdoNetStreamMessage queueMessage) + => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.MessageId)); + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + AdoNetStreamMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + var size = SegmentBuilder.CalculateAppendSize(queueMessage.Payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.Payload); + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.MessageId, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = queueMessage.CreatedOn, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + var offset = 0; + var payload = SegmentBuilder.ReadNextBytes(cachedMessage.Segment, ref offset).ToArray(); + var message = new AdoNetStreamMessage( + string.Empty, + string.Empty, + string.Empty, + cachedMessage.SequenceNumber, + cachedMessage.StreamId.FullKey.ToArray(), + cachedMessage.StreamId.Namespace.Length, + cachedMessage.EnqueueTimeUtc, + payload); + return AdoNetBatchContainer.FromMessage(serializer, message); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex); + + public string GetOffset(ref CachedMessage cachedMessage) + => cachedMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture); + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return token is EventSequenceTokenV2; + } +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs index 97a35ba4e6a..cb61ade02e7 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs @@ -1,10 +1,12 @@ namespace Orleans.Streaming.AdoNet; /// -/// A placeholder for the retained-log receiver integration. +/// Logs subscriber failures without mutating the shared retained log. /// -internal class AdoNetStreamFailureHandler : IStreamFailureHandler +internal partial class AdoNetStreamFailureHandler : IStreamFailureHandler { + private readonly ILogger _logger; + public AdoNetStreamFailureHandler( bool faultOnFailure, AdoNetStreamOptions streamOptions, @@ -14,30 +16,55 @@ public AdoNetStreamFailureHandler( ILogger logger) { ShouldFaultSubsriptionOnError = faultOnFailure; + _logger = logger; _ = streamOptions; _ = clusterOptions; _ = mapper; _ = queries; - _ = logger; } - /// - /// Gets a value indicating whether the subscription should fault when there is an error. - /// + /// public bool ShouldFaultSubsriptionOnError { get; } - /// - /// Failure handling is supplied by the shared recoverable stream pipeline. - /// - public Task OnDeliveryFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => - Task.FromException(CreatePipelineUnavailableException()); + /// + public Task OnDeliveryFailure( + GuidId subscriptionId, + string streamProviderName, + StreamId streamIdentity, + StreamSequenceToken? sequenceToken) + { + LogDeliveryFailure(_logger, subscriptionId, streamProviderName, streamIdentity, sequenceToken); + return Task.CompletedTask; + } + + /// + public Task OnSubscriptionFailure( + GuidId subscriptionId, + string streamProviderName, + StreamId streamIdentity, + StreamSequenceToken? sequenceToken) + { + LogSubscriptionFailure(_logger, subscriptionId, streamProviderName, streamIdentity, sequenceToken); + return Task.CompletedTask; + } - /// - /// Failure handling is supplied by the shared recoverable stream pipeline. - /// - public Task OnSubscriptionFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => - Task.FromException(CreatePipelineUnavailableException()); + [LoggerMessage( + Level = LogLevel.Warning, + Message = "ADO.NET stream delivery failed for subscription {SubscriptionId} on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The retained source record was not modified.")] + private static partial void LogDeliveryFailure( + ILogger logger, + GuidId subscriptionId, + string providerName, + StreamId streamId, + StreamSequenceToken? sequenceToken); - private static InvalidOperationException CreatePipelineUnavailableException() => - new("The ADO.NET retained-log failure policy requires the shared recoverable stream cache and receiver pipeline."); -} \ No newline at end of file + [LoggerMessage( + Level = LogLevel.Warning, + Message = "ADO.NET stream subscription {SubscriptionId} failed on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The retained source record was not modified.")] + private static partial void LogSubscriptionFailure( + ILogger logger, + GuidId subscriptionId, + string providerName, + StreamId streamId, + StreamSequenceToken? sequenceToken); +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index 66d541bc116..e0e2a97e6a2 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -36,6 +36,11 @@ public class AdoNetStreamOptions /// public bool StartFromNow { get; set; } + /// + /// Gets or sets a value indicating whether a subscription is faulted after delivery failure handling. + /// + public bool FaultOnDeliveryFailure { get; set; } + /// /// Gets or sets the maximum number of messages returned by a partition read. /// diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs index a9ea7669196..787b1f33c4f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs @@ -30,19 +30,20 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CheckpointPersistInterval)} must be greater than zero."); } - if (options.RetentionPeriod <= TimeSpan.Zero) + if (IsInvalidSqlInterval(options.RetentionPeriod)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.RetentionPeriod)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.RetentionPeriod)} must be between one second and {int.MaxValue} seconds."); } - if (options.MaximumRetentionPeriod is { } maximumRetentionPeriod && maximumRetentionPeriod < options.RetentionPeriod) + if (options.MaximumRetentionPeriod is { } maximumRetentionPeriod + && (IsInvalidSqlInterval(maximumRetentionPeriod) || maximumRetentionPeriod < options.RetentionPeriod)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaximumRetentionPeriod)} must be greater than or equal to {nameof(options.RetentionPeriod)}."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaximumRetentionPeriod)} must fit in SQL integer seconds and be greater than or equal to {nameof(options.RetentionPeriod)}."); } - if (options.CleanupInterval <= TimeSpan.Zero) + if (IsInvalidSqlInterval(options.CleanupInterval)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupInterval)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupInterval)} must be between one second and {int.MaxValue} seconds."); } if (options.CleanupBatchSize <= 0) @@ -55,4 +56,7 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.InitializationTimeout)} must be greater than zero."); } } + + private static bool IsInvalidSqlInterval(TimeSpan value) + => value < TimeSpan.FromSeconds(1) || value.TotalSeconds > int.MaxValue; } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index e7040c22b51..3dd569259db 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -51,12 +51,15 @@ await builder.RunAsync(); The provider stores each queue as an immutable partition log. Configure retained-log behavior with: - `StartFromNow`: initialize a new checkpoint at the retained tail instead of before the earliest retained message. +- `FaultOnDeliveryFailure`: optionally fault a failing subscription without altering the shared retained log. - `MaxMessagesPerRead`: bound each ordered storage read. - `CheckpointPersistInterval`: throttle durable checkpoint updates. - `RetentionPeriod`: retain checkpointed messages for at least this period (one day by default). - `MaximumRetentionPeriod`: optionally delete older messages even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed delivery gap. - `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. +The provider is rewindable while requested records remain retained. It resumes strictly after its durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. + ## Alpha schema upgrade The current streaming scripts use schema version 2 and are intentionally incompatible with the former queue, visibility-timeout, confirmation, and dead-letter schema. The provider fails during initialization when it detects old or mixed streaming query keys. diff --git a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs index 1450aa4eef9..df264c68161 100644 --- a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs +++ b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs @@ -22,6 +22,8 @@ public partial class AdoNetStreamOptions [Redact] public System.Data.Common.DbDataSource? DataSource { get { throw null; } set { } } + public bool FaultOnDeliveryFailure { get { throw null; } set { } } + public System.TimeSpan InitializationTimeout { get { throw null; } set { } } public string Invariant { get { throw null; } set { } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs index a788a51604f..ad5c76dab36 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs @@ -241,6 +241,15 @@ public void Streaming_ValidatesRetentionPeriod(int seconds, bool valid) AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.RetentionPeriod)); } + [Fact] + public void Streaming_RejectsSubSecondRetentionPeriod() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromMilliseconds(500); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + [Fact] public void Streaming_AllowsNullMaximumRetentionPeriod() { @@ -285,6 +294,15 @@ public void Streaming_ValidatesCleanupInterval(int seconds, bool valid) AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CleanupInterval)); } + [Fact] + public void Streaming_RejectsSubSecondCleanupInterval() + { + var options = ValidStreamOptions(); + options.CleanupInterval = TimeSpan.FromMilliseconds(500); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.CleanupInterval)); + } + [Theory] [InlineData(1, true)] [InlineData(0, false)] diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs index 9c0e6af48ce..1ceea788554 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs @@ -87,6 +87,42 @@ public void AdoNetBatchContainer_ToMessagePayload_CreatesPayload() Assert.Equal(0, container.Dequeued); } + [Fact] + public void RecoverableDataAdapter_UsesIdentityColumnsAndDecodesPayloadLazily() + { + var serializer = fixture.Serializer.GetSerializer(); + var streamId = StreamId.Create("MyNamespace", "MyKey"); + var payload = AdoNetBatchContainer.ToMessagePayload( + serializer, + streamId, + [new TestModel(1)], + requestContext: null); + var message = new AdoNetStreamMessage( + "service", + "provider", + "queue", + 42, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); + var adapter = new AdoNetRecoverableStreamDataAdapter(serializer); + + var position = adapter.GetStreamPosition(message); + var cached = adapter.FromQueueMessage( + position, + message, + DateTime.UtcNow, + size => new byte[size]); + + Assert.Equal(streamId, cached.StreamId); + Assert.Equal("42", adapter.GetOffset(ref cached)); + var batch = Assert.IsType(adapter.GetBatchContainer(ref cached)); + Assert.Equal(streamId, batch.StreamId); + Assert.Equal([new TestModel(1)], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(new EventSequenceTokenV2(42), batch.SequenceToken); + } + [Fact] public void AdoNetBatchContainer_GetEvents_ThrowsOnHalfBaked() { diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index 91a99732465..d06ae894b74 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -84,7 +84,7 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString + ConnectionString = _storage.ConnectionString, }; var clusterOptions = new ClusterOptions { @@ -104,8 +104,12 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() Assert.NotNull(adapter); Assert.IsType(adapter); Assert.Equal(name, adapter.Name); - Assert.False(adapter.IsRewindable); + Assert.True(adapter.IsRewindable); Assert.Equal(StreamProviderDirection.ReadWrite, adapter.Direction); + var queueId = factory.GetStreamQueueMapper().GetAllQueues().First(); + Assert.Same( + adapter.CreateReceiver(queueId), + factory.GetQueueAdapterCache().CreateQueueCache(queueId)); } /// @@ -119,7 +123,8 @@ public async Task AdoNetQueueAdapterFactory_GetsDeliveryFailureHandler() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString + ConnectionString = _storage.ConnectionString, + FaultOnDeliveryFailure = true, }; var clusterOptions = new ClusterOptions { @@ -139,11 +144,11 @@ public async Task AdoNetQueueAdapterFactory_GetsDeliveryFailureHandler() // assert Assert.NotNull(handler); Assert.IsType(handler); - Assert.False(handler.ShouldFaultSubsriptionOnError); + Assert.True(handler.ShouldFaultSubsriptionOnError); } /// - /// Tests that the gets a instance. + /// Tests that the exposes its shared receiver/cache registry. /// [Fact] public void AdoNetQueueAdapterFactory_GetsQueueAdapterCache() @@ -171,7 +176,7 @@ public void AdoNetQueueAdapterFactory_GetsQueueAdapterCache() // assert Assert.NotNull(cache); - Assert.IsType(cache); + Assert.Same(factory, cache); } /// diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index fe61c8c61c5..4281731985a 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -102,7 +102,7 @@ public void AdoNetQueueAdapter_Constructs() // assert Assert.Equal(name, adapter.Name); - Assert.False(adapter.IsRewindable); + Assert.True(adapter.IsRewindable); Assert.Equal(StreamProviderDirection.ReadWrite, adapter.Direction); } From 98a50bb62415ca589facc55a673eaefb325394e8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 12:41:48 -0700 Subject: [PATCH 09/90] fix(streaming): incorporate preserved recovery fixes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../Streams/DynamoDBStreamCheckpointStore.cs | 35 ++-- .../DynamoDBStreamQueueCheckpointer.cs | 19 +- .../Streams/KinesisAdapterFactory.cs | 3 +- .../Streams/KinesisPooledAdapterReceiver.cs | 167 ++++++++++++++++-- .../AdoNetQueueAdapter.cs | 99 +++++++---- .../AdoNetQueueAdapterReceiver.cs | 14 +- .../AzureTableStreamQueueCheckpointer.cs | 107 +++++++---- ...zureTableStreamQueueCheckpointerFactory.cs | 11 +- .../Streams/EventHub/EventHubCheckpointer.cs | 34 +++- .../Shared/Storage/AzureTableDataManager.cs | 40 ++++- .../IRecoverableStreamQueueCache.cs | 29 +++ .../QueueAdapterReceiverRegistry.cs | 7 + .../RecoverableStreamQueueCache.cs | 13 +- .../RecoverableStreamReceiver.cs | 142 ++++++++++++++- .../Orleans.Streaming.EventHubs.cs | 10 ++ .../Orleans.Streaming/Orleans.Streaming.cs | 14 +- .../AdoNetQueueAdapterFactoryTests.cs | 7 + ...bleStreamQueueCheckpointerContractTests.cs | 42 +++++ .../DynamoDBStreamQueueCheckpointerTests.cs | 38 ++-- .../KinesisRuntimeTests.cs | 49 +++++ .../RecoverableStreamReceiverTests.cs | 167 +++++++++++++++++- 21 files changed, 896 insertions(+), 151 deletions(-) create mode 100644 src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs create mode 100644 test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs index 0f023d86359..a1fbe7e1551 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs @@ -13,15 +13,7 @@ namespace Orleans.Streaming.Kinesis { - internal interface IDynamoDBStreamCheckpointStore - { - ValueTask Load(CancellationToken cancellationToken); - - ValueTask Update( - string checkpoint, - string expectedCheckpoint, - CancellationToken cancellationToken); - } + internal interface IDynamoDBStreamCheckpointStore : IStreamCheckpointStore; internal sealed partial class DynamoDBStreamCheckpointStore : IDynamoDBStreamCheckpointStore { @@ -57,13 +49,13 @@ public DynamoDBStreamCheckpointStore( }; } - public async ValueTask Load(CancellationToken cancellationToken) + public async ValueTask Load(CancellationToken cancellationToken) { await _mutex.WaitAsync(cancellationToken); try { await LoadCore(cancellationToken); - return _checkpoint; + return GetState(); } finally { @@ -71,13 +63,13 @@ public async ValueTask Load(CancellationToken cancellationToken) } } - public async ValueTask Update( + public async ValueTask Update( string checkpoint, - string expectedCheckpoint, + string expectedVersion, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(checkpoint); - ArgumentNullException.ThrowIfNull(expectedCheckpoint); + ArgumentNullException.ThrowIfNull(expectedVersion); await _mutex.WaitAsync(cancellationToken); try @@ -87,9 +79,12 @@ public async ValueTask Update( await LoadCore(cancellationToken); } - if (!string.Equals(_checkpoint, expectedCheckpoint, StringComparison.Ordinal)) + var currentVersion = _version == 0 + ? string.Empty + : _version.ToString(CultureInfo.InvariantCulture); + if (!string.Equals(currentVersion, expectedVersion, StringComparison.Ordinal)) { - return _checkpoint; + return GetState(); } try @@ -137,14 +132,20 @@ public async ValueTask Update( await LoadCore(cancellationToken); } - return _checkpoint; + return GetState(); } + finally { _mutex.Release(); } } + private StreamCheckpointStoreState GetState() + => new( + _checkpoint, + _version == 0 ? string.Empty : _version.ToString(CultureInfo.InvariantCulture)); + internal static async Task InitializeTable( IAmazonDynamoDB client, DynamoDBStreamQueueCheckpointerOptions options, diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 7cd46a09efe..6703cbc8605 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -18,7 +18,7 @@ internal DynamoDBStreamQueueCheckpointer( DynamoDBStreamQueueCheckpointerOptions options) { _inner = new StreamQueueCheckpointer( - new StreamCheckpointStoreAdapter(store), + store, new StreamQueueCheckpointerOptions { CheckpointComparer = StreamCheckpointComparers.Numeric, @@ -63,22 +63,5 @@ public void Update(string offset, DateTime utcNow, CancellationToken cancellatio /// public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); - private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointStore - { - public async ValueTask Load(CancellationToken cancellationToken) - { - var checkpoint = await store.Load(cancellationToken).ConfigureAwait(false); - return new(checkpoint, checkpoint); - } - - public async ValueTask Update( - string checkpoint, - string expectedVersion, - CancellationToken cancellationToken) - { - var persistedCheckpoint = await store.Update(checkpoint, expectedVersion, cancellationToken).ConfigureAwait(false); - return new(persistedCheckpoint, persistedCheckpoint); - } - } } } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index b37da5ad668..9eb953cc0c4 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -161,7 +161,8 @@ private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) _loggerFactory, _topologyMonitor, _options.GetRecordsInterval, - _timeProvider + _timeProvider, + receiver => _receivers.Remove(queueId, receiver) ); } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index 74d292cdcaf..df361637d6f 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -1,4 +1,6 @@ +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using Amazon.Kinesis; using Microsoft.Extensions.Logging; using Orleans.Configuration; @@ -17,7 +19,13 @@ internal sealed class KinesisPooledAdapterReceiver : IQueueAdapterReceiver, IQue private readonly KinesisRecoverableStreamSource _source; private readonly KinesisRecoverableStreamDataAdapter _dataAdapter; private readonly RecoverableStreamQueueCache _cache; + private readonly Action? _onShutdown; + private readonly object _lifecycleLock = new(); + private readonly CancellationTokenSource _lifecycleCancellation = new(); private RecoverableStreamReceiver? _inner; + private Task? _initializationTask; + private int _initialized; + private int _shutdown; public KinesisPooledAdapterReceiver( IAmazonKinesis client, @@ -29,10 +37,12 @@ public KinesisPooledAdapterReceiver( ILoggerFactory loggerFactory, KinesisShardTopologyMonitor topologyMonitor, TimeSpan getRecordsInterval, - TimeProvider timeProvider) + TimeProvider timeProvider, + Action? onShutdown = null) { _checkpointerFactory = checkpointerFactory; _partition = partition; + _onShutdown = onShutdown; var logger = loggerFactory.CreateLogger(); _source = new( client, @@ -59,21 +69,28 @@ public KinesisPooledAdapterReceiver( public async Task Initialize(TimeSpan timeout) { using var cancellation = new CancellationTokenSource(timeout); - var checkpointer = await _checkpointerFactory.Create(_partition, cancellation.Token); - _inner = new( - _source, - _dataAdapter, - _cache, - checkpointer, - startFromNow: false); - await _inner.Initialize(timeout); + await EnsureInitialized(cancellation.Token); } - public Task> GetQueueMessagesAsync(int maxCount) - => GetInner().GetQueueMessagesAsync(maxCount, CancellationToken.None); + public async Task> GetQueueMessagesAsync(int maxCount) + => await GetQueueMessagesAsync(maxCount, CancellationToken.None); - public Task> GetQueueMessagesAsync(int maxCount, CancellationToken cancellationToken) - => GetInner().GetQueueMessagesAsync(maxCount, cancellationToken); + public async Task> GetQueueMessagesAsync(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) + { + return []; + } + + await EnsureInitialized(cancellationToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return []; + } + + return await GetInner().GetQueueMessagesAsync(maxCount, cancellationToken); + } public Task MessagesDeliveredAsync(IList messages) => Task.CompletedTask; @@ -81,8 +98,81 @@ public Task MessagesDeliveredAsync(IList messages) public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; - public Task Shutdown(TimeSpan timeout) - => _inner is null ? _source.Shutdown(CancellationToken.None) : _inner.Shutdown(timeout); + public async Task Shutdown(TimeSpan timeout) + { + if (Interlocked.Exchange(ref _shutdown, 1) != 0) + { + return; + } + + try + { + _lifecycleCancellation.Cancel(); + var shutdownWatch = Stopwatch.StartNew(); + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + List? exceptions = null; + Task? initializationTask; + lock (_lifecycleLock) + { + initializationTask = _initializationTask; + } + + if (initializationTask is not null) + { + try + { + await initializationTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + } + + try + { + if (_inner is null) + { + await _source.Shutdown(cancellationToken); + } + else + { + var remaining = timeout == Timeout.InfiniteTimeSpan + ? Timeout.InfiniteTimeSpan + : timeout > shutdownWatch.Elapsed + ? timeout - shutdownWatch.Elapsed + : TimeSpan.Zero; + await _inner.Shutdown(remaining); + } + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + if (exceptions is [var singleException]) + { + ExceptionDispatchInfo.Capture(singleException).Throw(); + } + + if (exceptions is { Count: > 1 }) + { + throw new AggregateException(exceptions); + } + } + finally + { + _onShutdown?.Invoke(this); + } + } public int GetMaxAddCount() => _cache.GetMaxAddCount(); @@ -99,7 +189,52 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? public bool IsUnderPressure() => _cache.IsUnderPressure(); public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) - => GetInner().UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); + => _inner?.UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); + + private Task EnsureInitialized(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lifecycleLock) + { + if (Volatile.Read(ref _shutdown) != 0 || Volatile.Read(ref _initialized) != 0) + { + return Task.CompletedTask; + } + + if (_initializationTask is null || _initializationTask.IsCompleted) + { + _initializationTask = InitializeCore(); + } + + return _initializationTask.WaitAsync(cancellationToken); + } + } + + private async Task InitializeCore() + { + var lifecycleToken = _lifecycleCancellation.Token; + if (_inner is null) + { + var checkpointer = await _checkpointerFactory.Create(_partition, lifecycleToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + + _inner = new( + _source, + _dataAdapter, + _cache, + checkpointer, + startFromNow: false); + } + + await _inner.Initialize(lifecycleToken); + if (Volatile.Read(ref _shutdown) == 0) + { + Volatile.Write(ref _initialized, 1); + } + } private RecoverableStreamReceiver GetInner() => _inner ?? throw new InvalidOperationException("The Kinesis receiver has not been initialized."); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index 4b8c424e2ef..09c3aadd502 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -3,25 +3,45 @@ namespace Orleans.Streaming.AdoNet; /// /// Stream queue storage adapter for ADO.NET providers. /// -internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, Serializer serializer, ILogger logger, IServiceProvider serviceProvider) : IQueueAdapter, IQueueAdapterCache +internal partial class AdoNetQueueAdapter : IQueueAdapter, IQueueAdapterCache { - private readonly ILogger _logger = logger; - private readonly QueueAdapterReceiverRegistry _receivers = new( - queueId => ReceiverFactory( - serviceProvider, - [ - name, - mapper.GetAdoNetQueueId(queueId), - streamOptions, - clusterOptions, - cacheOptions, - queries, - ])); + private readonly AdoNetStreamOptions _streamOptions; + private readonly ClusterOptions _clusterOptions; + private readonly SimpleQueueCacheOptions _cacheOptions; + private readonly AdoNetStreamQueueMapper _mapper; + private readonly RelationalOrleansQueries _queries; + private readonly Serializer _serializer; + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + private readonly QueueAdapterReceiverRegistry _receivers; + + public AdoNetQueueAdapter( + string name, + AdoNetStreamOptions streamOptions, + ClusterOptions clusterOptions, + SimpleQueueCacheOptions cacheOptions, + AdoNetStreamQueueMapper mapper, + RelationalOrleansQueries queries, + Serializer serializer, + ILogger logger, + IServiceProvider serviceProvider) + { + Name = name; + _streamOptions = streamOptions; + _clusterOptions = clusterOptions; + _cacheOptions = cacheOptions; + _mapper = mapper; + _queries = queries; + _serializer = serializer; + _logger = logger; + _serviceProvider = serviceProvider; + _receivers = new QueueAdapterReceiverRegistry(CreateReceiverCore); + } /// /// Maps to the ProviderId in the database. /// - public string Name { get; } = name; + public string Name { get; } /// /// The retained-log ADO.NET provider supports replay from cached and durable checkpoints. @@ -39,7 +59,11 @@ public IQueueAdapterReceiver CreateReceiver(QueueId queueId) public IQueueCache CreateQueueCache(QueueId queueId) => _receivers.GetOrCreate(queueId); - public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable events, StreamSequenceToken? token, Dictionary? requestContext) + public async Task QueueMessageBatchAsync( + StreamId streamId, + IEnumerable events, + StreamSequenceToken? token, + Dictionary? requestContext) { // Producer-supplied tokens are not supported. Replay positions are consumer-side tokens. if (token is not null) @@ -47,39 +71,48 @@ public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable ev throw new ArgumentException($"{nameof(AdoNetQueueAdapter)} does not support a user supplied {nameof(StreamSequenceToken)}."); } - // map the Orleans stream id to the corresponding queue id - var queueId = mapper.GetAdoNetQueueId(streamId); + var queueId = _mapper.GetAdoNetQueueId(streamId); + var payload = AdoNetBatchContainer.ToMessagePayload( + _serializer, + streamId, + events.Cast().ToList(), + requestContext); - // create the payload from the events - var payload = AdoNetBatchContainer.ToMessagePayload(serializer, streamId, events.Cast().ToList(), requestContext); - - // Append the immutable payload together with the canonical stream identifier. try { - await queries.AppendStreamMessageAsync( - clusterOptions.ServiceId, + await _queries.AppendStreamMessageAsync( + _clusterOptions.ServiceId, Name, queueId, streamId.FullKey.ToArray(), streamId.Namespace.Length, payload); } - catch (Exception ex) + catch (Exception exception) { - LogFailedToAppendStreamMessage(ex, clusterOptions.ServiceId, Name, queueId); + LogFailedToAppendStreamMessage( + exception, + _clusterOptions.ServiceId, + Name, + queueId); throw; } } - /// - /// The receiver factory. - /// - private static readonly ObjectFactory ReceiverFactory = ActivatorUtilities.CreateFactory([typeof(string), typeof(string), typeof(AdoNetStreamOptions), typeof(ClusterOptions), typeof(SimpleQueueCacheOptions), typeof(RelationalOrleansQueries)]); - - #region Logging + private AdoNetQueueAdapterReceiver CreateReceiverCore(QueueId queueId) + { + var receiver = ActivatorUtilities.CreateInstance( + _serviceProvider, + Name, + _mapper.GetAdoNetQueueId(queueId), + _streamOptions, + _clusterOptions, + _cacheOptions, + _queries); + receiver.OnShutdown = current => _receivers.Remove(queueId, current); + return receiver; + } [LoggerMessage(1, LogLevel.Error, "Failed to append stream message with ({ServiceId}, {ProviderId}, {QueueId})")] private partial void LogFailedToAppendStreamMessage(Exception ex, string serviceId, string providerId, string queueId); - - #endregion Logging } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 99bb9afd9bf..ba390f792b8 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -10,6 +10,8 @@ internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueue private const int BufferSize = 1024 * 1024; private readonly RecoverableStreamReceiver _inner; + internal Action? OnShutdown { get; set; } + public AdoNetQueueAdapterReceiver( string providerId, string queueId, @@ -58,7 +60,17 @@ public AdoNetQueueAdapterReceiver( public Task Initialize(TimeSpan timeout) => _inner.Initialize(timeout); - public Task Shutdown(TimeSpan timeout) => _inner.Shutdown(timeout); + public async Task Shutdown(TimeSpan timeout) + { + try + { + await _inner.Shutdown(timeout); + } + finally + { + OnShutdown?.Invoke(this); + } + } public Task> GetQueueMessagesAsync(int maxCount) => _inner.GetQueueMessagesAsync(maxCount, CancellationToken.None); diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index b6601b8c0f6..3bbf49451d4 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -14,9 +14,12 @@ namespace Orleans.Streams /// public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer { - private readonly AzureTableCheckpointStore _store; + private readonly IStreamCheckpointStore _store; + private readonly Func _initialize; private readonly StreamQueueCheckpointer _inner; + internal IStreamCheckpointStore Store => _store; + private AzureTableStreamQueueCheckpointer( AzureTableStreamCheckpointerOptions options, string streamProviderName, @@ -41,13 +44,15 @@ private AzureTableStreamQueueCheckpointer( var dataManager = new AzureTableDataManager( options, loggerFactory.CreateLogger()); - _store = new AzureTableCheckpointStore( + var store = new AzureTableCheckpointStore( dataManager, StreamQueueCheckpointEntity.Create( partitionKeyPrefix ?? options.PartitionKeyPrefix, streamProviderName, serviceId, partition)); + _store = store; + _initialize = store.Initialize; _inner = new StreamQueueCheckpointer( _store, new StreamQueueCheckpointerOptions @@ -62,6 +67,23 @@ private AzureTableStreamQueueCheckpointer( serviceId); } + internal AzureTableStreamQueueCheckpointer( + IStreamCheckpointStore store, + TimeSpan persistInterval, + IComparer? checkpointComparer) + { + ArgumentNullException.ThrowIfNull(store); + _store = store; + _initialize = static _ => Task.CompletedTask; + _inner = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = checkpointComparer, + PersistInterval = persistInterval, + }); + } + /// public bool CheckpointExists => _inner.CheckpointExists; @@ -75,7 +97,35 @@ public static Task> Create( string serviceId, ILoggerFactory loggerFactory) { - return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: CancellationToken.None); + } + + /// + /// Creates and initializes an Azure Table stream queue checkpointer. + /// + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) + { + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: cancellationToken); } internal static async Task> Create( @@ -85,7 +135,8 @@ internal static async Task> Create( string serviceId, ILoggerFactory loggerFactory, IComparer? defaultComparer, - string? partitionKeyPrefix = null) + string? partitionKeyPrefix = null, + CancellationToken cancellationToken = default) { var checkpointer = new AzureTableStreamQueueCheckpointer( options, @@ -95,7 +146,7 @@ internal static async Task> Create( loggerFactory, defaultComparer, partitionKeyPrefix); - await checkpointer._store.Initialize(); + await checkpointer._initialize(cancellationToken); return checkpointer; } @@ -123,7 +174,8 @@ private sealed class AzureTableCheckpointStore( { public StreamQueueCheckpointEntity Entity { get; private set; } = entity; - public Task Initialize() => dataManager.InitTableAsync(); + public Task Initialize(CancellationToken cancellationToken) + => dataManager.InitTableAsync(cancellationToken); public async ValueTask Load(CancellationToken cancellationToken) { @@ -152,45 +204,34 @@ public async ValueTask Update( Offset = checkpoint, }; - try + string version; + if (string.IsNullOrEmpty(expectedVersion)) { - string version; - if (string.IsNullOrEmpty(expectedVersion)) - { - var result = await dataManager.InsertTableEntryAsync(updatedEntity); - if (!result.isSuccess) - { - return await Load(cancellationToken); - } - - version = result.eTag!; - } - else + var result = await dataManager.InsertTableEntryAsync(updatedEntity, cancellationToken); + if (!result.isSuccess) { - version = await dataManager.UpdateTableEntryAsync(updatedEntity, expectedVersion); + return await Load(cancellationToken); } - updatedEntity.ETag = new ETag(version); - Entity = updatedEntity; - return new(checkpoint, version); - } - catch (RequestFailedException exception) when (exception.Status is 409 or 412) - { - return await Load(cancellationToken); + version = result.eTag!; } - catch (RequestFailedException exception) when (exception.Status == 404) + else { - var result = await dataManager.InsertTableEntryAsync(updatedEntity); + var result = await dataManager.TryUpdateTableEntryAsync( + updatedEntity, + expectedVersion, + cancellationToken); if (!result.isSuccess) { return await Load(cancellationToken); } - var version = result.eTag!; - updatedEntity.ETag = new ETag(version); - Entity = updatedEntity; - return new(checkpoint, version); + version = result.eTag!; } + + updatedEntity.ETag = new ETag(version); + Entity = updatedEntity; + return new(checkpoint, version); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs index 6aae03ee714..81f470b7342 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -48,14 +49,22 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + /// + public Task> Create( + string partition, + CancellationToken cancellationToken) { return AzureTableStreamQueueCheckpointer.Create( _options, _providerName, partition, _clusterOptions.ServiceId.ToString(), - _loggerFactory); + _loggerFactory, + cancellationToken); } } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 1c13fac1227..2dbb15510a7 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -37,8 +37,19 @@ public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpoi /// public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) { - return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); + return EventHubCheckpointer.Create( + options, + providerName, + partition, + this.clusterOptions.ServiceId.ToString(), + loggerFactory, + cancellationToken); } /// @@ -75,6 +86,24 @@ public class EventHubCheckpointer : IStreamQueueCheckpointer /// The logger factory. /// A task which resolves to the initialized checkpointer. public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) + => await Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + + /// + /// Factory function that creates and initializes the checkpointer. + /// + public static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) { var inner = await AzureTableStreamQueueCheckpointer.Create( options, @@ -83,7 +112,8 @@ public static async Task> Create(AzureTableStre serviceId, loggerFactory, StreamCheckpointComparers.Numeric, - StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); + StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix, + cancellationToken); return new EventHubCheckpointer(inner); } diff --git a/src/Azure/Shared/Storage/AzureTableDataManager.cs b/src/Azure/Shared/Storage/AzureTableDataManager.cs index da64f019586..715deb4a54f 100644 --- a/src/Azure/Shared/Storage/AzureTableDataManager.cs +++ b/src/Azure/Shared/Storage/AzureTableDataManager.cs @@ -224,7 +224,9 @@ public async Task UpsertTableEntryAsync(T data) /// /// Data to be inserted or replaced in the table. /// Value promise with new Etag for this data entry after completing this storage operation. - public async Task<(bool isSuccess, string? eTag)> InsertTableEntryAsync(T data) + public async Task<(bool isSuccess, string? eTag)> InsertTableEntryAsync( + T data, + CancellationToken cancellationToken = default) { const string operation = "InsertTableEntry"; var startTime = DateTime.UtcNow; @@ -233,7 +235,7 @@ public async Task UpsertTableEntryAsync(T data) { try { - var opResult = await Table.AddEntityAsync(data); + var opResult = await Table.AddEntityAsync(data, cancellationToken); return (true, opResult.Headers.ETag.GetValueOrDefault().ToString()); } catch (RequestFailedException storageException) when (storageException.Status == (int)HttpStatusCode.Conflict) @@ -252,6 +254,40 @@ public async Task UpsertTableEntryAsync(T data) } } + /// + /// Conditionally replaces a data entry using an ETag. + /// + /// The replacement data. + /// The expected ETag. + /// The cancellation token. + /// The update result and the new ETag when successful. + public async Task<(bool isSuccess, string? eTag)> TryUpdateTableEntryAsync( + T data, + string dataEtag, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(dataEtag); + + data.ETag = new ETag(dataEtag); + try + { + var response = await Table.UpdateEntityAsync( + data, + data.ETag, + TableUpdateMode.Replace, + cancellationToken); + return (true, response.Headers.ETag.GetValueOrDefault().ToString()); + } + catch (RequestFailedException exception) + when (exception.Status is (int)HttpStatusCode.NotFound + or (int)HttpStatusCode.Conflict + or (int)HttpStatusCode.PreconditionFailed) + { + return (false, null); + } + } + /// /// Merges a data entry in the Azure table. /// diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs new file mode 100644 index 00000000000..15aa21eca78 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Cache contract used by . + /// + /// The immutable source record type. + public interface IRecoverableStreamQueueCache : IQueueCache, IDisposable + { + /// + /// Packs and adds ordered source records to the cache. + /// + IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc); + + /// + /// Tries to get the newest cached provider position. + /// + bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs index 03ee2db9c8f..10efa599299 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -32,5 +32,12 @@ public QueueAdapterReceiverRegistry(Func factory) /// Gets or creates the coordinator for a queue. /// public TReceiver GetOrCreate(QueueId queueId) => _receivers.GetOrAdd(queueId, _factory); + + /// + /// Removes a receiver if it is still the registered instance for the queue. + /// + public bool Remove(QueueId queueId, TReceiver receiver) + => ((ICollection>)_receivers) + .Remove(new(queueId, receiver)); } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index ee2428a2096..a20387c793f 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -11,7 +11,7 @@ namespace Orleans.Providers.Streams.Common /// Provider-neutral pooled cache for immutable recoverable stream records. /// /// The source record type. - public sealed class RecoverableStreamQueueCache : IQueueCache, IDisposable + public sealed class RecoverableStreamQueueCache : IRecoverableStreamQueueCache { private readonly int _defaultMaxAddCount; private readonly IObjectPool _bufferPool; @@ -237,7 +237,16 @@ public void Dispose() return _current; } - public bool MoveNext() => _cache.TryGetNextMessage(_cursor, out _current); + public bool MoveNext() + { + if (!_cache.TryGetNextMessage(_cursor, out var next)) + { + return false; + } + + _current = next; + return true; + } public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index f86928d09b1..6f8329e32a2 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; using Orleans.Streams; @@ -15,10 +16,14 @@ public sealed class RecoverableStreamReceiver : IQueueAdapterRece { private readonly IRecoverableStreamSource _source; private readonly IRecoverableStreamDataAdapter _dataAdapter; - private readonly RecoverableStreamQueueCache _cache; + private readonly IRecoverableStreamQueueCache _cache; private readonly IStreamQueueCheckpointer _checkpointer; private readonly bool _startFromNow; + private readonly object _lifecycleLock = new(); + private readonly CancellationTokenSource _lifecycleCancellation = new(); + private Task? _initializeTask; private int _running; + private int _shutdown; /// /// Initializes a new instance of the class. @@ -29,6 +34,19 @@ public RecoverableStreamReceiver( RecoverableStreamQueueCache cache, IStreamQueueCheckpointer checkpointer, bool startFromNow) + : this(source, dataAdapter, (IRecoverableStreamQueueCache)cache, checkpointer, startFromNow) + { + } + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + IRecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) { _source = source ?? throw new ArgumentNullException(nameof(source)); _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); @@ -44,12 +62,49 @@ public async Task Initialize(TimeSpan timeout) ? null : new CancellationTokenSource(timeout); var cancellationToken = cancellation?.Token ?? CancellationToken.None; - var checkpoint = await _checkpointer.Load(cancellationToken); + await EnsureInitialized(cancellationToken); + } + + /// + /// Initializes the receiver. + /// + /// The cancellation token. + public Task Initialize(CancellationToken cancellationToken) + => EnsureInitialized(cancellationToken); + + private Task EnsureInitialized(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lifecycleLock) + { + if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) + { + return Task.CompletedTask; + } + + if (_initializeTask is null || _initializeTask.IsCompleted) + { + _initializeTask = InitializeCore(); + } + + return _initializeTask.WaitAsync(cancellationToken); + } + } + + private async Task InitializeCore() + { + var lifecycleToken = _lifecycleCancellation.Token; + var checkpoint = await _checkpointer.Load(lifecycleToken); await _source.Initialize( new RecoverableStreamStartPosition( _checkpointer.CheckpointExists ? checkpoint : null, _startFromNow), - cancellationToken); + lifecycleToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + Volatile.Write(ref _running, 1); } @@ -63,7 +118,14 @@ public async Task> GetQueueMessagesAsync( int maxCount, CancellationToken cancellationToken) { - if (Volatile.Read(ref _running) == 0 || maxCount <= 0) + cancellationToken.ThrowIfCancellationRequested(); + if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) + { + return []; + } + + await EnsureInitialized(cancellationToken); + if (Volatile.Read(ref _shutdown) != 0) { return []; } @@ -109,18 +171,77 @@ public Task MessagesDeliveredAsync(IList messages, Cancellation /// public async Task Shutdown(TimeSpan timeout) { - if (Interlocked.Exchange(ref _running, 0) == 0) + if (Interlocked.Exchange(ref _shutdown, 1) != 0) { return; } + Volatile.Write(ref _running, 0); + _lifecycleCancellation.Cancel(); using var cancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); var cancellationToken = cancellation?.Token ?? CancellationToken.None; - await _checkpointer.FlushAsync(cancellationToken); - await _source.Shutdown(cancellationToken); - _cache.Dispose(); + List? exceptions = null; + Task? initializeTask; + lock (_lifecycleLock) + { + initializeTask = _initializeTask; + } + + if (initializeTask is not null) + { + try + { + await initializeTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + } + + try + { + await _checkpointer.FlushAsync(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + try + { + await _source.Shutdown(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + try + { + _cache.Dispose(); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + if (exceptions is [var singleException]) + { + ExceptionDispatchInfo.Capture(singleException).Throw(); + } + + if (exceptions is { Count: > 1 }) + { + throw new AggregateException(exceptions); + } } /// @@ -145,6 +266,11 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? /// public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + var progressToken = earliestSubscriptionToken; string? offset = null; if (progressToken is null) diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index aa43e70a6a1..b3950f63988 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -370,6 +370,9 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } @@ -391,6 +394,8 @@ public partial class EventHubCheckpointerFactory : Streams.IStreamQueueCheckpoin { public EventHubCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } @@ -782,6 +787,8 @@ internal AzureTableStreamQueueCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -799,6 +806,9 @@ public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueChec { public AzureTableStreamQueueCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 1329086d40f..2023d851482 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -811,6 +811,12 @@ public partial interface IRecoverableStreamDataAdapter : ICacheDa bool TryGetOffset(Orleans.Streams.StreamSequenceToken token, out string offset); } + public partial interface IRecoverableStreamQueueCache : Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable + { + System.Collections.Generic.IReadOnlyList Add(System.Collections.Generic.IReadOnlyList messages, System.DateTime dequeueTimeUtc); + bool TryGetNewestPosition(out Orleans.Streams.StreamSequenceToken? token, out string? offset); + } + public partial interface IRecoverableStreamSource { System.Threading.Tasks.Task Initialize(RecoverableStreamStartPosition position, System.Threading.CancellationToken cancellationToken); @@ -925,6 +931,8 @@ public QueueAdapterReceiverRegistry(System.Func Receivers { get { throw null; } } public TReceiver GetOrCreate(Orleans.Streams.QueueId queueId) { throw null; } + + public bool Remove(Orleans.Streams.QueueId queueId, TReceiver receiver) { throw null; } } public partial class ReceiverMonitorDimensions @@ -936,7 +944,7 @@ public ReceiverMonitorDimensions(string queueId) { } public string QueueId { get { throw null; } set { } } } - public sealed partial class RecoverableStreamQueueCache : Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable + public sealed partial class RecoverableStreamQueueCache : IRecoverableStreamQueueCache, Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable { public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) { } @@ -965,6 +973,8 @@ public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliest public sealed partial class RecoverableStreamReceiver : Orleans.Streams.IQueueAdapterReceiver, Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController { + public RecoverableStreamReceiver(IRecoverableStreamSource source, IRecoverableStreamDataAdapter dataAdapter, IRecoverableStreamQueueCache cache, Orleans.Streams.IStreamQueueCheckpointer checkpointer, bool startFromNow) { } + public RecoverableStreamReceiver(IRecoverableStreamSource source, IRecoverableStreamDataAdapter dataAdapter, RecoverableStreamQueueCache cache, Orleans.Streams.IStreamQueueCheckpointer checkpointer, bool startFromNow) { } public void AddToCache(System.Collections.Generic.IList messages) { } @@ -979,6 +989,8 @@ public void AddToCache(System.Collections.Generic.IList> GetQueueMessagesAsync(int maxCount) { throw null; } + public System.Threading.Tasks.Task Initialize(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Initialize(System.TimeSpan timeout) { throw null; } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index d06ae894b74..10009d375b1 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -110,6 +110,13 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() Assert.Same( adapter.CreateReceiver(queueId), factory.GetQueueAdapterCache().CreateQueueCache(queueId)); + var firstReceiver = adapter.CreateReceiver(queueId); + await firstReceiver.Shutdown(TimeSpan.FromSeconds(5)); + var reassignedReceiver = adapter.CreateReceiver(queueId); + Assert.NotSame(firstReceiver, reassignedReceiver); + Assert.Same( + reassignedReceiver, + factory.GetQueueAdapterCache().CreateQueueCache(queueId)); } /// diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs new file mode 100644 index 00000000000..cbf3ce824f0 --- /dev/null +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs @@ -0,0 +1,42 @@ +using Orleans.Streams; +using TestExtensions; +using UnitTests.StreamingTests; + +namespace ServiceBus.Tests.CheckpointerTests; + +[TestSuite("BVT")] +[TestProvider("EventHub")] +[TestArea("Streaming")] +[TestCategory("EventHub"), TestCategory("Streaming")] +public sealed class AzureTableStreamQueueCheckpointerContractTests : StreamQueueCheckpointerTests +{ + protected override OffsetRegressionPolicy RegressionPolicy => OffsetRegressionPolicy.Ignore; + + protected override Task> CreateCheckpointer( + ControllableCheckpointStore store) + => Task.FromResult>( + new AzureTableStreamQueueCheckpointer( + new TestStore(store), + PersistInterval, + StreamCheckpointComparers.Numeric)); + + private sealed class TestStore(ControllableCheckpointStore store) : IStreamCheckpointStore + { + public async ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var checkpoint = await store.Load().ConfigureAwait(false); + return new(checkpoint, checkpoint); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var persisted = await store.Write(checkpoint).ConfigureAwait(false); + return new(persisted, persisted); + } + } +} diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs index ba67802a1d4..df568ea1ef9 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs @@ -34,20 +34,21 @@ protected override Task> CreateCheckpointer( private sealed class TestCheckpointStore(ControllableCheckpointStore store) : IDynamoDBStreamCheckpointStore { - public ValueTask Load(CancellationToken cancellationToken) + public async ValueTask Load(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - return new(store.Load()); + var checkpoint = await store.Load().ConfigureAwait(false); + return new(checkpoint, checkpoint); } - public async ValueTask Update( + public async ValueTask Update( string checkpoint, - string expectedCheckpoint, + string expectedVersion, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); await store.Write(checkpoint).ConfigureAwait(false); - return checkpoint; + return new(checkpoint, checkpoint); } } } @@ -74,9 +75,11 @@ public async Task UpdatePersistsArbitrarySizeSequenceNumberAsString() }); var store = CreateStore(client); - await store.Update(checkpoint, string.Empty, TestContext.Current.CancellationToken); + var state = await store.Update(checkpoint, string.Empty, TestContext.Current.CancellationToken); Assert.NotNull(write); + Assert.Equal(checkpoint, state.Checkpoint); + Assert.Equal("1", state.Version); Assert.Equal(checkpoint, write.Item[DynamoDBStreamCheckpointStore.CheckpointAttribute].S); Assert.Equal("1", write.Item[DynamoDBStreamCheckpointStore.VersionAttribute].N); Assert.Equal( @@ -99,12 +102,14 @@ public async Task ConditionalConflictWithNewerCheckpointDoesNotOverwriteIt() new ConditionalCheckFailedException("stale checkpoint"))); var store = CreateStore(client); - Assert.Equal("20", await store.Update("10", string.Empty, TestContext.Current.CancellationToken)); + var result = await store.Update("10", string.Empty, TestContext.Current.CancellationToken); + Assert.Equal("20", result.Checkpoint); + Assert.Equal("7", result.Version); await client.Received(1).PutItemAsync( Arg.Any(), Arg.Any()); - Assert.Equal("20", await store.Load(TestContext.Current.CancellationToken)); + Assert.Equal("20", (await store.Load(TestContext.Current.CancellationToken)).Checkpoint); } [Fact] @@ -131,8 +136,12 @@ public async Task ConditionalConflictWithOlderCheckpointAllowsRetryUsingItsVersi }); var store = CreateStore(client); - Assert.Equal("20", await store.Update("30", string.Empty, TestContext.Current.CancellationToken)); - Assert.Equal("30", await store.Update("30", "20", TestContext.Current.CancellationToken)); + var conflicted = await store.Update("30", string.Empty, TestContext.Current.CancellationToken); + Assert.Equal("20", conflicted.Checkpoint); + Assert.Equal("7", conflicted.Version); + var updated = await store.Update("30", "7", TestContext.Current.CancellationToken); + Assert.Equal("30", updated.Checkpoint); + Assert.Equal("8", updated.Version); Assert.Equal(2, writes.Count); Assert.Equal("#version = :expectedVersion", writes[1].ConditionExpression); @@ -142,20 +151,21 @@ public async Task ConditionalConflictWithOlderCheckpointAllowsRetryUsingItsVersi writes[1].ExpressionAttributeNames["#version"]); Assert.Equal("7", writes[1].ExpressionAttributeValues[":expectedVersion"].N); Assert.Equal("8", writes[1].Item[DynamoDBStreamCheckpointStore.VersionAttribute].N); - Assert.Equal("30", await store.Load(TestContext.Current.CancellationToken)); + Assert.Equal("30", (await store.Load(TestContext.Current.CancellationToken)).Checkpoint); } [Fact] - public async Task ExpectedCheckpointMismatchReturnsPersistedCheckpointWithoutWriting() + public async Task ExpectedVersionMismatchReturnsPersistedStateWithoutWriting() { var client = Substitute.For(); client.GetItemAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(CreateReadResponse("20", 7))); var store = CreateStore(client); - var result = await store.Update("30", "10", TestContext.Current.CancellationToken); + var result = await store.Update("30", "6", TestContext.Current.CancellationToken); - Assert.Equal("20", result); + Assert.Equal("20", result.Checkpoint); + Assert.Equal("7", result.Version); await client.DidNotReceive().PutItemAsync( Arg.Any(), Arg.Any()); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index 6796a8a71b6..f38499ca770 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -113,6 +113,55 @@ public void QueueAdapterFactoryIsRewindable() Assert.Equal(StreamProviderDirection.ReadWrite, factory.Direction); } + [Fact] + public async Task PooledReceiver_ReadsAfterSuccessfulInitialization() + { + var client = Substitute.For(); + client.GetShardIteratorAsync( + Arg.Any(), + Arg.Any()) + .Returns(new GetShardIteratorResponse { ShardIterator = "iterator" }); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(new GetRecordsResponse + { + NextShardIterator = "iterator", + Records = [], + }); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var timeProvider = new FakeTimeProvider(); + var topologyMonitor = new KinesisShardTopologyMonitor( + client, + "stream", + ["shard-1"], + TimeSpan.FromMinutes(1), + timeProvider, + NullLogger.Instance); + var receiver = new KinesisPooledAdapterReceiver( + client, + "stream", + "shard-1", + checkpointerFactory, + new SimpleQueueCacheOptions(), + serializer, + NullLoggerFactory.Instance, + topologyMonitor, + TimeSpan.Zero, + timeProvider); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + await client.Received(1).GetRecordsAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + [Fact] public async Task InitialShardIteratorUsesTrimHorizonWhenNoCheckpointExists() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 3679f328e6a..3939a8ded9e 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -53,6 +53,9 @@ public async Task Receiver_ResumesAfterCheckpointAndPersistsDeliveryProgress() Assert.Null(exception); Assert.Equal("payload", batch.Payload); Assert.True(adapter.CompareCallCount > 0); + Assert.False(cursor.MoveNext()); + Assert.Same(batch, cursor.GetCurrent(out exception)); + Assert.Null(exception); receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(11), DateTime.UtcNow); Assert.Equal("11", checkpointer.LastUpdatedCheckpoint); @@ -116,14 +119,156 @@ public async Task DeliveryProgress_WithNoSubscribers_AdvancesToNewestCachedRecor await receiver.Shutdown(TimeSpan.FromSeconds(5)); } + [Fact] + public void Cache_AddsRawRecordsInOrderAndDecodesLazily() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var messages = new[] + { + new TestQueueMessage(streamA, 10, "first"), + new TestQueueMessage(streamB, 11, "second"), + new TestQueueMessage(streamA, 12, "third"), + }; + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + + var positions = cache.Add(messages, DateTime.UnixEpoch); + + Assert.Equal([10, 11, 12], positions.Select(position => position.SequenceToken.SequenceNumber)); + Assert.Equal(3, adapter.PositionCallCount); + Assert.Equal(3, adapter.FromQueueMessageCallCount); + Assert.Equal(0, adapter.GetBatchContainerCallCount); + + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + Assert.True(cursor.MoveNext()); + Assert.Equal("first", Assert.IsType(cursor.GetCurrent(out _)).Payload); + Assert.Equal(1, adapter.GetBatchContainerCallCount); + Assert.True(cursor.MoveNext()); + Assert.Equal("third", Assert.IsType(cursor.GetCurrent(out _)).Payload); + Assert.Equal(2, adapter.GetBatchContainerCallCount); + Assert.False(cursor.MoveNext()); + Assert.Equal(2, adapter.GetBatchContainerCallCount); + } + + [Fact] + public async Task Receiver_RetriesInitializationOnNextRead() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource( + [new TestQueueMessage(streamId, 1, "payload")], + initializationFailures: 1); + var receiver = CreateReceiver(source, new TestCheckpointer(string.Empty)); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + var messages = await receiver.GetQueueMessagesAsync(10, CancellationToken.None); + + Assert.Single(messages); + Assert.Equal(2, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Receiver_PreCanceledReadDoesNotInitialize() + { + var source = new TestSource([]); + var receiver = CreateReceiver(source, new TestCheckpointer(string.Empty)); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, cancellation.Token)); + + Assert.Equal(0, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Shutdown_WhenFlushFails_StillShutsDownSource() + { + var source = new TestSource([]); + var expected = new InvalidOperationException("flush failed"); + var receiver = CreateReceiver( + source, + new TestCheckpointer(string.Empty) { FlushException = expected }); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var actual = await Assert.ThrowsAsync( + () => receiver.Shutdown(TimeSpan.FromSeconds(5))); + + Assert.Same(expected, actual); + Assert.True(source.IsShutdown); + } + + [Fact] + public async Task Registry_ConcurrentRequestsReturnSameWinningInstance() + { + const int participantCount = 3; + var queue = QueueId.GetQueueId("queue", 0, 0); + using var barrier = new Barrier(participantCount); + var registry = new QueueAdapterReceiverRegistry(_ => + { + barrier.SignalAndWait(); + return new TestCombinedReceiver(); + }); + + var results = await Task.WhenAll( + Enumerable.Range(0, participantCount) + .Select(_ => Task.Run(() => registry.GetOrCreate(queue)))); + + Assert.All(results, result => Assert.Same(results[0], result)); + Assert.Same(results[0], Assert.Single(registry.Receivers).Value); + } + + [Fact] + public void Registry_RemoveAllowsFreshReceiverForReassignedQueue() + { + var queue = QueueId.GetQueueId("queue", 0, 0); + var registry = new QueueAdapterReceiverRegistry( + _ => new TestCombinedReceiver()); + var first = registry.GetOrCreate(queue); + + Assert.True(registry.Remove(queue, first)); + var second = registry.GetOrCreate(queue); + + Assert.NotSame(first, second); + Assert.Same(second, Assert.Single(registry.Receivers).Value); + } + + private static RecoverableStreamReceiver CreateReceiver( + TestSource source, + TestCheckpointer checkpointer) + { + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + return new(source, adapter, cache, checkpointer, startFromNow: false); + } + private sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); private sealed class TestDataAdapter : IRecoverableStreamDataAdapter { public int CompareCallCount { get; private set; } + public int PositionCallCount { get; private set; } + public int FromQueueMessageCallCount { get; private set; } + public int GetBatchContainerCallCount { get; private set; } public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) - => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + { + PositionCallCount++; + return new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + } public CachedMessage FromQueueMessage( StreamPosition streamPosition, @@ -131,6 +276,7 @@ public CachedMessage FromQueueMessage( DateTime dequeueTimeUtc, Func> getSegment) { + FromQueueMessageCallCount++; var size = SegmentBuilder.CalculateAppendSize(queueMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + SegmentBuilder.CalculateAppendSize(queueMessage.Payload); var segment = getSegment(size); @@ -150,6 +296,7 @@ public CachedMessage FromQueueMessage( public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) { + GetBatchContainerCallCount++; var offset = 0; _ = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); var payload = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); @@ -199,17 +346,27 @@ private sealed class TestBatchContainer( public bool ImportRequestContext() => false; } - private sealed class TestSource(IReadOnlyList messages) : IRecoverableStreamSource + private sealed class TestSource( + IReadOnlyList messages, + int initializationFailures = 0) : IRecoverableStreamSource { private bool read; + private int remainingInitializationFailures = initializationFailures; public RecoverableStreamStartPosition StartPosition { get; private set; } public bool IsShutdown { get; private set; } + public int InitializeCount { get; private set; } public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + InitializeCount++; + if (remainingInitializationFailures-- > 0) + { + throw new InvalidOperationException("initialization failed"); + } + StartPosition = position; return Task.CompletedTask; } @@ -241,6 +398,7 @@ private sealed class TestCheckpointer(string checkpoint) : IStreamQueueCheckpoin public string? LastUpdatedCheckpoint { get; private set; } public int FlushCount { get; private set; } + public Exception? FlushException { get; init; } public Task Load() => Task.FromResult(checkpoint); @@ -261,6 +419,11 @@ public Task FlushAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); FlushCount++; + if (FlushException is not null) + { + return Task.FromException(FlushException); + } + return Task.CompletedTask; } } From 590593dc4842b9867ee9fbeac870f5e059374b70 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 13 Aug 2026 13:13:32 -0700 Subject: [PATCH 10/90] fix(streaming): address recoverable cache review Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cd9c9fb-4325-4b4f-9bec-f04dc51b21b3 --- .../RecoverableStreamQueueCache.cs | 3 ++ .../RecoverableStreamReceiverTests.cs | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index a20387c793f..366722204f4 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -140,6 +140,9 @@ public void AddToCache(IList messages) public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) { purgedItems = null; + // Pressure indicates that a lagging cursor can still need the oldest records. Time-based + // eviction in that state would turn backpressure into data loss. UpdateDeliveryProgress + // removes records through the safe delivery watermark and releases pressure as consumers advance. if (!IsUnderPressure()) { _evictionStrategy.PerformPurge(DateTime.UtcNow); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 3939a8ded9e..9048c55b5a2 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -119,6 +119,33 @@ public async Task DeliveryProgress_WithNoSubscribers_AdvancesToNewestCachedRecor await receiver.Shutdown(TimeSpan.FromSeconds(5)); } + [Fact] + public void CachePressure_BlocksUnsafeTimePurgeUntilDeliveryProgressAdvances() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new TestDataAdapter(); + var evictionStrategy = new NoOpEvictionStrategy(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + evictionStrategy, + NullLogger.Instance, + flowController: new FixedFlowController(0)); + var positions = cache.Add( + [new TestQueueMessage(streamId, 11, "payload")], + DateTime.UnixEpoch); + + Assert.True(cache.IsUnderPressure()); + Assert.False(cache.TryPurgeFromCache(out _)); + Assert.Equal(0, evictionStrategy.PerformPurgeCount); + Assert.Equal(1, cache.ItemCount); + + cache.UpdateDeliveryProgress(positions[0].SequenceToken, DateTime.UtcNow); + + Assert.Equal(0, cache.ItemCount); + } + [Fact] public void Cache_AddsRawRecordsInOrderAndDecodesLazily() { @@ -430,12 +457,15 @@ public Task FlushAsync(CancellationToken cancellationToken) private sealed class NoOpEvictionStrategy : IEvictionStrategy { + public int PerformPurgeCount { get; private set; } + public IPurgeObservable PurgeObservable { private get; set; } = null!; public Action? OnPurged { get; set; } public void PerformPurge(DateTime utcNow) { + PerformPurgeCount++; } public void OnBlockAllocated(FixedSizeBuffer newBlock) @@ -443,6 +473,11 @@ public void OnBlockAllocated(FixedSizeBuffer newBlock) } } + private sealed class FixedFlowController(int maxAddCount) : IQueueFlowController + { + public int GetMaxAddCount() => maxAddCount; + } + private sealed class TrackingBufferPool : IObjectPool { public int FreeCount { get; private set; } From ba0368639ce766be10c6c1d15952ddd6d15056a6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 02:57:35 -0700 Subject: [PATCH 11/90] fix(streaming): restore retained-log invariants Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Streams/KinesisRecoverableStream.cs | 25 ++++++++++++++++--- .../MySQL-Streaming.sql | 12 --------- .../RecoverableStreamQueueCache.cs | 4 +++ .../Streaming/AdoNetClientStreamTests.cs | 8 ++---- .../Streaming/AdoNetQueueAdapterTests.cs | 8 ++++-- .../AdoNetStreamPartitionLogTests.cs | 5 ++-- .../Streaming/AdoNetStreamSchemaTests.cs | 10 ++++++++ .../KinesisBatchContainerTests.cs | 3 +++ 8 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs index 6bf2da85d4f..10ffa2f5fe1 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -1,5 +1,3 @@ -using System.Globalization; -using System.Numerics; using Amazon.Kinesis; using Amazon.Kinesis.Model; using Orleans.Providers.Streams.Common; @@ -230,11 +228,30 @@ public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) var offset = 0; var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; - var difference = BigInteger.Parse(shardSequence, NumberStyles.None, CultureInfo.InvariantCulture) - .CompareTo(BigInteger.Parse(kinesisToken.ShardSequence, NumberStyles.None, CultureInfo.InvariantCulture)); + var difference = CompareShardSequences(shardSequence, kinesisToken.ShardSequence); return difference != 0 ? difference : cachedMessage.EventIndex.CompareTo(kinesisToken.EventIndex); } + private static int CompareShardSequences(string left, string right) + { + var leftStart = 0; + while (leftStart < left.Length && left[leftStart] == '0') + { + leftStart++; + } + + var rightStart = 0; + while (rightStart < right.Length && right[rightStart] == '0') + { + rightStart++; + } + + var lengthComparison = (left.Length - leftStart).CompareTo(right.Length - rightStart); + return lengthComparison != 0 + ? lengthComparison + : left.AsSpan(leftStart).SequenceCompareTo(right.AsSpan(rightStart)); + } + public string GetOffset(ref CachedMessage cachedMessage) { var offset = 0; diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 26712881988..bfbe6b9c9d9 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -185,10 +185,6 @@ BEGIN _MessageId AS MessageId; END$$ -DELIMITER ; - -DELIMITER $$ - CREATE PROCEDURE AcquireStreamPartition ( IN _ServiceId VARCHAR(150), @@ -295,10 +291,6 @@ BEGIN _TailMessageId AS TailMessageId; END$$ -DELIMITER ; - -DELIMITER $$ - CREATE PROCEDURE AdvanceStreamCheckpoint ( IN _ServiceId VARCHAR(150), @@ -361,10 +353,6 @@ BEGIN WHERE _CurrentOwnerEpoch IS NOT NULL; END$$ -DELIMITER ; - -DELIMITER $$ - CREATE PROCEDURE CleanupStreamMessages ( IN _ServiceId VARCHAR(150), diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 366722204f4..0a38d710bfb 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -146,6 +146,10 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList if (!IsUnderPressure()) { _evictionStrategy.PerformPurge(DateTime.UtcNow); + if (_cache.IsEmpty) + { + _currentBuffer = null; + } } return false; diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs index 5c06241411b..d6dd54dee97 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs @@ -138,12 +138,8 @@ public virtual Task AdoNetStreamConsumerOnDroppedClientTest() AdoNetStreamProviderName, StreamNamespace, _output, - async () => (await _testing.Storage.ReadAsync( - "SELECT COUNT(*) FROM OrleansStreamDeadLetter", - _ => { }, - (record, i, ct) => Task.FromResult(record.GetInt32(0)), - cancellationToken: cancellationToken)) - .Single(), + getDeliveryFailureCount: null, + waitForRetryTimeouts: true, cancellationToken: cancellationToken); } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 4281731985a..99a560f1c59 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -143,8 +143,12 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() var afterEnqueued = DateTime.UtcNow.AddSeconds(1); // assert - stored messages are as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", + var stored = (await _queries.ReadStreamMessagesAsync( + serviceId, + providerId, + adoNetQueueId, + afterMessageId: 0, + maxCount: 100, TestContext.Current.CancellationToken)) .OrderBy(static message => message.MessageId) .ToList(); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs index b1d8f88a533..ffc527b430e 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs @@ -817,7 +817,8 @@ protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoe secondCommand.Parameters.AddWithValue("Payload", payload); var exception = await Assert.ThrowsAsync(() => secondCommand.ExecuteReaderAsync()); - Assert.Equal(1222, exception.Number); // "Lock request time out period exceeded." + Assert.Equal(51000, exception.Number); + Assert.Contains("initialization lock", exception.Message, StringComparison.Ordinal); } // Rolling back must not burn the allocated identifier: it is reused by the next append, @@ -827,7 +828,7 @@ protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoe var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); Assert.Equal(1, afterRollback.MessageId); - var rows = await _storage.ReadAsync("SELECT * FROM OrleansStreamMessage"); + var rows = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); var stored = Assert.Single(rows); Assert.Equal(1, stored.MessageId); } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 8e50ec736b7..5aab98d496f 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -70,6 +70,16 @@ public void MySqlAppendUsesSelectForUpdateThenUpdate() Assert.DoesNotContain("@@session.in_transaction", script, StringComparison.Ordinal); } + [Fact] + public void MySqlScriptDoesNotProduceWhitespaceOnlyBatches() + { + var batches = ReadScript("MySQL") + .Replace("END$$", "END;", StringComparison.Ordinal) + .Split(["DELIMITER $$", "DELIMITER ;"], StringSplitOptions.RemoveEmptyEntries); + + Assert.DoesNotContain(batches, string.IsNullOrWhiteSpace); + } + private static string ReadScript(string provider) => File.ReadAllText(Path.Combine(AppContext.BaseDirectory, $"{provider}-Streaming.sql")); } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index 4085bb6763c..9493349d596 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -155,6 +155,9 @@ public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering( Assert.True(adapter.Compare( ref cached, new KinesisSequenceToken("123456789012345678901234567889", 1000, 0)) > 0); + Assert.Equal(0, adapter.Compare( + ref cached, + new KinesisSequenceToken("000123456789012345678901234567890", 1000, 0))); var batch = Assert.IsType(adapter.GetBatchContainer(ref cached)); Assert.Equal(streamId, batch.StreamId); From 67206907b9ace9084006e89ff8b60b0e5476ef8f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 03:13:42 -0700 Subject: [PATCH 12/90] fix(streaming): preserve empty-cache progress Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Common/PooledCache/PooledQueueCache.cs | 24 ++++++++++++++----- .../RecoverableStreamReceiverTests.cs | 24 +++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 36983a9de26..8569835ca61 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -271,7 +271,7 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) // If nothing in cache, unset token and wait for more data. if (IsEmpty) { - cursor.State = CursorStates.Unset; + cursor.State = sequenceToken is null ? CursorStates.Idle : CursorStates.Unset; cursor.SequenceToken = sequenceToken; return null; } @@ -281,10 +281,22 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) // if sequenceToken is null, iterate from newest message in cache if (sequenceToken == null) { - cursor.State = CursorStates.Idle; - cursor.CurrentBlock = newestBlock; - cursor.Index = newestBlock.Value.NewestMessageIndex; - cursor.SequenceToken = newestBlock.Value.GetNewestSequenceToken(cacheDataAdapter); + if (cursor.State == CursorStates.Idle) + { + var waitingOldestBlock = messageBlocks.Last!; + cursor.State = CursorStates.Set; + cursor.CurrentBlock = waitingOldestBlock; + cursor.Index = waitingOldestBlock.Value.OldestMessageIndex; + cursor.SequenceToken = waitingOldestBlock.Value.GetOldestSequenceToken(cacheDataAdapter); + } + else + { + cursor.State = CursorStates.Idle; + cursor.CurrentBlock = newestBlock; + cursor.Index = newestBlock.Value.NewestMessageIndex; + cursor.SequenceToken = newestBlock.Value.GetNewestSequenceToken(cacheDataAdapter); + } + return null; } @@ -687,7 +699,7 @@ public Cursor(StreamId streamId) public CursorStates State; - // current sequence token; null while Unset (no sequence token has been established yet) + // current sequence token; null while waiting for the first message to arrive public StreamSequenceToken? SequenceToken; public long BlockGeneration; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 9048c55b5a2..d122669a5cf 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -119,6 +119,30 @@ public async Task DeliveryProgress_WithNoSubscribers_AdvancesToNewestCachedRecor await receiver.Shutdown(TimeSpan.FromSeconds(5)); } + [Fact] + public void Cache_CursorCreatedWhileEmptyReadsFirstLaterRecord() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + var initial = cache.Add([new TestQueueMessage(streamId, 10, "initial")], DateTime.UnixEpoch); + cache.UpdateDeliveryProgress(initial[0].SequenceToken, DateTime.UtcNow); + Assert.Equal(0, cache.ItemCount); + + using var cursor = cache.GetCacheCursor(streamId, token: null); + Assert.False(cursor.MoveNext()); + + _ = cache.Add([new TestQueueMessage(streamId, 11, "payload")], DateTime.UnixEpoch); + + Assert.True(cursor.MoveNext()); + Assert.Equal("payload", Assert.IsType(cursor.GetCurrent(out _)).Payload); + } + [Fact] public void CachePressure_BlocksUnsafeTimePurgeUntilDeliveryProgressAdvances() { From 38d7d174bf500bccf73bcc2bd0307846f4958690 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 03:31:42 -0700 Subject: [PATCH 13/90] fix(streaming): clarify cache initialization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AdoNetRecoverableStream.cs | 11 ++++++----- .../Common/PooledCache/PooledQueueCache.cs | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index e2f65bb3a62..f288dbd50dc 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -16,15 +16,16 @@ internal sealed partial class AdoNetRecoverableStream( public async ValueTask Load(CancellationToken cancellationToken) { - _partition = await queries.AcquireStreamPartitionAsync( + var partition = await queries.AcquireStreamPartitionAsync( serviceId, providerId, queueId, options.StartFromNow).WaitAsync(cancellationToken); - _readOffset = _partition.Checkpoint ?? 0; - ThrowIfRetentionGap(_partition); - var checkpoint = _partition.Checkpoint?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; - return new(checkpoint, _partition.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); + _partition = partition; + _readOffset = partition.Checkpoint ?? 0; + ThrowIfRetentionGap(partition); + var checkpoint = partition.Checkpoint?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + return new(checkpoint, partition.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); } public async ValueTask Update( diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 8569835ca61..7c658829b1a 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -278,7 +278,8 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) LinkedListNode newestBlock = messageBlocks.First!; // messageBlocks.Count != 0 (checked above). - // if sequenceToken is null, iterate from newest message in cache + // A cursor which waited on an empty cache starts at the oldest message; + // otherwise, a null token starts at the newest message. if (sequenceToken == null) { if (cursor.State == CursorStates.Idle) From 6f01ed2daa47afcd4926d80fc1e7d69306fd5b0e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 11:48:33 -0700 Subject: [PATCH 14/90] refactor(streaming): adopt stream partition terminology Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3100b74-af89-4744-ac56-b00e5d22a4ed --- .../content/docs/host/monitoring/signals.md | 2 +- .../streams-implementation/index.md | 2 +- .../docs/streaming/custom-queue-adapter.md | 2 +- .../content/docs/streaming/data-adapters.md | 2 +- .../docs/streaming/stream-providers.md | 6 ++--- .../AdoNetBatchContainer.cs | 4 +-- .../AdoNetQueueAdapter.cs | 2 +- .../AdoNetQueueAdapterReceiver.cs | 2 +- .../AdoNetRecoverableStream.cs | 4 +-- .../AdoNetStreamCleanupResult.cs | 2 +- .../AdoNetStreamFailureHandler.cs | 6 ++--- .../AdoNetStreamOptions.cs | 12 ++++----- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 14 +++++----- src/AdoNet/Shared/Storage/DbStoredQueries.cs | 12 ++++----- .../Storage/RelationalOrleansQueries.cs | 8 +++--- .../RecoverableStreamReceiver.cs | 2 +- ...k.cs => StreamPartitionAppendBenchmark.cs} | 6 ++--- ...StreamPartitionReadCheckpointBenchmark.cs} | 10 +++---- ...Tests.cs => AdoNetStreamPartitionTests.cs} | 26 +++++++++---------- .../Streaming/AdoNetStreamSchemaTests.cs | 2 +- 20 files changed, 63 insertions(+), 63 deletions(-) rename test/Benchmarks.AdoNet/Streaming/{RetainedLogAppendBenchmark.cs => StreamPartitionAppendBenchmark.cs} (88%) rename test/Benchmarks.AdoNet/Streaming/{RetainedLogReadCheckpointBenchmark.cs => StreamPartitionReadCheckpointBenchmark.cs} (94%) rename test/Extensions/Orleans.AdoNet.Tests/Streaming/{AdoNetStreamPartitionLogTests.cs => AdoNetStreamPartitionTests.cs} (97%) diff --git a/docs/site/src/content/docs/host/monitoring/signals.md b/docs/site/src/content/docs/host/monitoring/signals.md index 2b7d8bc0a22..eb474519606 100644 --- a/docs/site/src/content/docs/host/monitoring/signals.md +++ b/docs/site/src/content/docs/host/monitoring/signals.md @@ -73,4 +73,4 @@ Page on user impact or imminent data/availability risk. Route isolated warnings ## Dashboard and telemetry backends -The Orleans Dashboard provides a current operational view and method profiling. It isn't a replacement for retained logs, metrics, traces, or alerts. Secure it as an administrative endpoint and use OTLP for durable telemetry. See [Orleans Dashboard](../../dashboard/index.md). +The Orleans Dashboard provides a current operational view and method profiling. Use logs, metrics, traces, and alerts for durable telemetry. Secure the dashboard as an administrative endpoint and export telemetry through OTLP. See [Orleans Dashboard](../../dashboard/index.md). diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 1d7c287b758..49e0432b824 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -111,7 +111,7 @@ flowchart TB Cache capacity is not durability. The queue remains the durable boundary, subject to the adapter's acknowledgement contract. -Recoverable partition providers can compose with a partition source and data adapter. The coordinator admits immutable raw records into pooled storage, reconstructs batches lazily, reconciles the earliest subscription delivery watermark, and persists a checkpoint which resumes strictly after the last fully delivered record. +Recoverable partitioned stream providers can compose a stream partition pipeline from , a partition source, and a data adapter. The pipeline admits immutable stream records into pooled storage, reconstructs batches lazily, reconciles the earliest subscription delivery watermark, and persists a checkpoint which resumes strictly after the last fully delivered record. ## Pub-sub handshake diff --git a/docs/site/src/content/docs/streaming/custom-queue-adapter.md b/docs/site/src/content/docs/streaming/custom-queue-adapter.md index 406020a6b39..6b2bfbf56a4 100644 --- a/docs/site/src/content/docs/streaming/custom-queue-adapter.md +++ b/docs/site/src/content/docs/streaming/custom-queue-adapter.md @@ -53,7 +53,7 @@ The factory composes the adapter with queue mapping, caching, and failure handli Custom pooled caches use to position cursors against cached messages. The default implementation compares `SequenceNumber` and `EventIndex`, preserving the numeric ordering used by existing providers. Override it when the authoritative provider position is encoded in ; return an order consistent with the token produced by so cache bounds, block selection, and cache-miss detection use the same position contract. -`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a retained-log transport, implement an , have the receiver or cache load and update the per-partition position, and register it as a named component with `ConfigureComponent`. Persist a checkpoint only after all consumers have advanced beyond the corresponding cached messages. A no-op checkpointer is suitable only when replay position is deliberately disposable. +`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a partitioned stream transport, implement an , have the receiver or cache load and update the stream partition position, and register it as a named component with `ConfigureComponent`. Persist a checkpoint only after all consumers have advanced beyond the corresponding cached messages. A no-op checkpointer is suitable only when replay position is deliberately disposable. For a durable custom checkpoint backend, implement and pass it to with . Load the checkpointer before processing the partition. Each store update receives the expected backend version and returns the resulting , so a version conflict supplies the authoritative checkpoint and version for retry or reconciliation. diff --git a/docs/site/src/content/docs/streaming/data-adapters.md b/docs/site/src/content/docs/streaming/data-adapters.md index 8554da96bde..6b3dfe4048a 100644 --- a/docs/site/src/content/docs/streaming/data-adapters.md +++ b/docs/site/src/content/docs/streaming/data-adapters.md @@ -62,7 +62,7 @@ For Event Hubs, derive from to encode events published through Orleans; and - to select the physical Event Hubs partition key. -The adapter also participates in cache conversion and sequence positioning through . Preserve the Event Hubs offset and sequence number when constructing batch tokens so checkpoint and rewind behavior remains aligned with the partition log. +The adapter also participates in cache conversion and sequence positioning through . Preserve the Event Hubs offset and sequence number when constructing batch tokens so checkpoint and rewind behavior remains aligned with the stream partition. Register the adapter and Event Hubs connection under the same provider name on silos and publishing clients. The silo registration also configures durable Azure Table checkpoints: diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 3f8f5a573da..301bbacee47 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -18,7 +18,7 @@ A stream provider connects the Orleans streaming API to a transport and defines | Azure Event Hubs | [`Microsoft.Orleans.Streaming.EventHubs`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.EventHubs) | Stable | Yes, within Event Hubs retention | Yes | Event Hubs namespace, hub, consumer group, and checkpoint storage | | Amazon Kinesis | [`Microsoft.Orleans.Streaming.Kinesis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Kinesis) | Stable | Yes, within Kinesis retention | Yes | Kinesis data stream, AWS credentials, region, and durable checkpoint storage | | Amazon SQS | [`Microsoft.Orleans.Streaming.SQS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.SQS) | Stable | Yes, within SQS retention | No | AWS account, queue permissions, region/endpoint configuration | -| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in a retained relational partition log | Yes, while records remain retained | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | +| ADO.NET partitioned stream | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational stream partitions | Yes, while records remain retained | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | | NATS JetStream | [`Microsoft.Orleans.Streaming.NATS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.NATS) | **Alpha** | Configurable; file storage is the default | No | NATS server with JetStream and sufficient storage; subject/stream administration | | Redis Streams | [`Microsoft.Orleans.Streaming.Redis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Redis) | **Alpha** | Configurable through Redis persistence and stream retention | Yes, while entries remain | Redis deployment, persistence/HA policy, and retention sizing | @@ -68,9 +68,9 @@ Register [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/lates Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/welcome.html) with . Standard queues provide at-least-once delivery, while FIFO queues preserve ordering within each Orleans stream. SQS redelivers after the [visibility timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) when processing isn't acknowledged. See [Stream with Amazon SQS](sqs-streaming.md) for standard and FIFO configuration, custom data adapters, permissions, and operational guidance. -## ADO.NET streaming (alpha) +## ADO.NET partitioned stream provider (alpha) -Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue is an immutable, ordered partition log with a durable, ownership-fenced checkpoint. The provider is rewindable while records remain retained and resumes strictly after the checkpoint. retains checkpointed messages for one day by default. can impose a hard storage ceiling, but it can delete unprocessed messages and therefore produces gap diagnostics. Cleanup is bounded by . can fault one failing subscription without deleting or mutating the multicast source record. +Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline is rewindable while records remain retained and resumes strictly after the checkpoint. retains checkpointed records for one day by default. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Cleanup is bounded by . can fault one failing subscription while preserving the multicast partition record. The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs index 713fff77e5d..46d9aaacb33 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs @@ -4,8 +4,8 @@ namespace Orleans.Streaming.AdoNet; /// The implementation for the ADONET provider. /// /// -/// 1. This class only supports binary serialization as performance and data size is the priority for database storage. -/// 2. Though the is supported here, it is not yet used, as the ADO.NET provider is not rewindable. +/// This class uses binary serialization to prioritize database storage performance and payload size. +/// Its identifies a record position in the rewindable partition history. /// [GenerateSerializer] [Alias("Orleans.Streaming.AdoNet.AdoNetBatchContainer")] diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index 09c3aadd502..f9decb20b7a 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -44,7 +44,7 @@ public AdoNetQueueAdapter( public string Name { get; } /// - /// The retained-log ADO.NET provider supports replay from cached and durable checkpoints. + /// The ADO.NET partitioned stream provider supports replay from cached and durable checkpoints. /// public bool IsRewindable => true; diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index ba390f792b8..92e9bd332ab 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -3,7 +3,7 @@ namespace Orleans.Streaming.AdoNet; /// -/// Receives message batches from an individual retained-log partition. +/// Receives records from one stream partition through the recoverable stream partition pipeline. /// internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueueCache { diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index f288dbd50dc..ab9388446de 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -132,13 +132,13 @@ private void ThrowIfRetentionGap(AdoNetStreamPartitionState state) { throw new DataNotAvailableException( $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' has a retention gap: " - + $"checkpoint {checkpoint}, earliest retained message {earliest}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); + + $"checkpoint {checkpoint}, earliest retained record {earliest}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); } } [LoggerMessage( Level = LogLevel.Warning, - Message = "Hard stream retention deleted {DeletedCount} messages for {ServiceId}/{ProviderId}/{QueueId} from {DeletedFrom} through {DeletedThrough}, crossing checkpoint {Checkpoint}.")] + Message = "Hard stream retention deleted {DeletedCount} records for {ServiceId}/{ProviderId}/{QueueId} from {DeletedFrom} through {DeletedThrough}, crossing checkpoint {Checkpoint}.")] private static partial void LogHardRetentionCrossed( ILogger logger, string serviceId, diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs index b55c620c674..00fb1027756 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs @@ -1,7 +1,7 @@ namespace Orleans.Streaming.AdoNet; /// -/// Describes one bounded retained-log cleanup operation. +/// Describes one bounded stream partition retention cleanup operation. /// internal record AdoNetStreamCleanupResult( bool Ran, diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs index cb61ade02e7..b94c945002f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs @@ -1,7 +1,7 @@ namespace Orleans.Streaming.AdoNet; /// -/// Logs subscriber failures without mutating the shared retained log. +/// Logs subscriber failures while preserving shared partition records. /// internal partial class AdoNetStreamFailureHandler : IStreamFailureHandler { @@ -50,7 +50,7 @@ public Task OnSubscriptionFailure( [LoggerMessage( Level = LogLevel.Warning, - Message = "ADO.NET stream delivery failed for subscription {SubscriptionId} on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The retained source record was not modified.")] + Message = "ADO.NET stream delivery failed for subscription {SubscriptionId} on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The partition record remains available.")] private static partial void LogDeliveryFailure( ILogger logger, GuidId subscriptionId, @@ -60,7 +60,7 @@ private static partial void LogDeliveryFailure( [LoggerMessage( Level = LogLevel.Warning, - Message = "ADO.NET stream subscription {SubscriptionId} failed on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The retained source record was not modified.")] + Message = "ADO.NET stream subscription {SubscriptionId} failed on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The partition record remains available.")] private static partial void LogSubscriptionFailure( ILogger logger, GuidId subscriptionId, diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index e0e2a97e6a2..42287d57c24 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -28,10 +28,10 @@ public class AdoNetStreamOptions public DbDataSource? DataSource { get; set; } /// - /// Gets or sets a value indicating whether a new partition checkpoint starts at the retained log tail. + /// Gets or sets a value indicating whether a new partition checkpoint starts at the current partition history tail. /// /// - /// When , a new checkpoint starts immediately before the earliest retained message. + /// When , a new checkpoint starts immediately before the earliest retained record. /// This setting is only used while initializing a partition which does not have a checkpoint. /// public bool StartFromNow { get; set; } @@ -42,7 +42,7 @@ public class AdoNetStreamOptions public bool FaultOnDeliveryFailure { get; set; } /// - /// Gets or sets the maximum number of messages returned by a partition read. + /// Gets or sets the maximum number of stream records returned by a partition read. /// public int MaxMessagesPerRead { get; set; } = 1_000; @@ -52,7 +52,7 @@ public class AdoNetStreamOptions public TimeSpan CheckpointPersistInterval { get; set; } = TimeSpan.FromSeconds(5); /// - /// Gets or sets the minimum amount of time that a message is retained after it is checkpointed. + /// Gets or sets the minimum amount of time that a stream record is retained after it is checkpointed. /// public TimeSpan RetentionPeriod { get; set; } = TimeSpan.FromDays(1); @@ -60,7 +60,7 @@ public class AdoNetStreamOptions /// Gets or sets an optional hard retention ceiling. /// /// - /// Messages older than this value can be deleted even when they are newer than the checkpoint. + /// Stream records older than this value can be deleted even when they are newer than the checkpoint. /// Such deletions are reported by storage so that the receiver can emit gap diagnostics. /// public TimeSpan? MaximumRetentionPeriod { get; set; } @@ -71,7 +71,7 @@ public class AdoNetStreamOptions public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(1); /// - /// Gets or sets the maximum number of messages deleted by one cleanup operation. + /// Gets or sets the maximum number of stream records deleted by one cleanup operation. /// public int CleanupBatchSize { get; set; } = 1_000; diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index 3dd569259db..ec6af26f36e 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -1,7 +1,7 @@ # Microsoft Orleans Streaming for ADO.NET ## Introduction -Microsoft Orleans Streaming for ADO.NET provides a stream provider implementation for Orleans using ADO.NET-compatible databases (SQL Server, MySQL, PostgreSQL, etc.). This allows for publishing and subscribing to streams of events with relational databases as the underlying infrastructure. +Microsoft Orleans Streaming for ADO.NET provides a partitioned stream provider for Orleans using ADO.NET-compatible databases (SQL Server, MySQL, PostgreSQL, etc.). This allows for publishing and subscribing to streams of events with relational databases as the underlying infrastructure. ## Getting Started To use this package, install it via NuGet: @@ -48,17 +48,17 @@ var builder = Host.CreateApplicationBuilder(args) await builder.RunAsync(); ``` -The provider stores each queue as an immutable partition log. Configure retained-log behavior with: +The provider stores each queue as an immutable, ordered stream partition. Its stream partition pipeline appends records, reads partition history, and advances an ownership-fenced checkpoint. Configure it with: -- `StartFromNow`: initialize a new checkpoint at the retained tail instead of before the earliest retained message. -- `FaultOnDeliveryFailure`: optionally fault a failing subscription without altering the shared retained log. +- `StartFromNow`: initialize a new checkpoint at the current partition history tail instead of before the earliest retained record. +- `FaultOnDeliveryFailure`: optionally fault a failing subscription while preserving the shared partition records. - `MaxMessagesPerRead`: bound each ordered storage read. - `CheckpointPersistInterval`: throttle durable checkpoint updates. -- `RetentionPeriod`: retain checkpointed messages for at least this period (one day by default). -- `MaximumRetentionPeriod`: optionally delete older messages even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed delivery gap. +- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). +- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. - `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. -The provider is rewindable while requested records remain retained. It resumes strictly after its durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. +The partitioned stream provider is rewindable while requested records remain retained. It resumes strictly after its durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. ## Alpha schema upgrade diff --git a/src/AdoNet/Shared/Storage/DbStoredQueries.cs b/src/AdoNet/Shared/Storage/DbStoredQueries.cs index ec2d08c80ca..f16ddf20fce 100644 --- a/src/AdoNet/Shared/Storage/DbStoredQueries.cs +++ b/src/AdoNet/Shared/Storage/DbStoredQueries.cs @@ -94,7 +94,7 @@ private static void ValidateStreamingSchema(Dictionary queries) ? string.Join(", ", legacyKeys) : "none"; throw new InvalidOperationException( - $"The ADO.NET streaming schema is incompatible. Expected immutable partition-log schema version {CurrentStreamingSchemaVersion}, " + + $"The ADO.NET streaming schema is incompatible. Expected stream partition schema version {CurrentStreamingSchemaVersion}, " + $"but detected '{detectedVersion}'. Missing query keys: {missingDescription}. Legacy query keys: {legacyDescription}. " + "This alpha schema has no in-place migration. Drop the legacy OrleansStreamMessage, " + "OrleansStreamDeadLetter, OrleansStreamControl, and OrleansStreamMessageSequence objects, drop OrleansStreamPartition if it " + @@ -205,12 +205,12 @@ private static void ValidateStreamingSchema(Dictionary queries) #if STREAMING_ADONET || TESTER_SQLUTILS /// - /// The immutable partition-log schema version marker. + /// The stream partition schema version marker. /// internal string StreamSchemaVersionKey => queries[nameof(StreamSchemaVersionKey)]; /// - /// A query template to append a message to a stream partition. + /// A query template to append a record to a stream partition. /// internal string AppendStreamMessageKey => queries[nameof(AppendStreamMessageKey)]; @@ -220,7 +220,7 @@ private static void ValidateStreamingSchema(Dictionary queries) internal string AcquireStreamPartitionKey => queries[nameof(AcquireStreamPartitionKey)]; /// - /// A query template to read ordered messages after an exclusive position. + /// A query template to read ordered stream records after an exclusive position. /// internal string ReadStreamMessagesKey => queries[nameof(ReadStreamMessagesKey)]; @@ -230,12 +230,12 @@ private static void ValidateStreamingSchema(Dictionary queries) internal string AdvanceStreamCheckpointKey => queries[nameof(AdvanceStreamCheckpointKey)]; /// - /// A query template to read partition checkpoint and retained bounds. + /// A query template to read a partition checkpoint and partition history bounds. /// internal string GetStreamPartitionBoundsKey => queries[nameof(GetStreamPartitionBoundsKey)]; /// - /// A query template to perform bounded retained-log cleanup. + /// A query template to perform bounded stream partition retention cleanup. /// internal string CleanupStreamMessagesKey => queries[nameof(CleanupStreamMessagesKey)]; diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index 805dace9a94..3f1ab4719ff 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -452,7 +452,7 @@ private static MembershipTableData ConvertToMembershipTableData(IEnumerable - /// Appends an immutable message to a stream partition. + /// Appends an immutable stream record to a stream partition. /// /// The service identifier. /// The provider identifier. @@ -532,7 +532,7 @@ internal Task AcquireStreamPartitionAsync( } /// - /// Reads retained messages with identifiers strictly greater than . + /// Reads retained stream records with identifiers strictly greater than . /// internal Task> ReadStreamMessagesAsync( string serviceId, @@ -606,7 +606,7 @@ internal Task> ReadStreamMessagesAsync( } /// - /// Reads the current checkpoint, ownership epoch, and retained message bounds. + /// Reads the current checkpoint, ownership epoch, and retained bounds of partition history. /// internal Task GetStreamPartitionBoundsAsync( string serviceId, @@ -637,7 +637,7 @@ internal Task> ReadStreamMessagesAsync( } /// - /// Deletes an ordered, bounded batch of retained messages. + /// Deletes an ordered, bounded batch of retained stream records. /// internal Task CleanupStreamMessagesAsync( string serviceId, diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 6f8329e32a2..10f012869a0 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -9,7 +9,7 @@ namespace Orleans.Providers.Streams.Common { /// - /// Coordinates a recoverable partition source, pooled cache, and durable checkpoint. + /// Coordinates a recoverable stream partition pipeline comprising a partition source, pooled cache, and durable checkpoint. /// /// The source record type. public sealed class RecoverableStreamReceiver : IQueueAdapterReceiver, IQueueCache diff --git a/test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs similarity index 88% rename from test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs rename to test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs index a3249fe6de0..d7d7df3c95e 100644 --- a/test/Benchmarks.AdoNet/Streaming/RetainedLogAppendBenchmark.cs +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs @@ -7,7 +7,7 @@ namespace Benchmarks.AdoNet.Streaming; -public class SqlServerRetainedLogAppendBenchmark() : RetainedLogAppendBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +public class SqlServerStreamPartitionAppendBenchmark() : StreamPartitionAppendBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") { public override void GlobalSetup() { @@ -17,11 +17,11 @@ public override void GlobalSetup() } /// -/// Measures immutable log append throughput while varying payload size, concurrency, and partition contention. +/// Measures stream partition append throughput while varying payload size, concurrency, and partition contention. /// A partition count of one is the lock-wait baseline for database-side wait telemetry. /// [WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class RetainedLogAppendBenchmark(string invariant, string database) +public abstract class StreamPartitionAppendBenchmark(string invariant, string database) { private const int OperationsPerInvoke = 1_000; diff --git a/test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs similarity index 94% rename from test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs rename to test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs index 553fe4655a8..dfef147dbf8 100644 --- a/test/Benchmarks.AdoNet/Streaming/RetainedLogReadCheckpointBenchmark.cs +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs @@ -8,7 +8,7 @@ namespace Benchmarks.AdoNet.Streaming; -public class SqlServerRetainedLogReadCheckpointBenchmark() : RetainedLogReadCheckpointBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +public class SqlServerStreamPartitionReadCheckpointBenchmark() : StreamPartitionReadCheckpointBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") { public override void GlobalSetup() { @@ -21,7 +21,7 @@ public override void GlobalSetup() /// Measures exclusive ordered reads and epoch-fenced checkpoint updates across partitions. /// [WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class RetainedLogReadCheckpointBenchmark(string invariant, string database) +public abstract class StreamPartitionReadCheckpointBenchmark(string invariant, string database) { private const int OperationsPerInvoke = 1_000; @@ -126,7 +126,7 @@ await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => } } -public class SqlServerRetainedLogCleanupBenchmark() : RetainedLogCleanupBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +public class SqlServerStreamPartitionCleanupBenchmark() : StreamPartitionCleanupBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") { public override void GlobalSetup() { @@ -139,7 +139,7 @@ public override void GlobalSetup() /// Measures bounded cleanup sweeps while varying partition count, batch size, and eligible-row density. /// [WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class RetainedLogCleanupBenchmark(string invariant, string database) +public abstract class StreamPartitionCleanupBenchmark(string invariant, string database) { private const int MessagesPerPartition = 1_000; @@ -241,7 +241,7 @@ INNER JOIN Ranked [Benchmark] [BenchmarkCategory("Cleanup", "CleanupImpact")] - public async Task CleanupRetainedPartitions() + public async Task CleanupStreamPartitions() { var results = await Task.WhenAll(_partitionIds.Select(partitionId => _queries.CleanupStreamMessagesAsync( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs similarity index 97% rename from test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs rename to test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index ffc527b430e..dee4026d09b 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionLogTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -13,12 +13,12 @@ namespace Tester.AdoNet.Streaming; /// -/// Tests the immutable partition-log storage layer via against Sql Server. +/// Tests the stream partition storage layer via against Sql Server. /// [TestCategory("SqlServer"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] [TestProvider("SqlServer")] [TestSuite("Functional")] -public class SqlServerAdoNetStreamPartitionLogTests() : AdoNetStreamPartitionLogTests(AdoNetInvariants.InvariantNameSqlServer) +public class SqlServerAdoNetStreamPartitionTests() : AdoNetStreamPartitionTests(AdoNetInvariants.InvariantNameSqlServer) { /// /// Concurrent appends to the same partition must be serialized by the partition-row lock, and a @@ -41,14 +41,14 @@ public Task AppendStreamMessage_DifferentPartitionsDoNotShareTheAllocationLock() } /// -/// Tests the immutable partition-log storage layer via against MySQL. +/// Tests the stream partition storage layer via against MySQL. /// [TestCategory("MySql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] [TestProvider("MySql")] [TestSuite("Functional")] -public class MySqlAdoNetStreamPartitionLogTests : AdoNetStreamPartitionLogTests +public class MySqlAdoNetStreamPartitionTests : AdoNetStreamPartitionTests { - public MySqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.InvariantNameMySql) + public MySqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantNameMySql) { MySqlConnection.ClearAllPools(); } @@ -58,14 +58,14 @@ public MySqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.InvariantNam } /// -/// Tests the immutable partition-log storage layer via against PostgreSQL. +/// Tests the stream partition storage layer via against PostgreSQL. /// [TestCategory("PostgreSql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] [TestProvider("PostgreSql")] [TestSuite("Functional")] -public class PostgreSqlAdoNetStreamPartitionLogTests : AdoNetStreamPartitionLogTests +public class PostgreSqlAdoNetStreamPartitionTests : AdoNetStreamPartitionTests { - public PostgreSqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.InvariantNamePostgreSql) + public PostgreSqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantNamePostgreSql) { NpgsqlConnection.ClearAllPools(); } @@ -75,7 +75,7 @@ public PostgreSqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.Invaria } /// -/// Tests the immutable partition-log storage layer via : transactional +/// Tests the stream partition storage layer via : transactional /// append with rollback-safe allocation, exclusive ordered reads, epoch-fenced monotonic checkpoints, /// bounded retention/cleanup with hard-ceiling diagnostics, partition independence, and explicit schema /// version/query-key enforcement. @@ -83,7 +83,7 @@ public PostgreSqlAdoNetStreamPartitionLogTests() : base(AdoNetInvariants.Invaria [TestCategory("AdoNet"), TestCategory("Streaming")] [TestSuite("Functional")] [TestArea("Streaming")] -public abstract class AdoNetStreamPartitionLogTests(string invariant) : IAsyncLifetime +public abstract class AdoNetStreamPartitionTests(string invariant) : IAsyncLifetime { private const string TestDatabaseName = "OrleansStreamTest"; @@ -745,11 +745,11 @@ await Assert.ThrowsAsync(() => #region Explicit schema mismatch /// - /// An old or partially-migrated schema that is missing partition-log query keys must fail + /// An old or partially-migrated schema that is missing stream partition query keys must fail /// explicitly and name the missing keys, rather than fail lazily or silently on first use. /// [SkippableFact] - public async Task CreateInstance_MissingPartitionLogQueryKeys_FailsExplicitly() + public async Task CreateInstance_MissingStreamPartitionQueryKeys_FailsExplicitly() { await _storage.ExecuteAsync( "DELETE FROM OrleansQuery WHERE QueryKey IN ('AppendStreamMessageKey', 'StreamSchemaVersionKey')", @@ -763,7 +763,7 @@ await _storage.ExecuteAsync( } [SkippableFact] - public async Task CreateInstance_MixedLegacyAndPartitionLogQueryKeys_FailsExplicitly() + public async Task CreateInstance_MixedLegacyAndStreamPartitionQueryKeys_FailsExplicitly() { await _storage.ExecuteAsync( "INSERT INTO OrleansQuery (QueryKey, QueryText) VALUES ('QueueStreamMessageKey', 'legacy')", diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 5aab98d496f..0fc9d66b876 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -10,7 +10,7 @@ public sealed class AdoNetStreamSchemaTests [InlineData("SQLServer")] [InlineData("PostgreSQL")] [InlineData("MySQL")] - public void SchemaDefinesVersionedImmutablePartitionLog(string provider) + public void SchemaDefinesVersionedStreamPartitions(string provider) { var script = ReadScript(provider); From 006662bf23b4d39f80f02a9b6a0aeca97db67248 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 02:33:28 -0700 Subject: [PATCH 15/90] fix(streaming): reconcile ADO.NET checkpoint conflicts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3100b74-af89-4744-ac56-b00e5d22a4ed --- .../AdoNetRecoverableStream.cs | 26 +++++--- .../Streaming/AdoNetRecoverableStreamTests.cs | 46 ++++++++++++++ .../Streaming/AdoNetStreamPartitionTests.cs | 61 +++++++++++++++++++ 3 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index ab9388446de..5f6a2783d25 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -33,6 +33,11 @@ public async ValueTask Update( string expectedVersion, CancellationToken cancellationToken) { + if (_partition is not { } partition) + { + throw new InvalidOperationException("The ADO.NET stream partition checkpoint must be loaded before it can be updated."); + } + var checkpointValue = long.Parse(checkpoint, NumberStyles.None, CultureInfo.InvariantCulture); var ownerEpoch = long.Parse(expectedVersion, NumberStyles.None, CultureInfo.InvariantCulture); var update = await queries.AdvanceStreamCheckpointAsync( @@ -41,21 +46,26 @@ public async ValueTask Update( queueId, ownerEpoch, checkpointValue).WaitAsync(cancellationToken); - if (update is not null) - { - if (update.OwnerEpoch != ownerEpoch) - { - throw new InvalidOperationException( - $"ADO.NET stream partition ownership was lost for '{serviceId}/{providerId}/{queueId}' at epoch {ownerEpoch}. The stale receiver cannot advance its checkpoint."); - } + return ResolveCheckpointUpdate( + $"{serviceId}/{providerId}/{queueId}", + partition.OwnerEpoch, + update); + } + internal static StreamCheckpointStoreState ResolveCheckpointUpdate( + string partitionId, + long acquiredOwnerEpoch, + AdoNetStreamCheckpointUpdate? update) + { + if (update is not null && update.OwnerEpoch == acquiredOwnerEpoch) + { return new( (update.Checkpoint ?? 0).ToString(CultureInfo.InvariantCulture), update.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); } throw new InvalidOperationException( - $"ADO.NET stream partition ownership was lost for '{serviceId}/{providerId}/{queueId}' at epoch {ownerEpoch}. The stale receiver cannot advance its checkpoint."); + $"ADO.NET stream partition ownership was lost for '{partitionId}' at epoch {acquiredOwnerEpoch}. The stale receiver cannot advance its checkpoint."); } public Task Initialize( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs new file mode 100644 index 00000000000..03f675442da --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -0,0 +1,46 @@ +using Orleans.Streaming.AdoNet; + +namespace Tester.AdoNet.Streaming; + +[TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +public class AdoNetRecoverableStreamTests +{ + [Fact] + public void ResolveCheckpointUpdate_ReturnsAuthoritativeStateForExpectedVersionConflict() + { + var update = new AdoNetStreamCheckpointUpdate( + "service", + "provider", + "queue", + OwnerEpoch: 7, + Checkpoint: 42, + Updated: false); + + var result = AdoNetRecoverableStream.ResolveCheckpointUpdate("service/provider/queue", 7, update); + + Assert.Equal("42", result.Checkpoint); + Assert.Equal("7", result.Version); + } + + [Fact] + public void ResolveCheckpointUpdate_ThrowsWhenPartitionOwnershipIsLost() + { + var update = new AdoNetStreamCheckpointUpdate( + "service", + "provider", + "queue", + OwnerEpoch: 8, + Checkpoint: 42, + Updated: false); + + var exception = Assert.Throws( + () => AdoNetRecoverableStream.ResolveCheckpointUpdate("service/provider/queue", 7, update)); + + Assert.Contains("ownership was lost", exception.Message); + Assert.Contains("service/provider/queue", exception.Message); + Assert.Contains("epoch 7", exception.Message); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index dee4026d09b..0e2cc968e23 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -1,10 +1,13 @@ using System.Data; using System.Data.Common; +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Data.SqlClient; using MySql.Data.MySqlClient; using Npgsql; using Orleans.Configuration; using Orleans.Runtime; +using Orleans.Streams; using Orleans.Streaming.AdoNet; using Orleans.Streaming.AdoNet.Storage; using UnitTests.General; @@ -520,6 +523,64 @@ public async Task AdvanceStreamCheckpoint_FencesStaleOwnerEpochAfterReacquisitio Assert.Equal(2, currentAttempt.Checkpoint); } + [SkippableFact] + public async Task RecoverableStreamUpdate_ReturnsPersistedStateWhenExpectedVersionConflicts() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + IStreamCheckpointStore store = new AdoNetRecoverableStream( + serviceId, + providerId, + queueId, + new AdoNetStreamOptions(), + _queries, + NullLogger.Instance); + var loaded = await store.Load(CancellationToken.None); + Assert.Equal("0", loaded.Checkpoint); + Assert.Equal("1", loaded.Version); + + var conflict = await store.Update("1", expectedVersion: "2", CancellationToken.None); + + Assert.Equal(loaded.Checkpoint, conflict.Checkpoint); + Assert.Equal(loaded.Version, conflict.Version); + + var updated = await store.Update("1", conflict.Version, CancellationToken.None); + Assert.Equal("1", updated.Checkpoint); + Assert.Equal(loaded.Version, updated.Version); + } + + [SkippableFact] + public async Task RecoverableStreamUpdate_ThrowsWhenPartitionOwnershipIsLost() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + IStreamCheckpointStore store = new AdoNetRecoverableStream( + serviceId, + providerId, + queueId, + new AdoNetStreamOptions(), + _queries, + NullLogger.Instance); + var loaded = await store.Load(CancellationToken.None); + var newOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + Assert.NotEqual(long.Parse(loaded.Version, CultureInfo.InvariantCulture), newOwner.OwnerEpoch); + + var exception = await Assert.ThrowsAsync( + () => store.Update("1", loaded.Version, CancellationToken.None).AsTask()); + + Assert.Contains("ownership was lost", exception.Message); + } + [SkippableFact] public async Task AdvanceStreamCheckpoint_ReturnsNullForUnknownPartition() { From 005b2457a893190841cc17925fe9d2382215e096 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 22:46:43 -0700 Subject: [PATCH 16/90] test(streaming): adopt xunit v3 conventions --- .../Streaming/AdoNetStreamPartitionTests.cs | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index 0e2cc968e23..7f848fc76c6 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -27,18 +27,18 @@ public class SqlServerAdoNetStreamPartitionTests() : AdoNetStreamPartitionTests( /// Concurrent appends to the same partition must be serialized by the partition-row lock, and a /// rolled-back append must not permanently burn its allocated message identifier. /// - [SkippableFact] + [Fact] public Task AppendStreamMessage_ConcurrentAppends_AreSerializedAndRollbackDoesNotBurnIds() => VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoesNotBurnIds(); /// /// A reader must never observe a message allocated by a transaction that has not yet committed. /// - [SkippableFact] + [Fact] public Task ReadStreamMessages_ExcludesUncommittedInFlightAppend() => VerifySqlServerReadExcludesUncommittedInFlightAppend(); - [SkippableFact] + [Fact] public Task AppendStreamMessage_DifferentPartitionsDoNotShareTheAllocationLock() => VerifySqlServerPartitionsAreIndependent(); } @@ -56,7 +56,7 @@ public MySqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantNameMy MySqlConnection.ClearAllPools(); } - [SkippableFact] + [Fact] public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyMySqlRollbackRestoresAllocation(); } @@ -73,7 +73,7 @@ public PostgreSqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantN NpgsqlConnection.ClearAllPools(); } - [SkippableFact] + [Fact] public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyPostgreSqlRollbackRestoresAllocation(); } @@ -93,16 +93,16 @@ public abstract class AdoNetStreamPartitionTests(string invariant) : IAsyncLifet private IRelationalStorage _storage = null!; private RelationalOrleansQueries _queries = null!; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { var testing = await RelationalStorageForTesting.SetupInstance(invariant, TestDatabaseName); - Skip.If(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); + Assert.SkipWhen(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; #region Helpers @@ -175,7 +175,7 @@ private static void AddParameter(IDbCommand command, string name, object value) #region Append: sequential allocation and partition independence - [SkippableFact] + [Fact] public async Task AppendStreamMessage_AllocatesSequentialMessageIdsPerPartition() { var serviceId = RandomServiceId(); @@ -191,7 +191,7 @@ public async Task AppendStreamMessage_AllocatesSequentialMessageIdsPerPartition( Assert.Equal(3, third.MessageId); } - [SkippableFact] + [Fact] public async Task AppendStreamMessage_ConcurrentAppendsAreGapFree() { var serviceId = RandomServiceId(); @@ -204,7 +204,7 @@ public async Task AppendStreamMessage_ConcurrentAppendsAreGapFree() Assert.Equal(Enumerable.Range(1, count).Select(static value => (long)value), results.Select(static result => result.MessageId).Order()); } - [SkippableFact] + [Fact] public async Task AppendStreamMessage_PartitionsAreIndependent() { var serviceId = RandomServiceId(); @@ -228,7 +228,7 @@ public async Task AppendStreamMessage_PartitionsAreIndependent() Assert.Equal(1, boundsB.EarliestMessageId); } - [SkippableFact] + [Fact] public async Task AppendStreamMessage_ValidatesStreamNamespaceBoundary() { var serviceId = RandomServiceId(); @@ -264,7 +264,7 @@ await Assert.ThrowsAsync(() => Assert.Equal(2, maxNamespaceAck.MessageId); } - [SkippableFact] + [Fact] public async Task AppendStreamMessage_PersistsCanonicalStreamIdFullKeyAndNamespaceBoundary() { var serviceId = RandomServiceId(); @@ -290,7 +290,7 @@ public async Task AppendStreamMessage_PersistsCanonicalStreamIdFullKeyAndNamespa #region Acquire: ownership, epoch, and checkpoint initialization - [SkippableFact] + [Fact] public async Task AcquireStreamPartition_InitializesCheckpointBeforeEarliestWhenNotStartingFromNow() { var serviceId = RandomServiceId(); @@ -308,7 +308,7 @@ public async Task AcquireStreamPartition_InitializesCheckpointBeforeEarliestWhen Assert.Equal(2, state.TailMessageId); } - [SkippableFact] + [Fact] public async Task AcquireStreamPartition_InitializesCheckpointAtTailWhenStartingFromNow() { var serviceId = RandomServiceId(); @@ -324,7 +324,7 @@ public async Task AcquireStreamPartition_InitializesCheckpointAtTailWhenStarting Assert.Equal(last.MessageId, state.TailMessageId); } - [SkippableFact] + [Fact] public async Task AcquireStreamPartition_OnNeverAppendedPartitionHasNoBounds() { var serviceId = RandomServiceId(); @@ -339,7 +339,7 @@ public async Task AcquireStreamPartition_OnNeverAppendedPartitionHasNoBounds() Assert.Null(state.TailMessageId); } - [SkippableFact] + [Fact] public async Task AcquireStreamPartition_ReacquisitionIncrementsOwnerEpochAndPreservesCheckpoint() { var serviceId = RandomServiceId(); @@ -357,7 +357,7 @@ public async Task AcquireStreamPartition_ReacquisitionIncrementsOwnerEpochAndPre Assert.Equal(1, second.Checkpoint); // preserved from the prior owner, not re-initialized } - [SkippableFact] + [Fact] public async Task AcquireStreamPartition_OwnerEpochAndCheckpointArePerPartition() { var serviceId = RandomServiceId(); @@ -383,7 +383,7 @@ public async Task AcquireStreamPartition_OwnerEpochAndCheckpointArePerPartition( #region Read: exclusive ordered ranges - [SkippableFact] + [Fact] public async Task ReadStreamMessages_ReturnsExclusiveOrderedRangeRespectingMaxCount() { var serviceId = RandomServiceId(); @@ -402,7 +402,7 @@ public async Task ReadStreamMessages_ReturnsExclusiveOrderedRangeRespectingMaxCo Assert.Equal(page.OrderBy(m => m.MessageId).Select(m => m.MessageId), page.Select(m => m.MessageId)); } - [SkippableFact] + [Fact] public async Task ReadStreamMessages_EmptyWhenAfterMessageIdIsAtOrBeyondTail() { var serviceId = RandomServiceId(); @@ -418,7 +418,7 @@ public async Task ReadStreamMessages_EmptyWhenAfterMessageIdIsAtOrBeyondTail() Assert.Empty(beyondTail); } - [SkippableFact] + [Fact] public async Task ReadStreamMessages_ValidatesArgumentBounds() { var serviceId = RandomServiceId(); @@ -446,7 +446,7 @@ await Assert.ThrowsAsync(() => #region Checkpoint: monotonicity, non-regression, and epoch fencing - [SkippableFact] + [Fact] public async Task AdvanceStreamCheckpoint_AdvancesMonotonicallyAndRejectsRegression() { var serviceId = RandomServiceId(); @@ -479,7 +479,7 @@ public async Task AdvanceStreamCheckpoint_AdvancesMonotonicallyAndRejectsRegress Assert.Equal(3, advanceTo3.Checkpoint); } - [SkippableFact] + [Fact] public async Task AdvanceStreamCheckpoint_RejectsCheckpointAtOrBeyondTail() { var serviceId = RandomServiceId(); @@ -497,7 +497,7 @@ public async Task AdvanceStreamCheckpoint_RejectsCheckpointAtOrBeyondTail() Assert.False(result!.Updated); } - [SkippableFact] + [Fact] public async Task AdvanceStreamCheckpoint_FencesStaleOwnerEpochAfterReacquisition() { var serviceId = RandomServiceId(); @@ -523,7 +523,7 @@ public async Task AdvanceStreamCheckpoint_FencesStaleOwnerEpochAfterReacquisitio Assert.Equal(2, currentAttempt.Checkpoint); } - [SkippableFact] + [Fact] public async Task RecoverableStreamUpdate_ReturnsPersistedStateWhenExpectedVersionConflicts() { var serviceId = RandomServiceId(); @@ -554,7 +554,7 @@ public async Task RecoverableStreamUpdate_ReturnsPersistedStateWhenExpectedVersi Assert.Equal(loaded.Version, updated.Version); } - [SkippableFact] + [Fact] public async Task RecoverableStreamUpdate_ThrowsWhenPartitionOwnershipIsLost() { var serviceId = RandomServiceId(); @@ -581,7 +581,7 @@ public async Task RecoverableStreamUpdate_ThrowsWhenPartitionOwnershipIsLost() Assert.Contains("ownership was lost", exception.Message); } - [SkippableFact] + [Fact] public async Task AdvanceStreamCheckpoint_ReturnsNullForUnknownPartition() { var result = await _queries.AdvanceStreamCheckpointAsync(RandomServiceId(), RandomProviderId(), RandomQueueId(), ownerEpoch: 1, checkpoint: 1); @@ -589,7 +589,7 @@ public async Task AdvanceStreamCheckpoint_ReturnsNullForUnknownPartition() Assert.Null(result); } - [SkippableFact] + [Fact] public async Task AdvanceStreamCheckpoint_ValidatesArgumentBounds() { var serviceId = RandomServiceId(); @@ -617,7 +617,7 @@ await Assert.ThrowsAsync(() => #region Bounds: partition state reporting - [SkippableFact] + [Fact] public async Task GetStreamPartitionBounds_ReflectsCurrentState() { var serviceId = RandomServiceId(); @@ -644,7 +644,7 @@ public async Task GetStreamPartitionBounds_ReflectsCurrentState() #region Cleanup: retention, hard ceiling diagnostics, batching, and throttling - [SkippableFact] + [Fact] public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentionElapses() { var serviceId = RandomServiceId(); @@ -682,7 +682,7 @@ public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentio Assert.Empty(remaining); } - [SkippableFact] + [Fact] public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics() { var serviceId = RandomServiceId(); @@ -715,7 +715,7 @@ public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics( Assert.Null(result.Checkpoint); } - [SkippableFact] + [Fact] public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() { var serviceId = RandomServiceId(); @@ -753,7 +753,7 @@ public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() Assert.Empty(remaining); } - [SkippableFact] + [Fact] public async Task CleanupStreamMessages_ThrottlesRepeatedRunsWithinInterval() { var serviceId = RandomServiceId(); @@ -774,7 +774,7 @@ public async Task CleanupStreamMessages_ThrottlesRepeatedRunsWithinInterval() Assert.Equal(0, second.DeletedCount); } - [SkippableFact] + [Fact] public async Task CleanupStreamMessages_ValidatesArgumentBounds() { var serviceId = RandomServiceId(); @@ -809,7 +809,7 @@ await Assert.ThrowsAsync(() => /// An old or partially-migrated schema that is missing stream partition query keys must fail /// explicitly and name the missing keys, rather than fail lazily or silently on first use. /// - [SkippableFact] + [Fact] public async Task CreateInstance_MissingStreamPartitionQueryKeys_FailsExplicitly() { await _storage.ExecuteAsync( @@ -823,7 +823,7 @@ await _storage.ExecuteAsync( Assert.Contains("StreamSchemaVersionKey", exception.Message, StringComparison.Ordinal); } - [SkippableFact] + [Fact] public async Task CreateInstance_MixedLegacyAndStreamPartitionQueryKeys_FailsExplicitly() { await _storage.ExecuteAsync( From 91b9d5fc28058b61eef308426afacd0133317f85 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 01:26:12 -0700 Subject: [PATCH 17/90] fix(streaming): release recoverable cache buffers --- .../RecoverableStreamQueueCache.cs | 11 ++++++ .../RecoverableStreamReceiverTests.cs | 39 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 0a38d710bfb..b11afd62a01 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -191,6 +191,17 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke /// public void Dispose() { + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest) + { + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); + } + + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + _currentBuffer = null; _evictionStrategy.OnPurged = null; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index d122669a5cf..cfe5c860319 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -170,6 +170,37 @@ [new TestQueueMessage(streamId, 11, "payload")], Assert.Equal(0, cache.ItemCount); } + [Fact] + public void Cache_DisposeDrainsMessagesReturnsBuffersAndAllocatesFreshBufferIfReused() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + new TestDataAdapter(), + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + _ = cache.Add([new TestQueueMessage(streamId, 11, "payload")], DateTime.UnixEpoch); + + cache.Dispose(); + + Assert.Equal(0, cache.ItemCount); + Assert.Equal(1, bufferPool.AllocateCount); + Assert.Equal(1, bufferPool.FreeCount); + + _ = cache.Add([new TestQueueMessage(streamId, 12, "next")], DateTime.UnixEpoch); + + Assert.Equal(1, cache.ItemCount); + Assert.Equal(2, bufferPool.AllocateCount); + cache.Dispose(); + Assert.Equal(2, bufferPool.FreeCount); + } + [Fact] public void Cache_AddsRawRecordsInOrderAndDecodesLazily() { @@ -504,9 +535,15 @@ private sealed class FixedFlowController(int maxAddCount) : IQueueFlowController private sealed class TrackingBufferPool : IObjectPool { + public int AllocateCount { get; private set; } + public int FreeCount { get; private set; } - public FixedSizeBuffer Allocate() => new(4 * 1024) { Pool = this }; + public FixedSizeBuffer Allocate() + { + AllocateCount++; + return new(4 * 1024) { Pool = this }; + } public void Free(FixedSizeBuffer resource) { From 36fe122d30c67efa528c83b34f84b2fded2525ef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 01:56:28 -0700 Subject: [PATCH 18/90] fix(streaming): seed subscription delivery progress --- .../PersistentStreamPullingAgent.cs | 14 +++ .../PersistentStreamPullingAgentTests.cs | 98 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index faebc3f343d..943f084f4d7 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -57,6 +57,7 @@ internal interface ITestAccessor Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount); Task ReadFromQueueWithCancellation(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount, CancellationToken cancellationToken); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); + Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); Task> GetPubSubCache(); Task AddSubscriber(StreamConsumerData consumerData); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); @@ -136,6 +137,9 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke return Task.CompletedTask; }).Unwrap(); + Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) + => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); + Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); @@ -427,6 +431,7 @@ private async Task DoHandshakeWithConsumer( StreamHandshakeToken? requestedHandshakeToken = null; StreamHandshakeToken? effectiveHandshakeToken = null; var providerDefaultRequest = false; + var effectiveStartToken = cacheToken ?? consumerData.PendingStartToken; // if not cache, then we can't get cursor and there is no reason to ask consumer for token. if (queueCache != null) { @@ -471,6 +476,7 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { + effectiveStartToken = requestedToken; consumerData.SafeDisposeCursor(logger); if (effectiveHandshakeToken is DeliveryToken || effectiveHandshakeToken is StartToken @@ -486,6 +492,11 @@ private async Task DoHandshakeWithConsumer( else { var result = queueCache.TryGetCacheCursor(consumerData.StreamId, requestedToken); + if (result.Kind == QueueCacheCursorResultKind.CacheMiss && cacheToken is not null) + { + effectiveStartToken = cacheToken; + } + consumerData.Cursor = result.Kind switch { QueueCacheCursorResultKind.Success => result.Cursor!, @@ -552,13 +563,16 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null try { var registrationToken = cacheToken ?? consumerData.PendingStartToken; + effectiveStartToken = registrationToken; consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, registrationToken); } catch (Exception) { consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, null); + effectiveStartToken = null; } } + consumerData.LastProcessedToken = effectiveStartToken ?? consumerData.LastProcessedToken; consumerData.HandshakeGeneration++; return true; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 330ee9c7b50..e2d49048a8c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4259,6 +4259,67 @@ public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparis await testAccessor.Shutdown(); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(SubscriptionStartTokenSource.Handshake, 10)] + [InlineData(SubscriptionStartTokenSource.Cache, 20)] + [InlineData(SubscriptionStartTokenSource.Pending, 30)] + public async Task DeliveryProgress_UsesEffectiveSubscriptionStartBeforeFirstBatch( + SubscriptionStartTokenSource tokenSource, + long expectedSequenceNumber) + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver: null, + queueAdapterCache, + timeProvider, + options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var handshakeToken = new EventSequenceTokenV2(10); + var consumer = new StartingConsumer( + tokenSource == SubscriptionStartTokenSource.Handshake + ? StreamHandshakeToken.CreateStartToken(handshakeToken) + : null); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + consumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + var cacheToken = tokenSource is SubscriptionStartTokenSource.Handshake or SubscriptionStartTokenSource.Cache + ? new EventSequenceTokenV2(20) + : null; + consumerData.PendingStartToken = new EventSequenceTokenV2(30); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken)); + consumerData.IsRegistered = true; + Assert.Equal(expectedSequenceNumber, consumerData.LastProcessedToken?.SequenceNumber); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + + var progress = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.Equal(expectedSequenceNumber, progress?.SequenceNumber); + await testAccessor.Shutdown(); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4461,6 +4522,43 @@ public void DeliveryProgress_RejectsNonPositiveUpdateInterval() Assert.Equal(nameof(options.DeliveryProgressUpdateInterval), exception.ParamName); } + + public enum SubscriptionStartTokenSource + { + Handshake, + Cache, + Pending, + } + + private sealed class StartingConsumer(StreamHandshakeToken? startToken) : IStreamConsumerExtension + { + public Task DeliverImmutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + + public Task DeliverMutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + + public Task DeliverBatch( + GuidId subscriptionId, + QualifiedStreamId streamId, + IBatchContainer item, + StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + + public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId) + => Task.FromResult(startToken); + } } } From 7dddf93fccae77ce8ac4684904f746a4acf6db3c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 19:47:30 -0700 Subject: [PATCH 19/90] fix(streaming): preserve recoverable progress invariants --- .../streams-implementation/index.md | 6 +- .../docs/streaming/custom-queue-adapter.md | 6 +- .../docs/streaming/stream-providers.md | 4 +- .../AdoNetQueueAdapterReceiver.cs | 34 +- .../AdoNetRecoverableStream.cs | 26 +- .../AdoNetStreamOptions.cs | 5 + .../AdoNetStreamOptionsValidator.cs | 3 +- .../AdoNetStreamTime.cs | 25 ++ src/AdoNet/Orleans.Streaming.AdoNet/README.md | 10 +- .../Storage/RelationalOrleansQueries.cs | 26 +- .../Common/PooledCache/PooledQueueCache.cs | 158 ++++++-- .../RecoverableStreamQueueCache.cs | 8 +- .../RecoverableStreamReceiver.cs | 9 +- .../PersistentStreamPullingAgent.cs | 99 ++++- .../QueueStreamDataStructures.cs | 9 + .../QueueAdapters/IQueueCache.cs | 4 +- .../IQueueCacheCursorProgress.cs | 10 + .../AdoNetOptionsValidatorTests.cs | 22 + .../Streaming/AdoNetRecoverableStreamTests.cs | 369 +++++++++++++++++ .../PersistentStreamPullingAgentTests.cs | 376 +++++++++++++++++- .../RecoverableStreamReceiverTests.cs | 317 ++++++++++++++- 21 files changed, 1417 insertions(+), 109 deletions(-) create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs create mode 100644 src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 49e0432b824..7d5c5347313 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -95,7 +95,7 @@ Subscription removal revokes in-flight handshake and delivery ownership. A termi An decouples queue reads from consumer delivery. Each subscription has an , so a slow consumer does not directly block a fast consumer at a later cursor. -The cache tracks the earliest delivery progress across active subscriptions. Purging must not remove an item still needed by any cursor. uses pressure buckets to stop or slow reads as lag grows instead of discarding undelivered events. Its default capacity is 4,096 batch containers. +The cache tracks the earliest contiguous partition position which is safe across active subscriptions. A matching record becomes safe after delivery or intentional filtering. A cursor also advances safely across records for other streams when no earlier matching delivery is pending, so a quiet stream does not pin an otherwise busy partition. Purging must not remove an item still needed by any cursor. uses pressure buckets to stop or slow reads as lag grows instead of discarding undelivered events. Its default capacity is 4,096 batch containers. ```mermaid flowchart TB @@ -111,13 +111,13 @@ flowchart TB Cache capacity is not durability. The queue remains the durable boundary, subject to the adapter's acknowledgement contract. -Recoverable partitioned stream providers can compose a stream partition pipeline from , a partition source, and a data adapter. The pipeline admits immutable stream records into pooled storage, reconstructs batches lazily, reconciles the earliest subscription delivery watermark, and persists a checkpoint which resumes strictly after the last fully delivered record. +Recoverable partitioned stream providers can compose a stream partition pipeline from , a partition source, and a data adapter. The pipeline admits immutable stream records into pooled storage, reconstructs batches lazily, reconciles the earliest safe subscription scan/delivery watermark, and persists a checkpoint which resumes strictly after that position. ## Pub-sub handshake The agent registers as a producer for each stream and obtains subscription records from stream pub-sub. It holds a pin cursor while subscription handshakes complete so cache cleanup cannot pass the requested start token. New subscription notifications update the agent's local pub-sub cache. -Sequence tokens allow a rewindable adapter to start from a supported historical position. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. +Sequence tokens allow a rewindable adapter to start from a supported historical position. A start token is inclusive and remains unsafe until its record is delivered or intentionally filtered. A delivery handshake token confirms that its position was already processed. Exact `EventSequenceToken` and `EventSequenceTokenV2` values interoperate for legacy compatibility; provider-derived tokens define and use their own symmetric equality, ordering, and hashing contract. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. ## Delivery and failure semantics diff --git a/docs/site/src/content/docs/streaming/custom-queue-adapter.md b/docs/site/src/content/docs/streaming/custom-queue-adapter.md index 6b2bfbf56a4..bea79175b64 100644 --- a/docs/site/src/content/docs/streaming/custom-queue-adapter.md +++ b/docs/site/src/content/docs/streaming/custom-queue-adapter.md @@ -53,7 +53,7 @@ The factory composes the adapter with queue mapping, caching, and failure handli Custom pooled caches use to position cursors against cached messages. The default implementation compares `SequenceNumber` and `EventIndex`, preserving the numeric ordering used by existing providers. Override it when the authoritative provider position is encoded in ; return an order consistent with the token produced by so cache bounds, block selection, and cache-miss detection use the same position contract. -`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a partitioned stream transport, implement an , have the receiver or cache load and update the stream partition position, and register it as a named component with `ConfigureComponent`. Persist a checkpoint only after all consumers have advanced beyond the corresponding cached messages. A no-op checkpointer is suitable only when replay position is deliberately disposable. +`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a partitioned stream transport, implement an , have the receiver or cache load and update the stream partition position, and register it as a named component with `ConfigureComponent`. Treat a requested cursor start as inclusive: selecting that position does not confirm its record. Persist only the earliest contiguous partition position which every subscription has delivered, intentionally filtered, or safely scanned as belonging to another stream. A no-op checkpointer is suitable only when replay position is deliberately disposable. For a durable custom checkpoint backend, implement and pass it to with . Load the checkpointer before processing the partition. Each store update receives the expected backend version and returns the resulting , so a version conflict supplies the authoritative checkpoint and version for retry or reconciliation. @@ -98,7 +98,9 @@ Test the adapter against the real queue service, including: 1. queue ownership moving between silos during membership changes; 1. duplicate delivery and consumer idempotency; 1. stable stream-to-partition mapping across restarts and upgrades; -1. sustained load beyond cache capacity to verify backpressure and queue retention; and +1. sustained load beyond cache capacity to verify backpressure and queue retention; +1. quiet and busy streams sharing a partition, including restart after the quiet cursor scans unrelated records; +1. cancellation while partition ownership acquisition is blocked, followed by reassignment and late command completion; and 1. sequence-token equality, ordering, and hashing in both comparison directions. Monitor queue depth and oldest-message age by partition, receive and acknowledgement latency, redelivery count, throttling, pulling-agent errors, and consumer delivery failures. Alert before retention or visibility limits can cause data loss or a redelivery storm. diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 301bbacee47..1e0308d4ee6 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -62,7 +62,7 @@ The Event Hubs provider supports a custom data adapter for provider-specific wir ## Amazon Kinesis -Register [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/introduction.html) with . Kinesis retains events independently of Orleans, and the provider persists each shard's last delivered sequence number so that delivery can resume after shutdown or queue reassignment. See [Stream with Amazon Kinesis](kinesis-streaming.md) for configuration, checkpoint choices, and operational constraints. +Register [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/introduction.html) with . Kinesis retains events independently of Orleans, and the provider persists each shard's earliest safe scan/delivery position so that delivery can resume after shutdown or queue reassignment. Kinesis sequence tokens compare only with other Kinesis tokens using the numeric shard sequence and event index. See [Stream with Amazon Kinesis](kinesis-streaming.md) for configuration, checkpoint choices, and operational constraints. ## Amazon SQS @@ -70,7 +70,7 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S ## ADO.NET partitioned stream provider (alpha) -Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline is rewindable while records remain retained and resumes strictly after the checkpoint. retains checkpointed records for one day by default. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Cleanup is bounded by . can fault one failing subscription while preserving the multicast partition record. +Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline is rewindable while records remain retained and resumes strictly after the earliest safe scan/delivery checkpoint. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. retains checkpointed records for one day by default. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Retention and cleanup intervals round fractional seconds upward. Cleanup is bounded by . can fault one failing subscription while preserving the multicast partition record. The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 92e9bd332ab..33e7c149b96 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -9,6 +9,8 @@ internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueue { private const int BufferSize = 1024 * 1024; private readonly RecoverableStreamReceiver _inner; + private readonly AdoNetRecoverableStream _source; + private int _shutdownNotified; internal Action? OnShutdown { get; set; } @@ -22,7 +24,7 @@ public AdoNetQueueAdapterReceiver( Serializer serializer, ILogger logger) { - var source = new AdoNetRecoverableStream( + _source = new AdoNetRecoverableStream( clusterOptions.ServiceId, providerId, queueId, @@ -30,7 +32,7 @@ public AdoNetQueueAdapterReceiver( queries, logger); var checkpointer = new StreamQueueCheckpointer( - source, + _source, new StreamQueueCheckpointerOptions { CheckpointComparer = StreamCheckpointComparers.Numeric, @@ -51,7 +53,7 @@ public AdoNetQueueAdapterReceiver( logger, maxCacheSize: cacheOptions.CacheSize); _inner = new RecoverableStreamReceiver( - source, + _source, dataAdapter, cache, checkpointer, @@ -67,6 +69,32 @@ public async Task Shutdown(TimeSpan timeout) await _inner.Shutdown(timeout); } finally + { + var acquisitionCompletion = _source.AcquisitionCompletion; + if (acquisitionCompletion.IsCompleted) + { + NotifyShutdown(); + } + else + { + _ = NotifyShutdownAfterAcquisition(acquisitionCompletion, NotifyShutdown); + } + } + } + + internal static async Task NotifyShutdownAfterAcquisition( + Task acquisitionCompletion, + Action notifyShutdown) + { + ArgumentNullException.ThrowIfNull(acquisitionCompletion); + ArgumentNullException.ThrowIfNull(notifyShutdown); + await acquisitionCompletion.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + notifyShutdown(); + } + + private void NotifyShutdown() + { + if (Interlocked.Exchange(ref _shutdownNotified, 1) == 0) { OnShutdown?.Invoke(this); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index 5f6a2783d25..3d206164333 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -13,14 +13,21 @@ internal sealed partial class AdoNetRecoverableStream( { private AdoNetStreamPartitionState? _partition; private long _readOffset; + private Task? _acquisitionTask; + + internal Task AcquisitionCompletion => Volatile.Read(ref _acquisitionTask) ?? Task.CompletedTask; public async ValueTask Load(CancellationToken cancellationToken) { - var partition = await queries.AcquireStreamPartitionAsync( + var acquisitionTask = queries.AcquireStreamPartitionAsync( serviceId, providerId, queueId, - options.StartFromNow).WaitAsync(cancellationToken); + options.StartFromNow, + cancellationToken); + Volatile.Write(ref _acquisitionTask, acquisitionTask); + var partition = await acquisitionTask; + cancellationToken.ThrowIfCancellationRequested(); _partition = partition; _readOffset = partition.Checkpoint ?? 0; ThrowIfRetentionGap(partition); @@ -45,7 +52,8 @@ public async ValueTask Update( providerId, queueId, ownerEpoch, - checkpointValue).WaitAsync(cancellationToken); + checkpointValue, + cancellationToken); return ResolveCheckpointUpdate( $"{serviceId}/{providerId}/{queueId}", partition.OwnerEpoch, @@ -93,18 +101,20 @@ public async Task> Read( providerId, queueId, _readOffset, - Math.Min(maxCount, options.MaxMessagesPerRead)).WaitAsync(cancellationToken); + Math.Min(maxCount, options.MaxMessagesPerRead), + cancellationToken); var cleanup = await queries.CleanupStreamMessagesAsync( serviceId, providerId, queueId, - checked((int)options.RetentionPeriod.TotalSeconds), + AdoNetStreamTime.ToSqlSeconds(options.RetentionPeriod), options.MaximumRetentionPeriod is { } maximum - ? checked((int)maximum.TotalSeconds) + ? AdoNetStreamTime.ToSqlSeconds(maximum) : null, - checked((int)options.CleanupInterval.TotalSeconds), - options.CleanupBatchSize).WaitAsync(cancellationToken); + AdoNetStreamTime.ToSqlSeconds(options.CleanupInterval), + options.CleanupBatchSize, + cancellationToken); if (cleanup.HardDeletedCount > 0) { LogHardRetentionCrossed( diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index 42287d57c24..aa9074ff0a7 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -54,6 +54,9 @@ public class AdoNetStreamOptions /// /// Gets or sets the minimum amount of time that a stream record is retained after it is checkpointed. /// + /// + /// Storage routines use whole seconds. Fractional values are rounded upward so configured retention is never shortened. + /// public TimeSpan RetentionPeriod { get; set; } = TimeSpan.FromDays(1); /// @@ -62,12 +65,14 @@ public class AdoNetStreamOptions /// /// Stream records older than this value can be deleted even when they are newer than the checkpoint. /// Such deletions are reported by storage so that the receiver can emit gap diagnostics. + /// Fractional values are rounded upward to whole seconds. /// public TimeSpan? MaximumRetentionPeriod { get; set; } /// /// Gets or sets the interval between cleanup attempts for a partition. /// + /// Fractional values are rounded upward to whole seconds. public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(1); /// diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs index 787b1f33c4f..620b98fdfbb 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs @@ -1,4 +1,5 @@ using static System.String; +using Orleans.Streaming.AdoNet; namespace Orleans.Configuration; @@ -58,5 +59,5 @@ public void ValidateConfiguration() } private static bool IsInvalidSqlInterval(TimeSpan value) - => value < TimeSpan.FromSeconds(1) || value.TotalSeconds > int.MaxValue; + => !AdoNetStreamTime.IsValidSqlInterval(value); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs new file mode 100644 index 00000000000..74da9aa349f --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs @@ -0,0 +1,25 @@ +namespace Orleans.Streaming.AdoNet; + +internal static class AdoNetStreamTime +{ + private static readonly TimeSpan MaxSqlInterval = TimeSpan.FromSeconds(int.MaxValue); + + internal static bool IsValidSqlInterval(TimeSpan value) + => value >= TimeSpan.FromSeconds(1) && value <= MaxSqlInterval; + + internal static int ToSqlSeconds(TimeSpan value) + { + if (value < TimeSpan.Zero || value > MaxSqlInterval) + { + throw new OverflowException("The interval does not fit in SQL integer seconds."); + } + + var seconds = value.Ticks / TimeSpan.TicksPerSecond; + if (value.Ticks % TimeSpan.TicksPerSecond != 0) + { + seconds++; + } + + return checked((int)seconds); + } +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index ec6af26f36e..d3169b67b81 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -54,11 +54,13 @@ The provider stores each queue as an immutable, ordered stream partition. Its st - `FaultOnDeliveryFailure`: optionally fault a failing subscription while preserving the shared partition records. - `MaxMessagesPerRead`: bound each ordered storage read. - `CheckpointPersistInterval`: throttle durable checkpoint updates. -- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). -- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. -- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. +- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). Fractional seconds round upward. +- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. Fractional seconds round upward. +- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. Fractional cleanup intervals round upward. -The partitioned stream provider is rewindable while requested records remain retained. It resumes strictly after its durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. +The partitioned stream provider is rewindable while requested records remain retained. Inclusive subscription start positions remain pending until the corresponding record is delivered or intentionally filtered. The queue checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. It resumes strictly after that durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. + +Partition acquisition is cancellation-aware. A receiver whose acquisition command is still completing retains its queue reservation, so a late database result settles before a replacement receiver acquires a newer ownership epoch. ## Alpha schema upgrade diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index 3f1ab4719ff..7b0527c0dbd 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -56,7 +56,7 @@ internal class RelationalOrleansQueries /// /// the underlying relational storage /// Orleans functional queries - private RelationalOrleansQueries(IRelationalStorage storage, DbStoredQueries dbStoredQueries) + internal RelationalOrleansQueries(IRelationalStorage storage, DbStoredQueries dbStoredQueries) { this.storage = storage; this.dbStoredQueries = dbStoredQueries; @@ -505,7 +505,8 @@ internal Task AcquireStreamPartitionAsync( string serviceId, string providerId, string queueId, - bool startFromNow) + bool startFromNow, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); @@ -528,7 +529,8 @@ internal Task AcquireStreamPartitionAsync( QueueId = queueId, StartFromNow = startFromNow }, - result => result.Single()); + result => result.Single(), + cancellationToken); } /// @@ -539,7 +541,8 @@ internal Task> ReadStreamMessagesAsync( string providerId, string queueId, long afterMessageId, - int maxCount) + int maxCount, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); @@ -566,7 +569,8 @@ internal Task> ReadStreamMessagesAsync( AfterMessageId = afterMessageId, MaxCount = maxCount }, - result => result.ToList()); + result => result.ToList(), + cancellationToken); } /// @@ -577,7 +581,8 @@ internal Task> ReadStreamMessagesAsync( string providerId, string queueId, long ownerEpoch, - long checkpoint) + long checkpoint, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); @@ -602,7 +607,8 @@ internal Task> ReadStreamMessagesAsync( OwnerEpoch = ownerEpoch, Checkpoint = checkpoint }, - result => result.SingleOrDefault()); + result => result.SingleOrDefault(), + cancellationToken); } /// @@ -646,7 +652,8 @@ internal Task CleanupStreamMessagesAsync( int retentionPeriodSeconds, int? maximumRetentionPeriodSeconds, int cleanupIntervalSeconds, - int cleanupBatchSize) + int cleanupBatchSize, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); @@ -681,7 +688,8 @@ internal Task CleanupStreamMessagesAsync( CleanupIntervalSeconds = cleanupIntervalSeconds, CleanupBatchSize = cleanupBatchSize }, - result => result.Single()); + result => result.Single(), + cancellationToken); } private static long? GetNullableInt64(IDataRecord record, string fieldName) diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 7c658829b1a..cc9dbc4c45e 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -573,56 +573,24 @@ public QueueCacheCursorMoveResult TryGetNextMessageWithResult(object cursorObj, return QueueCacheCursorMoveResult.FromCacheMiss(CreateCacheMissInfo(cursor.SequenceToken!)); } - // Iterate forward (in time) in the cache until we find a message on the stream or run out of cached messages. - // Note that we get the message from the current cursor location, then move it forward. This means that if we return true, the cursor - // will point to the next message after the one we're returning. + // Iterate forward in partition order. Records for other streams are safe as soon as they + // are scanned. A matching record and everything after it remain pending until its delivery + // is confirmed by the owner. while (cursor.State is CursorStates.Set or CursorStates.EarliestAvailableSet) { CachedMessage currentMessage = cursor.Message; - - // Have we caught up to the newest event, if so set cursor to idle. - if (cursor.CurrentBlock == messageBlocks.First && cursor.IsNewestInBlock) - { - if (cursor.State == CursorStates.EarliestAvailableSet) - { - SetWaitingAfter(cursor, cursor.CurrentBlock!, cursor.Index); - } - else - { - cursor.State = CursorStates.Idle; - cursor.SequenceToken = messageBlocks.First!.Value.GetNewestSequenceToken(cacheDataAdapter); // Just compared equal to cursor.CurrentBlock, which is non-null while cursor.State is Set. - } - } - else // move to next - { - int index; - if (cursor.IsNewestInBlock) - { - // cursor.CurrentBlock is non-null while cursor.State is Set. It is not messageBlocks.First here (checked above), - // so it is not the newest block in the cache, and therefore has a non-null Previous. - cursor.CurrentBlock = cursor.CurrentBlock!.Previous; - cursor.CurrentBlock!.Value.TryFindFirstMessage(cursor.StreamId, this.cacheDataAdapter, out index); - } - else - { - cursor.CurrentBlock!.Value.TryFindNextMessage(cursor.Index + 1, cursor.StreamId, this.cacheDataAdapter, out index); // Non-null while cursor.State is Set. - } - cursor.Index = index; - if (cursor.State == CursorStates.EarliestAvailableSet) - { - cursor.BlockGeneration = cursor.CurrentBlock!.Value.Generation; - } - } + var currentToken = cacheDataAdapter.GetSequenceToken(ref currentMessage); + MoveCursorForward(cursor, currentToken); // check if this message is in the cursor's stream if (currentMessage.CompareStreamId(cursor.StreamId)) { + cursor.RecordPending(currentToken); message = cacheDataAdapter.GetBatchContainer(ref currentMessage); - cursor.SequenceToken = cursor.IsEarliestAvailable - ? cacheDataAdapter.GetSequenceToken(ref currentMessage) - : cursor.CurrentBlock!.Value.GetSequenceToken(cursor.Index, cacheDataAdapter); // Non-null while cursor.State is Set. return QueueCacheCursorMoveResult.Success; } + + cursor.RecordScanned(currentToken); } return QueueCacheCursorMoveResult.NoData; @@ -634,6 +602,77 @@ private QueueCacheMissInfo CreateCacheMissInfo(StreamSequenceToken requestedToke messageBlocks.Last!.Value.GetOldestSequenceToken(cacheDataAdapter), messageBlocks.First!.Value.GetNewestSequenceToken(cacheDataAdapter)); + internal StreamSequenceToken? GetSafeSequenceToken(object cursorObj) + => GetCursor(cursorObj).SafeSequenceToken; + + internal void AdvanceCursorPast(object cursorObj, StreamSequenceToken token) + { + ArgumentNullException.ThrowIfNull(token); + var cursor = GetCursor(cursorObj); + if (cursor.State != CursorStates.Set) + { + SetCursor(cursor, cursor.SequenceToken); + } + + while (cursor.State == CursorStates.Set) + { + var currentMessage = cursor.Message; + if (cacheDataAdapter.Compare(ref currentMessage, token) > 0) + { + break; + } + + var currentToken = cacheDataAdapter.GetSequenceToken(ref currentMessage); + MoveCursorForward(cursor, currentToken); + cursor.SafeSequenceToken = currentToken; + } + + cursor.ClearPending(); + } + + internal void RecordDeliverySuccess(object cursorObj) + => GetCursor(cursorObj).RecordDeliverySuccess(); + + private Cursor GetCursor(object cursorObj) + => cursorObj as Cursor + ?? throw new ArgumentOutOfRangeException(nameof(cursorObj), "Cursor is bad"); + + private void MoveCursorForward(Cursor cursor, StreamSequenceToken currentToken) + { + if (cursor.CurrentBlock == messageBlocks.First && cursor.IsNewestInBlock) + { + if (cursor.State == CursorStates.EarliestAvailableSet) + { + SetWaitingAfter(cursor, cursor.CurrentBlock!, cursor.Index); + } + else + { + cursor.State = CursorStates.Idle; + cursor.SequenceToken = currentToken; + } + + return; + } + + if (cursor.IsNewestInBlock) + { + // The current block is not the newest block, so Previous is non-null. + cursor.CurrentBlock = cursor.CurrentBlock!.Previous; + cursor.Index = cursor.CurrentBlock!.Value.OldestMessageIndex; + } + else + { + cursor.Index++; + } + + if (cursor.State == CursorStates.EarliestAvailableSet) + { + cursor.BlockGeneration = cursor.CurrentBlock!.Value.Generation; + } + + cursor.SequenceToken = cursor.CurrentBlock!.Value.GetSequenceToken(cursor.Index, cacheDataAdapter); + } + /// /// Add a list of queue message to the cache /// @@ -703,6 +742,9 @@ public Cursor(StreamId streamId) // current sequence token; null while waiting for the first message to arrive public StreamSequenceToken? SequenceToken; public long BlockGeneration; + public StreamSequenceToken? SafeSequenceToken; + private StreamSequenceToken? pendingSequenceToken; + private bool hasPendingDelivery; // reference into cache; non-null while State is Set public LinkedListNode? CurrentBlock; @@ -712,6 +754,42 @@ public Cursor(StreamId streamId) public bool IsEarliestAvailable => State is CursorStates.EarliestAvailableSet or CursorStates.EarliestAvailableWaiting; public bool IsNewestInBlock => Index == CurrentBlock!.Value.NewestMessageIndex; // Only accessed while State is Set, at which point CurrentBlock is non-null. public CachedMessage Message => CurrentBlock!.Value[Index]; // Only accessed while State is Set, at which point CurrentBlock is non-null. + + public void RecordScanned(StreamSequenceToken token) + { + if (hasPendingDelivery) + { + pendingSequenceToken = token; + } + else + { + SafeSequenceToken = token; + } + } + + public void RecordPending(StreamSequenceToken token) + { + hasPendingDelivery = true; + pendingSequenceToken = token; + } + + public void RecordDeliverySuccess() + { + if (!hasPendingDelivery) + { + return; + } + + SafeSequenceToken = pendingSequenceToken; + pendingSequenceToken = null; + hasPendingDelivery = false; + } + + public void ClearPending() + { + pendingSequenceToken = null; + hasPendingDelivery = false; + } } } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index b11afd62a01..2688c7a351a 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -233,7 +233,7 @@ private ArraySegment GetSegment(int size) return segment; } - private sealed class Cursor : IQueueCacheCursor + private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress { private readonly PooledQueueCache _cache; private readonly object _cursor; @@ -249,6 +249,10 @@ public void Dispose() { } + public StreamSequenceToken? SafeSequenceToken => _cache.GetSafeSequenceToken(_cursor); + + public void AdvancePast(StreamSequenceToken token) => _cache.AdvanceCursorPast(_cursor, token); + public IBatchContainer? GetCurrent(out Exception? exception) { exception = null; @@ -271,6 +275,8 @@ public bool MoveNext() public void RecordDeliveryFailure() { } + + public void RecordDeliverySuccess() => _cache.RecordDeliverySuccess(_cursor); } } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 10f012869a0..4461cfecd3b 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -84,16 +84,19 @@ private Task EnsureInitialized(CancellationToken cancellationToken) if (_initializeTask is null || _initializeTask.IsCompleted) { - _initializeTask = InitializeCore(); + _initializeTask = InitializeCore(cancellationToken); } return _initializeTask.WaitAsync(cancellationToken); } } - private async Task InitializeCore() + private async Task InitializeCore(CancellationToken initializationToken) { - var lifecycleToken = _lifecycleCancellation.Token; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + _lifecycleCancellation.Token, + initializationToken); + var lifecycleToken = cancellation.Token; var checkpoint = await _checkpointer.Load(lifecycleToken); await _source.Initialize( new RecoverableStreamStartPosition( diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 943f084f4d7..15c9add1f2d 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -431,7 +431,8 @@ private async Task DoHandshakeWithConsumer( StreamHandshakeToken? requestedHandshakeToken = null; StreamHandshakeToken? effectiveHandshakeToken = null; var providerDefaultRequest = false; - var effectiveStartToken = cacheToken ?? consumerData.PendingStartToken; + var cursorStartToken = cacheToken ?? consumerData.PendingStartToken; + var cursorRepositioned = false; // if not cache, then we can't get cursor and there is no reason to ask consumer for token. if (queueCache != null) { @@ -471,12 +472,14 @@ private async Task DoHandshakeWithConsumer( consumerData.StreamId, startPositionToken, cacheToken ?? consumerData.PendingStartToken); + cursorStartToken = cacheToken ?? consumerData.PendingStartToken; + cursorRepositioned = true; } } else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { - effectiveStartToken = requestedToken; + cursorStartToken = requestedToken; consumerData.SafeDisposeCursor(logger); if (effectiveHandshakeToken is DeliveryToken || effectiveHandshakeToken is StartToken @@ -488,13 +491,14 @@ private async Task DoHandshakeWithConsumer( requestedToken, cacheToken, out consumerData.PendingBatch); + cursorRepositioned = true; } else { var result = queueCache.TryGetCacheCursor(consumerData.StreamId, requestedToken); if (result.Kind == QueueCacheCursorResultKind.CacheMiss && cacheToken is not null) { - effectiveStartToken = cacheToken; + cursorStartToken = cacheToken; } consumerData.Cursor = result.Kind switch @@ -505,6 +509,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null QueueCacheCursorResultKind.CacheMiss => throw result.CacheMiss!.Value.ToException(), _ => throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."), }; + cursorRepositioned = true; } } else if (effectiveHandshakeToken is not null) @@ -516,7 +521,10 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null { var registrationToken = cacheToken ?? consumerData.PendingStartToken; if (consumerData.Cursor == null) // if the consumer did not ask for a specific token and we already have a cursor, just keep using it. + { consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, registrationToken); + cursorRepositioned = true; + } } } catch (Exception exception) @@ -563,16 +571,35 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null try { var registrationToken = cacheToken ?? consumerData.PendingStartToken; - effectiveStartToken = registrationToken; + cursorStartToken = registrationToken; consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, registrationToken); + cursorRepositioned = true; } catch (Exception) { consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, null); - effectiveStartToken = null; + cursorStartToken = null; + cursorRepositioned = true; + } + } + + if (cursorRepositioned) + { + consumerData.CursorStartToken = cursorStartToken; + if (requestedHandshakeToken is DeliveryToken deliveryToken) + { + consumerData.LastProcessedToken = deliveryToken.Token; + consumerData.LastSafePartitionToken = deliveryToken.Token; + (consumerData.Cursor as IQueueCacheCursorProgress)?.AdvancePast(deliveryToken.Token); + } + else + { + // Start/cache/pending tokens are inclusive positions. They become safe only + // after the matching record is delivered or intentionally filtered. + consumerData.LastProcessedToken = null; + consumerData.LastSafePartitionToken = null; } } - consumerData.LastProcessedToken = effectiveStartToken ?? consumerData.LastProcessedToken; consumerData.HandshakeGeneration++; return true; } @@ -987,17 +1014,25 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); - foreach (var group in - multiBatch - .Where(m => m is not null) - .GroupBy(container => container.StreamId)) + var availableMessages = multiBatch.Where(m => m is not null).ToList(); + if (availableMessages.Count == 0) + { + return false; + } + + var partitionStartToken = availableMessages[0].SequenceToken; + foreach (var streamData in pubSubCache.Values) + { + StartInactiveCursors(streamData, partitionStartToken, CancellationToken.None); + } + + foreach (var group in availableMessages.GroupBy(container => container.StreamId)) { var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); StreamSequenceToken startToken = group.First().SequenceToken; if (pubSubCache.TryGetValue(streamId, out var streamData)) { streamData.RefreshActivity(now); - StartInactiveCursors(streamData, startToken, CancellationToken.None); } else { @@ -1077,6 +1112,13 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) } var current = consumer.LastProcessedToken; + var safePartition = consumer.LastSafePartitionToken; + if (safePartition is not null + && (current is null || IsBefore(current, safePartition))) + { + current = safePartition; + } + if (current is null) { return false; @@ -1129,6 +1171,22 @@ private void ScheduleDeliveryProgressUpdate() private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => EventSequenceTokenCompatibility.Compare(current, other) < 0; + private static void UpdateCursorProgress( + StreamConsumerData consumerData, + IQueueCacheCursorProgress? progressCursor) + { + if (progressCursor?.SafeSequenceToken is not { } safeToken) + { + return; + } + + if (consumerData.LastSafePartitionToken is null + || IsBefore(consumerData.LastSafePartitionToken, safeToken)) + { + consumerData.LastSafePartitionToken = safeToken; + } + } + private void RegisterStream( QualifiedStreamId streamId, StreamSequenceToken firstToken, @@ -1318,6 +1376,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati while (!IsShutdown && !cancellationToken.IsCancellationRequested && consumerData.Cursor is not null) { var handshakeGeneration = consumerData.HandshakeGeneration; + var progressCursor = consumerData.Cursor as IQueueCacheCursorProgress; var batchCursor = options.BatchContainerBatchSize > 1 ? consumerData.Cursor as IQueueCacheCursorBatchDelivery : null; @@ -1328,6 +1387,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati try { nextBatch = GetBatchForConsumer(consumerData); + UpdateCursorProgress(consumerData, progressCursor); if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { consumerData.SafeDisposeCursor(logger); @@ -1367,7 +1427,9 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is null) { + progressCursor?.RecordDeliverySuccess(); consumerData.LastProcessedToken = nextBatch.ProgressToken; + UpdateCursorProgress(consumerData, progressCursor); continue; } } @@ -1413,6 +1475,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; + var resumedFromFallback = false; if (newToken is StartPositionToken startPositionToken) { consumerData.LastProcessedToken = null; @@ -1472,11 +1535,25 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.SafeDisposeCursor(logger); consumerData.Cursor = newCursor; consumerData.PendingBatch = pendingBatch; + consumerData.CursorStartToken = resumedFromFallback ? batch.SequenceToken : newToken.Token; + if (newToken is DeliveryToken deliveryToken) + { + consumerData.LastProcessedToken = deliveryToken.Token; + consumerData.LastSafePartitionToken = deliveryToken.Token; + UpdateCursorProgress(consumerData, newCursor as IQueueCacheCursorProgress); + } + else + { + consumerData.LastProcessedToken = null; + consumerData.LastSafePartitionToken = null; + } } else { // Track progress for the periodic delivery scan. + progressCursor?.RecordDeliverySuccess(); consumerData.LastProcessedToken = nextBatch.ProgressToken; + UpdateCursorProgress(consumerData, progressCursor); } } } diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index 46e9650e2be..3a39f2da9f6 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -38,6 +38,7 @@ internal sealed class StreamConsumerData public IBatchContainer? PendingBatch; [NonSerialized] public bool StartPositionIsProviderDefault; + public StreamSequenceToken? CursorStartToken; /// /// The sequence token of the last batch processed (delivered or filtered) by this subscription. @@ -55,6 +56,14 @@ internal sealed class StreamConsumerData [NonSerialized] public long HandshakeRequestId; + /// + /// The last contiguous partition record which is safe for this subscription. + /// This includes successfully processed matching records and scanned records + /// belonging to other streams. + /// + [NonSerialized] + public StreamSequenceToken? LastSafePartitionToken; + public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { SubscriptionId = subscriptionId; diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs index cd27e0da14f..63c1d8a7b0a 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs @@ -102,7 +102,9 @@ QueueCacheCursorResult TryGetCacheCursorAtPosition( /// Updates the cache with the current delivery progress of all active subscriptions. /// /// - /// The earliest last processed sequence token across registered subscriptions. + /// The earliest contiguous partition position which is safe across registered subscriptions. + /// A position becomes safe after a matching record is delivered or intentionally filtered, + /// or after a cursor scans an unrelated record without an earlier pending delivery. /// A value indicates that there are no active subscriptions. /// The token is only valid for the duration of the call and must not be stored. /// diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs new file mode 100644 index 00000000000..4d4856f3788 --- /dev/null +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs @@ -0,0 +1,10 @@ +namespace Orleans.Streams; + +internal interface IQueueCacheCursorProgress +{ + StreamSequenceToken? SafeSequenceToken { get; } + + void AdvancePast(StreamSequenceToken token); + + void RecordDeliverySuccess(); +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs index ad5c76dab36..a6257f84bfd 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs @@ -5,6 +5,7 @@ using Orleans.GrainDirectory.AdoNet; using Orleans.Runtime; using Orleans.Storage; +using Orleans.Streaming.AdoNet; using Orleans.Tests.SqlUtils; using UnitTests.StorageTests.Relational; @@ -250,6 +251,18 @@ public void Streaming_RejectsSubSecondRetentionPeriod() AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.RetentionPeriod)); } + [Theory] + [InlineData(1.1)] + [InlineData(1.9)] + public void Streaming_AllowsFractionalRetentionWhichRoundsUp(double seconds) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(seconds); + + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + Assert.Equal(2, AdoNetStreamTime.ToSqlSeconds(options.RetentionPeriod)); + } + [Fact] public void Streaming_AllowsNullMaximumRetentionPeriod() { @@ -303,6 +316,15 @@ public void Streaming_RejectsSubSecondCleanupInterval() AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.CleanupInterval)); } + [Fact] + public void Streaming_RejectsRetentionWhoseCeilingExceedsSqlIntegerRange() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(int.MaxValue) + TimeSpan.FromTicks(1); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + [Theory] [InlineData(1, true)] [InlineData(0, false)] diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 03f675442da..c950fd71ced 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -1,4 +1,12 @@ +using System.Data; +using System.Reflection; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; using Orleans.Streaming.AdoNet; +using Orleans.Streaming.AdoNet.Storage; +using Orleans.Streams; namespace Tester.AdoNet.Streaming; @@ -43,4 +51,365 @@ public void ResolveCheckpointUpdate_ThrowsWhenPartitionOwnershipIsLost() Assert.Contains("service/provider/queue", exception.Message); Assert.Contains("epoch 7", exception.Message); } + + [Theory] + [InlineData(1.1, 2)] + [InlineData(1.9, 2)] + [InlineData(2.0, 2)] + public void ToSqlSeconds_RoundsUpWithoutShorteningRetention(double seconds, int expected) + => Assert.Equal(expected, AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(seconds))); + + [Fact] + public void ToSqlSeconds_ThrowsWhenCeilingExceedsSqlIntegerRange() + { + var value = TimeSpan.FromSeconds(int.MaxValue) + TimeSpan.FromTicks(1); + + Assert.Throws(() => AdoNetStreamTime.ToSqlSeconds(value)); + } + + [Fact] + public void CleanupParameters_UseRoundedIntegerSecondsAndNullableMaximum() + { + using var command = new SqlCommand(); + _ = new DbStoredQueries.Columns(command) + { + RetentionPeriodSeconds = AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(1.1)), + MaximumRetentionPeriodSeconds = null, + CleanupIntervalSeconds = AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(1.9)), + }; + + Assert.Equal(2, command.Parameters[nameof(DbStoredQueries.Columns.RetentionPeriodSeconds)].Value); + var maximum = command.Parameters[nameof(DbStoredQueries.Columns.MaximumRetentionPeriodSeconds)]; + Assert.Equal(DBNull.Value, maximum.Value); + Assert.Equal(DbType.Int32, maximum.DbType); + Assert.Equal(2, command.Parameters[nameof(DbStoredQueries.Columns.CleanupIntervalSeconds)].Value); + } + + [Fact] + public async Task Load_PropagatesCancellationAndDiscardsLateAcquisition() + { + var storage = new BlockingRelationalStorage(); + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions { StartFromNow = false }, + CreateQueries(storage), + NullLogger.Instance); + using var cancellation = new CancellationTokenSource(); + + var loadTask = source.Load(cancellation.Token).AsTask(); + await storage.AcquisitionStarted.Task; + cancellation.Cancel(); + + Assert.True(storage.CapturedCancellationToken.IsCancellationRequested); + Assert.False(source.AcquisitionCompletion.IsCompleted); + + storage.CompleteAcquisition(ownerEpoch: 1); + await Assert.ThrowsAnyAsync(() => loadTask); + Assert.True(source.AcquisitionCompletion.IsCompletedSuccessfully); + await Assert.ThrowsAsync( + () => source.Update("1", "1", CancellationToken.None).AsTask()); + } + + [Theory] + [InlineData(AcquisitionCompletionKind.Success)] + [InlineData(AcquisitionCompletionKind.Fault)] + [InlineData(AcquisitionCompletionKind.Canceled)] + public async Task ShutdownNotification_WaitsForAnyAcquisitionCompletionBeforeAllowingReplacement( + AcquisitionCompletionKind completionKind) + { + var queueId = QueueId.GetQueueId("queue", 0, 0); + var registry = new QueueAdapterReceiverRegistry(_ => new ReservedReceiver()); + var first = registry.GetOrCreate(queueId); + var acquisition = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var released = AdoNetQueueAdapterReceiver.NotifyShutdownAfterAcquisition( + acquisition.Task, + () => registry.Remove(queueId, first)); + + Assert.Same(first, registry.GetOrCreate(queueId)); + switch (completionKind) + { + case AcquisitionCompletionKind.Success: + acquisition.SetResult(); + break; + case AcquisitionCompletionKind.Fault: + acquisition.SetException(new InvalidOperationException("late failure")); + break; + case AcquisitionCompletionKind.Canceled: + acquisition.SetCanceled(); + break; + } + + await released; + + var replacement = registry.GetOrCreate(queueId); + Assert.NotSame(first, replacement); + Assert.Same(replacement, Assert.Single(registry.Receivers).Value); + } + + [Theory] + [InlineData(StreamQueryKind.Read)] + [InlineData(StreamQueryKind.Advance)] + [InlineData(StreamQueryKind.Cleanup)] + public async Task StreamingQueries_PropagateCancellationToken(StreamQueryKind queryKind) + { + var storage = new CapturingRelationalStorage(); + var queries = CreateQueries(storage); + using var cancellation = new CancellationTokenSource(); + + switch (queryKind) + { + case StreamQueryKind.Read: + _ = await queries.ReadStreamMessagesAsync( + "service", "provider", "queue", 0, 1, cancellation.Token); + break; + case StreamQueryKind.Advance: + _ = await queries.AdvanceStreamCheckpointAsync( + "service", "provider", "queue", 1, 1, cancellation.Token); + break; + case StreamQueryKind.Cleanup: + _ = await queries.CleanupStreamMessagesAsync( + "service", "provider", "queue", 1, 2, 3, 4, cancellation.Token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(queryKind)); + } + + Assert.Equal(cancellation.Token, storage.CapturedCancellationToken); + } + + [Fact] + public async Task Read_UsesDistinctRoundedRetentionParameters() + { + var storage = new CapturingRelationalStorage(); + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions + { + MaxMessagesPerRead = 10, + RetentionPeriod = TimeSpan.FromSeconds(2.1), + MaximumRetentionPeriod = TimeSpan.FromSeconds(5.1), + CleanupInterval = TimeSpan.FromSeconds(3.1), + CleanupBatchSize = 9, + }, + CreateQueries(storage), + NullLogger.Instance); + + Assert.Empty(await source.Read(10, CancellationToken.None)); + + var parameters = storage.Parameters[nameof(DbStoredQueries.CleanupStreamMessagesKey)]; + Assert.Equal(3, parameters[nameof(DbStoredQueries.Columns.RetentionPeriodSeconds)]); + Assert.Equal(6, parameters[nameof(DbStoredQueries.Columns.MaximumRetentionPeriodSeconds)]); + Assert.Equal(4, parameters[nameof(DbStoredQueries.Columns.CleanupIntervalSeconds)]); + Assert.Equal(9, parameters[nameof(DbStoredQueries.Columns.CleanupBatchSize)]); + } + + private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage) + { + var queryValues = typeof(DbStoredQueries) + .GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) + .ToDictionary(property => property.Name, property => + property.Name == nameof(DbStoredQueries.StreamSchemaVersionKey) ? "2" : property.Name); + return new RelationalOrleansQueries(storage, new DbStoredQueries(queryValues)); + } + + private sealed class BlockingRelationalStorage : IRelationalStorage + { + private readonly TaskCompletionSource acquisition = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AcquisitionStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken CapturedCancellationToken { get; private set; } + + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; + + public string ConnectionString => string.Empty; + + public void CompleteAcquisition(long ownerEpoch) + => acquisition.SetResult(new( + "service", + "provider", + "queue", + ownerEpoch, + Checkpoint: 0, + EarliestMessageId: null, + TailMessageId: null)); + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + CapturedCancellationToken = cancellationToken; + AcquisitionStarted.TrySetResult(); + var state = await acquisition.Task; + using var command = new SqlCommand(); + parameterProvider?.Invoke(command); + var record = new DictionaryDataRecord(new Dictionary + { + [nameof(AdoNetStreamPartitionState.ServiceId)] = state.ServiceId, + [nameof(AdoNetStreamPartitionState.ProviderId)] = state.ProviderId, + [nameof(AdoNetStreamPartitionState.QueueId)] = state.QueueId, + [nameof(AdoNetStreamPartitionState.OwnerEpoch)] = state.OwnerEpoch, + [nameof(AdoNetStreamPartitionState.Checkpoint)] = state.Checkpoint, + [nameof(AdoNetStreamPartitionState.EarliestMessageId)] = state.EarliestMessageId, + [nameof(AdoNetStreamPartitionState.TailMessageId)] = state.TailMessageId, + }); + return [await selector(record, 0, cancellationToken)]; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + private sealed class DictionaryDataRecord(IReadOnlyDictionary values) : IDataRecord + { + private readonly string[] names = values.Keys.ToArray(); + + public object this[int i] => GetValue(i); + public object this[string name] => values[name] ?? DBNull.Value; + public int FieldCount => names.Length; + public bool GetBoolean(int i) => (bool)GetValue(i); + public byte GetByte(int i) => (byte)GetValue(i); + public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => throw new NotSupportedException(); + public char GetChar(int i) => (char)GetValue(i); + public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => throw new NotSupportedException(); + public IDataReader GetData(int i) => throw new NotSupportedException(); + public string GetDataTypeName(int i) => GetFieldType(i).Name; + public DateTime GetDateTime(int i) => (DateTime)GetValue(i); + public decimal GetDecimal(int i) => (decimal)GetValue(i); + public double GetDouble(int i) => (double)GetValue(i); + public Type GetFieldType(int i) => GetValue(i).GetType(); + public float GetFloat(int i) => (float)GetValue(i); + public Guid GetGuid(int i) => (Guid)GetValue(i); + public short GetInt16(int i) => (short)GetValue(i); + public int GetInt32(int i) => (int)GetValue(i); + public long GetInt64(int i) => (long)GetValue(i); + public string GetName(int i) => names[i]; + public int GetOrdinal(string name) => Array.IndexOf(names, name); + public string GetString(int i) => (string)GetValue(i); + public object GetValue(int i) => values[names[i]] ?? DBNull.Value; + public int GetValues(object[] destination) + { + var count = Math.Min(destination.Length, FieldCount); + for (var i = 0; i < count; i++) + { + destination[i] = GetValue(i); + } + + return count; + } + public bool IsDBNull(int i) => values[names[i]] is null or DBNull; + } + + private sealed class CapturingRelationalStorage : IRelationalStorage + { + public CancellationToken CapturedCancellationToken { get; private set; } + + public Dictionary> Parameters { get; } = []; + + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; + + public string ConnectionString => string.Empty; + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + CapturedCancellationToken = cancellationToken; + using var command = new SqlCommand(); + parameterProvider?.Invoke(command); + Parameters[query] = command.Parameters.Cast() + .ToDictionary(parameter => parameter.ParameterName, parameter => + parameter.Value is DBNull ? null : parameter.Value); + var records = query switch + { + nameof(DbStoredQueries.ReadStreamMessagesKey) => Array.Empty(), + nameof(DbStoredQueries.AdvanceStreamCheckpointKey) => + [ + Record( + (nameof(AdoNetStreamCheckpointUpdate.ServiceId), "service"), + (nameof(AdoNetStreamCheckpointUpdate.ProviderId), "provider"), + (nameof(AdoNetStreamCheckpointUpdate.QueueId), "queue"), + (nameof(AdoNetStreamCheckpointUpdate.OwnerEpoch), 1L), + (nameof(AdoNetStreamCheckpointUpdate.Checkpoint), 1L), + (nameof(AdoNetStreamCheckpointUpdate.Updated), true)), + ], + nameof(DbStoredQueries.CleanupStreamMessagesKey) => + [ + Record( + (nameof(AdoNetStreamCleanupResult.Ran), true), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 0), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), null), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 0), + (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), null), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), null), + (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), null), + (nameof(AdoNetStreamCleanupResult.TailMessageId), null)), + ], + _ => throw new ArgumentOutOfRangeException(nameof(query), query, null), + }; + var results = new List(); + foreach (var record in records) + { + results.Add(await selector(record, 0, cancellationToken)); + } + + return results; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + private static IDataRecord Record(params (string Name, object? Value)[] values) + => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); + } + + private sealed class ReservedReceiver : IQueueAdapterReceiver, IQueueCache + { + public Task Initialize(TimeSpan timeout) => Task.CompletedTask; + public Task> GetQueueMessagesAsync(int maxCount) => Task.FromResult>([]); + public Task MessagesDeliveredAsync(IList messages) => Task.CompletedTask; + public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; + public int GetMaxAddCount() => 1; + public void AddToCache(IList messages) { } + public bool TryPurgeFromCache(out IList purgedItems) + { + purgedItems = null!; + return false; + } + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => throw new NotSupportedException(); + public bool IsUnderPressure() => false; + } + + public enum AcquisitionCompletionKind + { + Success, + Fault, + Canceled, + } + + public enum StreamQueryKind + { + Read, + Advance, + Cleanup, + } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index e2d49048a8c..794bfb8d120 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -97,6 +97,12 @@ public async Task ReadFromQueue_DoesNotWaitForColdStreamRegistration() await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); } + private sealed class FilterByDataStreamFilter : IStreamFilter + { + public bool ShouldDeliver(StreamId streamId, object item, string? filterData) + => !string.Equals(filterData, "reject", StringComparison.Ordinal); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -750,16 +756,19 @@ public void RecordDeliveryFailure() } } - private sealed class ScriptedQueueCache : IQueueCache + private sealed class ScriptedQueueCache(int maxCacheSize = 1000) : IQueueCache { private readonly List messages = new(); private TaskCompletionSource deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); + private StreamSequenceToken? purgedThrough; public int DeliveryProgressCallCount { get; private set; } public List DeliveryProgressTokens { get; } = new(); public Task DeliveryProgressUpdated => deliveryProgressUpdated.Task; - public int GetMaxAddCount() => 1000; + public int GetMaxAddCount() + => Math.Max(0, maxCacheSize - messages.Count(message => + purgedThrough is null || message.SequenceToken.Newer(purgedThrough))); public void AddToCache(IList messages) { @@ -777,12 +786,17 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? return new ScriptedQueueCursor(messages, streamId, token); } - public bool IsUnderPressure() => false; + public bool IsUnderPressure() => GetMaxAddCount() == 0; public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { DeliveryProgressCallCount++; DeliveryProgressTokens.Add(earliestSubscriptionToken); + if (earliestSubscriptionToken is not null + && (purgedThrough is null || earliestSubscriptionToken.Newer(purgedThrough))) + { + purgedThrough = earliestSubscriptionToken; + } deliveryProgressUpdated.TrySetResult(true); } @@ -794,10 +808,17 @@ public void ClearDeliveryProgress() } } - private sealed class ScriptedQueueCursor(List messages, StreamId streamId, StreamSequenceToken? token) : IQueueCacheCursor + private sealed class ScriptedQueueCursor( + List messages, + StreamId streamId, + StreamSequenceToken? token) : IQueueCacheCursor, IQueueCacheCursorProgress { private int index = -1; private IBatchContainer? current; + private StreamSequenceToken? pendingSequenceToken; + private bool hasPendingDelivery; + + public StreamSequenceToken? SafeSequenceToken { get; private set; } public void Dispose() { @@ -814,14 +835,31 @@ public bool MoveNext() for (index++; index < messages.Count; index++) { var candidate = messages[index]; - if (candidate.StreamId.Equals(streamId) && (token is null || candidate.SequenceToken.Newer(token))) + if (token is not null && candidate.SequenceToken.CompareTo(token) < 0) + { + continue; + } + + if (candidate.StreamId.Equals(streamId)) { + hasPendingDelivery = true; + pendingSequenceToken = candidate.SequenceToken; current = candidate; return true; } + + if (hasPendingDelivery) + { + pendingSequenceToken = candidate.SequenceToken; + } + else + { + SafeSequenceToken = candidate.SequenceToken; + } } current = null; + index = messages.Count - 1; return false; } @@ -832,6 +870,31 @@ public void Refresh(StreamSequenceToken token) public void RecordDeliveryFailure() { } + + public void AdvancePast(StreamSequenceToken deliveredToken) + { + while (index + 1 < messages.Count + && messages[index + 1].SequenceToken.CompareTo(deliveredToken) <= 0) + { + index++; + SafeSequenceToken = messages[index].SequenceToken; + } + + hasPendingDelivery = false; + pendingSequenceToken = null; + } + + public void RecordDeliverySuccess() + { + if (!hasPendingDelivery) + { + return; + } + + SafeSequenceToken = pendingSequenceToken; + pendingSequenceToken = null; + hasPendingDelivery = false; + } } private sealed class CacheMissQueueCursor : IQueueCacheCursor @@ -2389,7 +2452,7 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamConsumerExtension { - private bool rewindRequested; + private int returnedRewindToken; public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -2406,13 +2469,8 @@ private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamC public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) { Delivered.TrySetResult(true); - if (!rewindRequested) - { - rewindRequested = true; - return Task.FromResult(rewindToken); - } - - return Task.FromResult(null); + return Task.FromResult( + Interlocked.Exchange(ref returnedRewindToken, 1) == 0 ? rewindToken : null); } public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; @@ -2701,7 +2759,7 @@ private static async Task RunCacheMissRecovery( [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_UsesAcceptedRedeliveryForDeliveryProgress() + public async Task Shutdown_AdvancesAfterConsumerAcceptsRetryFollowingHandshakeToken() { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) @@ -4201,6 +4259,40 @@ public void EventSequenceTokens_CompareAndEqualAcrossVersions() Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); } + [Fact] + public void EventSequenceTokens_CoalesceInHashAndSortedCollectionsAcrossVersions() + { + StreamSequenceToken v1 = new EventSequenceToken(10, 2); + StreamSequenceToken v2 = new EventSequenceTokenV2(10, 2); + + Assert.Single(new HashSet { v1, v2 }); + Assert.Single(new SortedSet { v2, v1 }); + } + + [Fact] + public void EventSequenceTokens_RejectDerivedTokensSymmetrically() + { + StreamSequenceToken baseToken = new EventSequenceTokenV2(10, 2); + StreamSequenceToken derivedToken = new DerivedEventSequenceToken(10, 2); + + Assert.False(baseToken.Equals(derivedToken)); + Assert.False(derivedToken.Equals(baseToken)); + Assert.Throws(() => baseToken.CompareTo(derivedToken)); + Assert.Throws(() => derivedToken.CompareTo(baseToken)); + Assert.Equal(2, new HashSet { baseToken, derivedToken }.Count); + Assert.Throws( + () => new SortedSet { baseToken, derivedToken }); + Assert.Throws( + () => new SortedSet { derivedToken, baseToken }); + + StreamSequenceToken v1Base = new EventSequenceToken(10, 2); + StreamSequenceToken v1Derived = new DerivedEventSequenceTokenV1(10, 2); + Assert.False(v1Base.Equals(v1Derived)); + Assert.False(v1Derived.Equals(v1Base)); + Assert.Throws(() => v1Base.CompareTo(v1Derived)); + Assert.Throws(() => v1Derived.CompareTo(v1Base)); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4266,7 +4358,7 @@ public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparis [InlineData(SubscriptionStartTokenSource.Handshake, 10)] [InlineData(SubscriptionStartTokenSource.Cache, 20)] [InlineData(SubscriptionStartTokenSource.Pending, 30)] - public async Task DeliveryProgress_UsesEffectiveSubscriptionStartBeforeFirstBatch( + public async Task DeliveryProgress_InclusiveStartRemainsUnsafeBeforeFirstBatch( SubscriptionStartTokenSource tokenSource, long expectedSequenceNumber) { @@ -4309,14 +4401,248 @@ public async Task DeliveryProgress_UsesEffectiveSubscriptionStartBeforeFirstBatc Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken)); consumerData.IsRegistered = true; - Assert.Equal(expectedSequenceNumber, consumerData.LastProcessedToken?.SequenceNumber); + Assert.Equal(expectedSequenceNumber, consumerData.CursorStartToken?.SequenceNumber); + Assert.Null(consumerData.LastProcessedToken); + Assert.Null(consumerData.LastSafePartitionToken); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + + Assert.Empty(queueCache.DeliveryProgressTokens); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Handshake_DeliveryTokenStartsAfterConfirmedRecord() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var queueCache = new ScriptedQueueCache(); + queueCache.AddToCache( + [ + new TestBatchContainer(streamId, new EventSequenceTokenV2(10)), + new TestBatchContainer(streamId, new EventSequenceTokenV2(11)), + ]); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [new TestBatchContainer(streamId, new EventSequenceTokenV2(12))])); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumer = new StartingConsumer( + StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(10))); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + consumer, + filterData: null, + now: DateTime.UtcNow); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken: null)); + consumerData.IsRegistered = true; + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + await testAccessor.GetPubSubCache(); + + Assert.Equal([11L, 12L], consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task ReadFromQueue_WakesIdleCursorAtFirstPartitionRecord() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [ + new TestBatchContainer(streamId, new EventSequenceTokenV2(2)), + new TestBatchContainer(streamId, new EventSequenceTokenV2(3)), + ])); + var queueCache = new PurgeablePooledQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumer = new StartingConsumer(startToken: null); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + consumer, + filterData: null, + now: DateTime.UtcNow); + consumerData.IsRegistered = true; + consumerData.Cursor = queueCache.GetCacheCursor(streamId, token: null); + Assert.False(consumerData.Cursor.MoveNext()); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 2)); + await testAccessor.GetPubSubCache(); + + Assert.Equal([2L, 3L], consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var deliveredToken = new EventSequenceTokenV2(10); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + new StartingConsumer(StreamHandshakeToken.CreateDeliveyToken(deliveredToken)), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken: null)); + consumerData.IsRegistered = true; + Assert.Equal(deliveredToken, consumerData.LastProcessedToken); + Assert.Equal(deliveredToken, consumerData.LastSafePartitionToken); queueCache.ClearDeliveryProgress(); timeProvider.Advance(options.DeliveryProgressUpdateInterval); await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); - var progress = Assert.Single(queueCache.DeliveryProgressTokens); - Assert.Equal(expectedSequenceNumber, progress?.SequenceNumber); + Assert.Equal(deliveredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_QuietSubscriptionAdvancesAcrossBusyStreamBeyondCapacity() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var quietStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var busyStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var quietQualifiedId = new QualifiedStreamId("provider", quietStreamId); + var busyQualifiedId = new QualifiedStreamId("provider", busyStreamId); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(2)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(3)), + ]), + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(4)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(5)), + ]), + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(6)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(7)), + ])); + var queueCache = new ScriptedQueueCache(maxCacheSize: 3); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver, + queueAdapterCache, + timeProvider, + options, + new FilterByDataStreamFilter()); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(quietQualifiedId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + await testAccessor.RegisterStream(busyQualifiedId, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + queueCache.AddToCache([new TestBatchContainer(quietStreamId, new EventSequenceTokenV2(1))]); + var cache = await testAccessor.GetPubSubCache(); + var quietData = cache[quietQualifiedId]; + var quietConsumer = quietData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + quietQualifiedId, + new ImmediateConsumer(), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + quietConsumer.IsRegistered = true; + quietConsumer.Cursor = queueCache.GetCacheCursor(quietStreamId, new EventSequenceTokenV2(1)); + var busyData = cache[busyQualifiedId]; + var busyConsumer = busyData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + busyQualifiedId, + new ImmediateConsumer(), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + busyConsumer.IsRegistered = true; + busyConsumer.Cursor = queueCache.GetCacheCursor(busyStreamId, new EventSequenceTokenV2(2)); + var filteredConsumer = busyData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + busyQualifiedId, + streamConsumer: null!, + filterData: "reject", + now: timeProvider.GetUtcNow().UtcDateTime); + filteredConsumer.IsRegistered = true; + filteredConsumer.Cursor = queueCache.GetCacheCursor(busyStreamId, new EventSequenceTokenV2(2)); + + foreach (var expectedCheckpoint in new long[] { 3, 5, 7 }) + { + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 2)); + await testAccessor.GetPubSubCache(); + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(expectedCheckpoint, checkpoint.SequenceNumber); + Assert.False(queueCache.IsUnderPressure()); + queueCache.ClearDeliveryProgress(); + } + + Assert.Equal(7, quietConsumer.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(7, busyConsumer.LastProcessedToken?.SequenceNumber); + Assert.Equal(7, filteredConsumer.LastProcessedToken?.SequenceNumber); await testAccessor.Shutdown(); } @@ -4530,8 +4856,16 @@ public enum SubscriptionStartTokenSource Pending, } + private sealed class DerivedEventSequenceToken(long sequenceNumber, int eventIndex) + : EventSequenceTokenV2(sequenceNumber, eventIndex); + + private sealed class DerivedEventSequenceTokenV1(long sequenceNumber, int eventIndex) + : EventSequenceToken(sequenceNumber, eventIndex); + private sealed class StartingConsumer(StreamHandshakeToken? startToken) : IStreamConsumerExtension { + public List DeliveredTokens { get; } = []; + public Task DeliverImmutable( GuidId subscriptionId, QualifiedStreamId streamId, @@ -4550,7 +4884,11 @@ private sealed class StartingConsumer(StreamHandshakeToken? startToken) : IStrea GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, - StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + StreamHandshakeToken? handshakeToken) + { + DeliveredTokens.Add(item.SequenceToken); + return Task.FromResult(null); + } public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index cfe5c860319..bef1c6e6d6c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -65,6 +65,148 @@ public async Task Receiver_ResumesAfterCheckpointAndPersistsDeliveryProgress() Assert.True(source.IsShutdown); } + private sealed class BlockingInitializationCheckpointer : IStreamQueueCheckpointer + { + public TaskCompletionSource FirstLoadStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken FirstLoadCancellation { get; private set; } + + public int LoadCount { get; private set; } + + public bool CheckpointExists => false; + + public Task Load() => Load(CancellationToken.None); + + public async Task Load(CancellationToken cancellationToken) + { + LoadCount++; + if (LoadCount == 1) + { + FirstLoadCancellation = cancellationToken; + FirstLoadStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + + return string.Empty; + } + + public void Update(string offset, DateTime utcNow) { } + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => cancellationToken.ThrowIfCancellationRequested(); + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + [Fact] + public async Task Receiver_RestartRedeliversInclusiveBatchWhenCheckpointDidNotAdvance() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var messages = new[] + { + new TestQueueMessage(streamId, 9, "old-9"), + new TestQueueMessage(streamId, 10, "old-10"), + new TestQueueMessage(streamId, 11, "payload"), + }; + var store = new TestCheckpointStore("10"); + var firstSource = new ReplaySource(messages); + var firstReceiver = CreateReceiver( + firstSource, + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await firstReceiver.Initialize(TimeSpan.FromSeconds(5)); + var firstNotification = Assert.Single(await firstReceiver.GetQueueMessagesAsync(10, CancellationToken.None)); + using (var cursor = firstReceiver.GetCacheCursor(streamId, firstNotification.SequenceToken)) + { + Assert.True(cursor.MoveNext()); + Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + // The inclusive first batch was observed but never confirmed as delivered. + } + + await firstReceiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal("10", store.State.Checkpoint); + + var secondSource = new ReplaySource(messages); + var secondReceiver = CreateReceiver( + secondSource, + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await secondReceiver.Initialize(TimeSpan.FromSeconds(5)); + + var redelivered = Assert.Single(await secondReceiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(11, redelivered.SequenceToken.SequenceNumber); + await secondReceiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Receiver_QuietScanCheckpointRestartsAfterBusyTail() + { + var quietStream = StreamId.Create("namespace", Guid.NewGuid()); + var busyStream = StreamId.Create("namespace", Guid.NewGuid()); + var initialMessages = new[] + { + new TestQueueMessage(quietStream, 1, "quiet"), + new TestQueueMessage(busyStream, 2, "busy-2"), + new TestQueueMessage(busyStream, 3, "busy-3"), + }; + var store = new TestCheckpointStore(string.Empty); + var receiver = CreateReceiver( + new ReplaySource(initialMessages), + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + var notifications = await receiver.GetQueueMessagesAsync(10, CancellationToken.None); + using var cursor = receiver.GetCacheCursor(quietStream, notifications[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + progress.RecordDeliverySuccess(); + Assert.False(cursor.MoveNext()); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + + receiver.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UtcNow); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal("3", store.State.Checkpoint); + + var restarted = CreateReceiver( + new ReplaySource( + [ + .. initialMessages, + new TestQueueMessage(busyStream, 4, "busy-4"), + ]), + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await restarted.Initialize(TimeSpan.FromSeconds(5)); + + var next = Assert.Single(await restarted.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(4, next.SequenceToken.SequenceNumber); + await restarted.Shutdown(TimeSpan.FromSeconds(5)); + } + [Fact] public void Registry_ReturnsSameReceiverAndCacheInstanceForQueue() { @@ -238,6 +380,120 @@ public void Cache_AddsRawRecordsInOrderAndDecodesLazily() Assert.Equal(2, adapter.GetBatchContainerCallCount); } + [Fact] + public void Cache_CursorProgressRequiresDeliveryAndIncludesUnrelatedScans() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamB, 3, "b-3"), + new TestQueueMessage(streamA, 4, "a-4"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + + Assert.True(cursor.MoveNext()); + Assert.Null(progress.SafeSequenceToken); + + progress.RecordDeliverySuccess(); + Assert.Equal(1, progress.SafeSequenceToken?.SequenceNumber); + + Assert.True(cursor.MoveNext()); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + + progress.RecordDeliverySuccess(); + Assert.Equal(4, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public void Cache_BatchedMatchesRemainPendingUntilWholeDeliverySucceeds() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamA, 3, "a-3"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + + Assert.True(cursor.MoveNext()); + Assert.True(cursor.MoveNext()); + Assert.Null(progress.SafeSequenceToken); + + progress.RecordDeliverySuccess(); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetry() + { + var source = new TestSource([]); + var checkpointer = new BlockingInitializationCheckpointer(); + var receiver = CreateReceiver(source, checkpointer); + using var cancellation = new CancellationTokenSource(); + + var initialization = receiver.Initialize(cancellation.Token); + await checkpointer.FirstLoadStarted.Task; + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => initialization); + Assert.True(checkpointer.FirstLoadCancellation.IsCancellationRequested); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(2, checkpointer.LoadCount); + Assert.Equal(1, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void Cache_QuietCursorAdvancesAcrossUnrelatedRecords() + { + var quietStream = StreamId.Create("namespace", Guid.NewGuid()); + var busyStream = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(quietStream, 1, "quiet"), + new TestQueueMessage(busyStream, 2, "busy-2"), + new TestQueueMessage(busyStream, 3, "busy-3"), + new TestQueueMessage(busyStream, 4, "busy-4"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(quietStream, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + progress.RecordDeliverySuccess(); + + Assert.False(cursor.MoveNext()); + + Assert.Equal(4, progress.SafeSequenceToken?.SequenceNumber); + } + [Fact] public async Task Receiver_RetriesInitializationOnNextRead() { @@ -324,8 +580,8 @@ public void Registry_RemoveAllowsFreshReceiverForReassignedQueue() } private static RecoverableStreamReceiver CreateReceiver( - TestSource source, - TestCheckpointer checkpointer) + IRecoverableStreamSource source, + IStreamQueueCheckpointer checkpointer) { var adapter = new TestDataAdapter(); var cache = new RecoverableStreamQueueCache( @@ -510,6 +766,63 @@ public Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class TestCheckpointStore(string checkpoint) : IStreamCheckpointStore + { + public StreamCheckpointStoreState State { get; private set; } = new(checkpoint, "1"); + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(State); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Assert.Equal(State.Version, expectedVersion); + State = new(checkpoint, (int.Parse(State.Version) + 1).ToString(CultureInfo.InvariantCulture)); + return ValueTask.FromResult(State); + } + } + + private sealed class ReplaySource(IReadOnlyList messages) + : IRecoverableStreamSource + { + private long checkpoint; + private bool read; + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + checkpoint = string.IsNullOrEmpty(position.Checkpoint) + ? 0 + : long.Parse(position.Checkpoint, CultureInfo.InvariantCulture); + return Task.CompletedTask; + } + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (read) + { + return Task.FromResult>([]); + } + + read = true; + return Task.FromResult>( + messages.Where(message => message.SequenceNumber > checkpoint).Take(maxCount).ToList()); + } + + public Task Shutdown(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + private sealed class NoOpEvictionStrategy : IEvictionStrategy { public int PerformPurgeCount { get; private set; } From 6966d2bd3d532ee9353c0424dd0fe9cce168e61f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 20:06:29 -0700 Subject: [PATCH 20/90] fix(streaming): serialize receiver creation --- .../streams-implementation/index.md | 2 +- .../QueueAdapterReceiverRegistry.cs | 30 +++++++++++++++---- .../PersistentStreamPullingAgentTests.cs | 12 ++++++++ .../RecoverableStreamReceiverTests.cs | 18 +++++++---- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 7d5c5347313..dd2bd25d67e 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -117,7 +117,7 @@ Recoverable partitioned stream providers can compose a stream partition pipeline The agent registers as a producer for each stream and obtains subscription records from stream pub-sub. It holds a pin cursor while subscription handshakes complete so cache cleanup cannot pass the requested start token. New subscription notifications update the agent's local pub-sub cache. -Sequence tokens allow a rewindable adapter to start from a supported historical position. A start token is inclusive and remains unsafe until its record is delivered or intentionally filtered. A delivery handshake token confirms that its position was already processed. Exact `EventSequenceToken` and `EventSequenceTokenV2` values interoperate for legacy compatibility; provider-derived tokens define and use their own symmetric equality, ordering, and hashing contract. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. +Sequence tokens allow a rewindable adapter to start from a supported historical position. A start token is inclusive and remains unsafe until its record is delivered or intentionally filtered. A delivery handshake token confirms that its position was already processed. Exact `EventSequenceToken` and `EventSequenceTokenV2` values interoperate for legacy compatibility. Derived tokens compare only with the same concrete type unless the provider overrides equality, ordering, and hashing together. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. ## Delivery and failure semantics diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs index 10efa599299..2eea3a72380 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; +using System.Threading; using Orleans.Streams; namespace Orleans.Providers.Streams.Common @@ -12,7 +14,7 @@ namespace Orleans.Providers.Streams.Common public sealed class QueueAdapterReceiverRegistry where TReceiver : class, IQueueAdapterReceiver, IQueueCache { - private readonly ConcurrentDictionary _receivers = new(); + private readonly ConcurrentDictionary> _receivers = new(); private readonly Func _factory; /// @@ -26,18 +28,36 @@ public QueueAdapterReceiverRegistry(Func factory) /// /// Gets the registered receiver instances. /// - public IReadOnlyDictionary Receivers => _receivers; + public IReadOnlyDictionary Receivers + => _receivers + .Where(pair => pair.Value.IsValueCreated) + .ToDictionary(pair => pair.Key, pair => pair.Value.Value); /// /// Gets or creates the coordinator for a queue. /// - public TReceiver GetOrCreate(QueueId queueId) => _receivers.GetOrAdd(queueId, _factory); + public TReceiver GetOrCreate(QueueId queueId) + => _receivers.GetOrAdd( + queueId, + static (id, factory) => new( + () => factory(id), + LazyThreadSafetyMode.ExecutionAndPublication), + _factory).Value; /// /// Removes a receiver if it is still the registered instance for the queue. /// public bool Remove(QueueId queueId, TReceiver receiver) - => ((ICollection>)_receivers) - .Remove(new(queueId, receiver)); + { + if (!_receivers.TryGetValue(queueId, out var registered) + || !registered.IsValueCreated + || !ReferenceEquals(registered.Value, receiver)) + { + return false; + } + + return ((ICollection>>)_receivers) + .Remove(new(queueId, registered)); + } } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 794bfb8d120..a85c8331a39 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4291,6 +4291,18 @@ public void EventSequenceTokens_RejectDerivedTokensSymmetrically() Assert.False(v1Derived.Equals(v1Base)); Assert.Throws(() => v1Base.CompareTo(v1Derived)); Assert.Throws(() => v1Derived.CompareTo(v1Base)); + + StreamSequenceToken v2DerivedPeer = new DerivedEventSequenceToken(10, 2); + Assert.True(derivedToken.Equals(v2DerivedPeer)); + Assert.True(v2DerivedPeer.Equals(derivedToken)); + Assert.Equal(0, derivedToken.CompareTo(v2DerivedPeer)); + Assert.Equal(derivedToken.GetHashCode(), v2DerivedPeer.GetHashCode()); + + StreamSequenceToken v1DerivedPeer = new DerivedEventSequenceTokenV1(10, 2); + Assert.True(v1Derived.Equals(v1DerivedPeer)); + Assert.True(v1DerivedPeer.Equals(v1Derived)); + Assert.Equal(0, v1Derived.CompareTo(v1DerivedPeer)); + Assert.Equal(v1Derived.GetHashCode(), v1DerivedPeer.GetHashCode()); } [TestSuite("BVT")] diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index bef1c6e6d6c..005273b8c5a 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -549,18 +549,26 @@ public async Task Registry_ConcurrentRequestsReturnSameWinningInstance() { const int participantCount = 3; var queue = QueueId.GetQueueId("queue", 0, 0); - using var barrier = new Barrier(participantCount); + var factoryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var created = 0; var registry = new QueueAdapterReceiverRegistry(_ => { - barrier.SignalAndWait(); + Interlocked.Increment(ref created); + factoryStarted.TrySetResult(); + releaseFactory.Task.GetAwaiter().GetResult(); return new TestCombinedReceiver(); }); - var results = await Task.WhenAll( - Enumerable.Range(0, participantCount) - .Select(_ => Task.Run(() => registry.GetOrCreate(queue)))); + var requests = Enumerable.Range(0, participantCount) + .Select(_ => Task.Run(() => registry.GetOrCreate(queue))) + .ToArray(); + await factoryStarted.Task; + releaseFactory.SetResult(); + var results = await Task.WhenAll(requests); Assert.All(results, result => Assert.Same(results[0], result)); + Assert.Equal(1, created); Assert.Same(results[0], Assert.Single(registry.Receivers).Value); } From 33b8b076febb8c81b1e472dfda112ed8af35be2a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 20:32:08 -0700 Subject: [PATCH 21/90] fix(streaming): roll back failed cache packing --- .../RecoverableStreamQueueCache.cs | 68 ++++++++++++------- .../RecoverableStreamReceiverTests.cs | 59 ++++++++++++++++ 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 2688c7a351a..fbdf5622372 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -102,15 +102,55 @@ public IReadOnlyList Add( ArgumentNullException.ThrowIfNull(messages); var positions = new List(messages.Count); var cachedMessages = new List(messages.Count); - foreach (var message in messages) + var allocatedBuffers = new List(); + FixedSizeBuffer? batchBuffer = null; + try { - var position = _dataAdapter.GetStreamPosition(message); - cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetSegment)); - positions.Add(position); + foreach (var message in messages) + { + var position = _dataAdapter.GetStreamPosition(message); + cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); + positions.Add(position); + } + } + catch + { + foreach (var buffer in allocatedBuffers) + { + buffer.Dispose(); + } + + throw; } _cache.Add(cachedMessages, dequeueTimeUtc); + foreach (var buffer in allocatedBuffers) + { + _evictionStrategy.OnBlockAllocated(buffer); + } + + _currentBuffer = batchBuffer; return positions; + + ArraySegment GetBatchSegment(int size) + { + if (batchBuffer is not null && batchBuffer.TryGetSegment(size, out var segment)) + { + return segment; + } + + var buffer = _bufferPool.Allocate(); + if (!buffer.TryGetSegment(size, out segment)) + { + buffer.Dispose(); + buffer = new FixedSizeBuffer(size); + _ = buffer.TryGetSegment(size, out segment); + } + + allocatedBuffers.Add(buffer); + batchBuffer = buffer; + return segment; + } } /// @@ -213,26 +253,6 @@ private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) } } - private ArraySegment GetSegment(int size) - { - if (_currentBuffer is not null && _currentBuffer.TryGetSegment(size, out var segment)) - { - return segment; - } - - var buffer = _bufferPool.Allocate(); - if (!buffer.TryGetSegment(size, out segment)) - { - buffer.Dispose(); - buffer = new FixedSizeBuffer(size); - _ = buffer.TryGetSegment(size, out segment); - } - - _currentBuffer = buffer; - _evictionStrategy.OnBlockAllocated(buffer); - return segment; - } - private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress { private readonly PooledQueueCache _cache; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 005273b8c5a..eaefab55bd0 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -343,6 +343,31 @@ public void Cache_DisposeDrainsMessagesReturnsBuffersAndAllocatesFreshBufferIfRe Assert.Equal(2, bufferPool.FreeCount); } + [Fact] + public void Cache_FailedPackingReturnsUncommittedPooledBuffers() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + new ThrowingDataAdapter(), + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + var message = new TestQueueMessage(streamId, 1, "payload"); + + Assert.Throws(() => cache.Add([message], DateTime.UnixEpoch)); + Assert.Throws(() => cache.Add([message], DateTime.UnixEpoch)); + + Assert.Equal(0, cache.ItemCount); + Assert.Equal(2, bufferPool.AllocateCount); + Assert.Equal(2, bufferPool.FreeCount); + } + [Fact] public void Cache_AddsRawRecordsInOrderAndDecodesLazily() { @@ -676,6 +701,40 @@ public bool TryGetOffset(StreamSequenceToken token, out string offset) } } + private sealed class ThrowingDataAdapter : IRecoverableStreamDataAdapter + { + public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) + => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TestQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + _ = getSegment(16); + throw new InvalidOperationException("packing failed"); + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + => throw new NotSupportedException(); + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber); + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + => cachedMessage.SequenceNumber.CompareTo(token.SequenceNumber); + + public string GetOffset(ref CachedMessage cachedMessage) + => cachedMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture); + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return true; + } + } + private sealed class TestBatchContainer( StreamId streamId, StreamSequenceToken sequenceToken, From 9b0156979ff1c6e5f56fe9407a1ed55fce21d7bd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 20:44:54 -0700 Subject: [PATCH 22/90] fix(streaming): separate scan and delivery progress --- .../Common/PooledCache/PooledQueueCache.cs | 39 ++------ .../RecoverableStreamQueueCache.cs | 3 +- .../PersistentStreamPullingAgent.cs | 45 ++++++--- .../IQueueCacheCursorProgress.cs | 2 +- .../PersistentStreamPullingAgentTests.cs | 96 ++++++++++++++++--- .../RecoverableStreamReceiverTests.cs | 28 ++++++ 6 files changed, 158 insertions(+), 55 deletions(-) diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index cc9dbc4c45e..f67cf176549 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -585,6 +585,13 @@ public QueueCacheCursorMoveResult TryGetNextMessageWithResult(object cursorObj, // check if this message is in the cursor's stream if (currentMessage.CompareStreamId(cursor.StreamId)) { + if (cursor.DeliveredThroughToken is { } deliveredThrough + && cacheDataAdapter.Compare(ref currentMessage, deliveredThrough) <= 0) + { + cursor.RecordScanned(currentToken); + continue; + } + cursor.RecordPending(currentToken); message = cacheDataAdapter.GetBatchContainer(ref currentMessage); return QueueCacheCursorMoveResult.Success; @@ -605,30 +612,8 @@ private QueueCacheMissInfo CreateCacheMissInfo(StreamSequenceToken requestedToke internal StreamSequenceToken? GetSafeSequenceToken(object cursorObj) => GetCursor(cursorObj).SafeSequenceToken; - internal void AdvanceCursorPast(object cursorObj, StreamSequenceToken token) - { - ArgumentNullException.ThrowIfNull(token); - var cursor = GetCursor(cursorObj); - if (cursor.State != CursorStates.Set) - { - SetCursor(cursor, cursor.SequenceToken); - } - - while (cursor.State == CursorStates.Set) - { - var currentMessage = cursor.Message; - if (cacheDataAdapter.Compare(ref currentMessage, token) > 0) - { - break; - } - - var currentToken = cacheDataAdapter.GetSequenceToken(ref currentMessage); - MoveCursorForward(cursor, currentToken); - cursor.SafeSequenceToken = currentToken; - } - - cursor.ClearPending(); - } + internal void SetCursorDeliveredThrough(object cursorObj, StreamSequenceToken token) + => GetCursor(cursorObj).DeliveredThroughToken = token; internal void RecordDeliverySuccess(object cursorObj) => GetCursor(cursorObj).RecordDeliverySuccess(); @@ -743,6 +728,7 @@ public Cursor(StreamId streamId) public StreamSequenceToken? SequenceToken; public long BlockGeneration; public StreamSequenceToken? SafeSequenceToken; + public StreamSequenceToken? DeliveredThroughToken; private StreamSequenceToken? pendingSequenceToken; private bool hasPendingDelivery; @@ -785,11 +771,6 @@ public void RecordDeliverySuccess() hasPendingDelivery = false; } - public void ClearPending() - { - pendingSequenceToken = null; - hasPendingDelivery = false; - } } } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index fbdf5622372..b36fba76e0a 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -271,7 +271,8 @@ public void Dispose() public StreamSequenceToken? SafeSequenceToken => _cache.GetSafeSequenceToken(_cursor); - public void AdvancePast(StreamSequenceToken token) => _cache.AdvanceCursorPast(_cursor, token); + public void SetDeliveredThrough(StreamSequenceToken token) + => _cache.SetCursorDeliveredThrough(_cursor, token); public IBatchContainer? GetCurrent(out Exception? exception) { diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 15c9add1f2d..1b60032b8d6 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -58,6 +58,7 @@ internal interface ITestAccessor Task ReadFromQueueWithCancellation(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount, CancellationToken cancellationToken); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); + Task RunConsumerCursor(StreamConsumerData consumerData); Task> GetPubSubCache(); Task AddSubscriber(StreamConsumerData consumerData); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); @@ -140,6 +141,9 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); + Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) + => this.RunOrQueueTask(() => RunConsumerCursor(consumerData)); + Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); @@ -479,7 +483,9 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { - cursorStartToken = requestedToken; + cursorStartToken = requestedHandshakeToken is DeliveryToken + ? cacheToken ?? consumerData.PendingStartToken ?? requestedToken + : requestedToken; consumerData.SafeDisposeCursor(logger); if (effectiveHandshakeToken is DeliveryToken || effectiveHandshakeToken is StartToken @@ -490,7 +496,8 @@ private async Task DoHandshakeWithConsumer( consumerData.StreamId, requestedToken, cacheToken, - out consumerData.PendingBatch); + out consumerData.PendingBatch, + cursorStartToken); cursorRepositioned = true; } else @@ -589,8 +596,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null if (requestedHandshakeToken is DeliveryToken deliveryToken) { consumerData.LastProcessedToken = deliveryToken.Token; - consumerData.LastSafePartitionToken = deliveryToken.Token; - (consumerData.Cursor as IQueueCacheCursorProgress)?.AdvancePast(deliveryToken.Token); + consumerData.LastSafePartitionToken = null; } else { @@ -746,10 +752,11 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( StreamId streamId, StreamSequenceToken token, StreamSequenceToken? fallbackToken, - out IBatchContainer? pendingBatch) + out IBatchContainer? pendingBatch, + StreamSequenceToken? startToken = null) { pendingBatch = null; - var acquisitionResult = queueCache!.TryGetCacheCursor(streamId, token); + var acquisitionResult = queueCache!.TryGetCacheCursor(streamId, startToken ?? token); if (acquisitionResult.Kind == QueueCacheCursorResultKind.CacheMiss) { return fallbackToken is not null @@ -763,6 +770,18 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( } var cursor = acquisitionResult.Cursor!; + if (cursor is IQueueCacheCursorProgress progressCursor) + { + progressCursor.SetDeliveredThrough(token); + return cursor; + } + + if (startToken is not null && !Equals(startToken, token)) + { + cursor.Dispose(); + return GetAdvancedCursorOrFallback(streamId, token, fallbackToken, out pendingBatch); + } + var retainCursor = false; try { @@ -1111,9 +1130,11 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return false; } - var current = consumer.LastProcessedToken; - var safePartition = consumer.LastSafePartitionToken; - if (safePartition is not null + var current = consumer.Cursor is IQueueCacheCursorProgress + ? consumer.LastSafePartitionToken + : consumer.LastProcessedToken; + if (consumer.Cursor is not IQueueCacheCursorProgress + && consumer.LastSafePartitionToken is { } safePartition && (current is null || IsBefore(current, safePartition))) { current = safePartition; @@ -1472,6 +1493,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.StartPositionIsProviderDefault = false; if (newToken is not null) { + var previousSafePartitionToken = consumerData.LastSafePartitionToken; consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; @@ -1524,7 +1546,8 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.StreamId, sequenceToken, batch.SequenceToken, - out pendingBatch); + out pendingBatch, + previousSafePartitionToken); } else { @@ -1539,7 +1562,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (newToken is DeliveryToken deliveryToken) { consumerData.LastProcessedToken = deliveryToken.Token; - consumerData.LastSafePartitionToken = deliveryToken.Token; + consumerData.LastSafePartitionToken = previousSafePartitionToken; UpdateCursorProgress(consumerData, newCursor as IQueueCacheCursorProgress); } else diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs index 4d4856f3788..8dd8ba7662b 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs @@ -4,7 +4,7 @@ internal interface IQueueCacheCursorProgress { StreamSequenceToken? SafeSequenceToken { get; } - void AdvancePast(StreamSequenceToken token); + void SetDeliveredThrough(StreamSequenceToken token); void RecordDeliverySuccess(); } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index a85c8331a39..263a15348de 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -817,6 +817,7 @@ private sealed class ScriptedQueueCursor( private IBatchContainer? current; private StreamSequenceToken? pendingSequenceToken; private bool hasPendingDelivery; + private StreamSequenceToken? deliveredThroughToken; public StreamSequenceToken? SafeSequenceToken { get; private set; } @@ -842,6 +843,21 @@ public bool MoveNext() if (candidate.StreamId.Equals(streamId)) { + if (deliveredThroughToken is not null + && candidate.SequenceToken.CompareTo(deliveredThroughToken) <= 0) + { + if (hasPendingDelivery) + { + pendingSequenceToken = candidate.SequenceToken; + } + else + { + SafeSequenceToken = candidate.SequenceToken; + } + + continue; + } + hasPendingDelivery = true; pendingSequenceToken = candidate.SequenceToken; current = candidate; @@ -871,18 +887,8 @@ public void RecordDeliveryFailure() { } - public void AdvancePast(StreamSequenceToken deliveredToken) - { - while (index + 1 < messages.Count - && messages[index + 1].SequenceToken.CompareTo(deliveredToken) <= 0) - { - index++; - SafeSequenceToken = messages[index].SequenceToken; - } - - hasPendingDelivery = false; - pendingSequenceToken = null; - } + public void SetDeliveredThrough(StreamSequenceToken deliveredToken) + => deliveredThroughToken = deliveredToken; public void RecordDeliverySuccess() { @@ -2800,6 +2806,7 @@ public async Task Shutdown_AdvancesAfterConsumerAcceptsRetryFollowingHandshakeTo consumerData.IsRegistered = true; consumerData.LastToken = rewindToken; consumerData.LastProcessedToken = previousToken; + consumerData.LastSafePartitionToken = previousToken; consumerData.Cursor = queueCache.GetCacheCursor(qualifiedStreamId, previousToken); queueCache.ClearDeliveryProgress(); @@ -4549,7 +4556,7 @@ public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken: null)); consumerData.IsRegistered = true; Assert.Equal(deliveredToken, consumerData.LastProcessedToken); - Assert.Equal(deliveredToken, consumerData.LastSafePartitionToken); + Assert.Null(consumerData.LastSafePartitionToken); queueCache.ClearDeliveryProgress(); timeProvider.Advance(options.DeliveryProgressUpdateInterval); @@ -4559,6 +4566,66 @@ public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() await testAccessor.Shutdown(); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DeliveryTokenWaitsForPartitionScanInProgressAwareCache() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedA = new QualifiedStreamId("provider", streamA); + var qualifiedB = new QualifiedStreamId("provider", streamB); + var queueCache = new ScriptedQueueCache(); + queueCache.AddToCache( + [ + .. Enumerable.Range(2, 8) + .Select(sequence => (IBatchContainer)new TestBatchContainer( + streamB, + new EventSequenceTokenV2(sequence))), + new TestBatchContainer(streamA, new EventSequenceTokenV2(10)), + ]); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedA, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + await testAccessor.RegisterStream(qualifiedB, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache())[qualifiedA]; + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedA, + new StartingConsumer(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(10))), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + + Assert.True(await testAccessor.DoHandshakeWithConsumer( + consumerData, + cacheToken: new EventSequenceTokenV2(2))); + consumerData.IsRegistered = true; + Assert.Equal(10, consumerData.LastProcessedToken?.SequenceNumber); + Assert.Null(consumerData.LastSafePartitionToken); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + await testAccessor.RunConsumerCursor(consumerData); + Assert.Equal(10, consumerData.LastSafePartitionToken?.SequenceNumber); + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(10, Assert.Single(queueCache.DeliveryProgressTokens)?.SequenceNumber); + await testAccessor.Shutdown(); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4740,6 +4807,7 @@ public async Task DeliveryProgress_IncludesFilteredBatches() now: timeProvider.GetUtcNow().UtcDateTime); consumerData.IsRegistered = true; consumerData.LastProcessedToken = previousToken; + consumerData.LastSafePartitionToken = previousToken; consumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); queueCache.ClearDeliveryProgress(); @@ -4791,6 +4859,7 @@ public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() now: timeProvider.GetUtcNow().UtcDateTime); slowConsumerData.IsRegistered = true; slowConsumerData.LastProcessedToken = previousToken; + slowConsumerData.LastSafePartitionToken = previousToken; slowConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); var fastConsumer = new ImmediateConsumer(); @@ -4802,6 +4871,7 @@ public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() now: timeProvider.GetUtcNow().UtcDateTime); fastConsumerData.IsRegistered = true; fastConsumerData.LastProcessedToken = previousToken; + fastConsumerData.LastSafePartitionToken = previousToken; fastConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); queueCache.ClearDeliveryProgress(); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index eaefab55bd0..3298f22bf93 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -469,6 +469,34 @@ public void Cache_BatchedMatchesRemainPendingUntilWholeDeliverySucceeds() Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); } + [Fact] + public void Cache_DeliveredThroughScansIntermediatePartitionRecordsWithoutRedelivery() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamB, 3, "b-3"), + new TestQueueMessage(streamA, 10, "a-10"), + new TestQueueMessage(streamA, 11, "a-11"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + progress.SetDeliveredThrough(new EventSequenceTokenV2(10)); + + Assert.True(cursor.MoveNext()); + Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.Equal(10, progress.SafeSequenceToken?.SequenceNumber); + } + [Fact] public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetry() { From bc500511194bacb7dc8ee4db125d59841517ede0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 20:59:43 -0700 Subject: [PATCH 23/90] fix(streaming): rewind failed cache delivery --- .../Common/PooledCache/PooledQueueCache.cs | 35 +++++++++++++++++++ .../RecoverableStreamQueueCache.cs | 1 + .../RecoverableStreamReceiverTests.cs | 35 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index f67cf176549..fe4e074cfaf 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -618,6 +618,20 @@ internal void SetCursorDeliveredThrough(object cursorObj, StreamSequenceToken to internal void RecordDeliverySuccess(object cursorObj) => GetCursor(cursorObj).RecordDeliverySuccess(); + internal void RecordDeliveryFailure(object cursorObj) + { + var cursor = GetCursor(cursorObj); + if (cursor.TakePendingStartToken() is not { } retryToken) + { + return; + } + + cursor.State = CursorStates.Unset; + cursor.CurrentBlock = null; + cursor.SequenceToken = retryToken; + SetCursor(cursor, retryToken); + } + private Cursor GetCursor(object cursorObj) => cursorObj as Cursor ?? throw new ArgumentOutOfRangeException(nameof(cursorObj), "Cursor is bad"); @@ -730,6 +744,7 @@ public Cursor(StreamId streamId) public StreamSequenceToken? SafeSequenceToken; public StreamSequenceToken? DeliveredThroughToken; private StreamSequenceToken? pendingSequenceToken; + private StreamSequenceToken? pendingStartToken; private bool hasPendingDelivery; // reference into cache; non-null while State is Set @@ -755,6 +770,11 @@ public void RecordScanned(StreamSequenceToken token) public void RecordPending(StreamSequenceToken token) { + if (!hasPendingDelivery) + { + pendingStartToken = token; + } + hasPendingDelivery = true; pendingSequenceToken = token; } @@ -768,7 +788,22 @@ public void RecordDeliverySuccess() SafeSequenceToken = pendingSequenceToken; pendingSequenceToken = null; + pendingStartToken = null; + hasPendingDelivery = false; + } + + public StreamSequenceToken? TakePendingStartToken() + { + if (!hasPendingDelivery) + { + return null; + } + + var result = pendingStartToken; + pendingSequenceToken = null; + pendingStartToken = null; hasPendingDelivery = false; + return result; } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index b36fba76e0a..ddb5136f565 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -295,6 +295,7 @@ public bool MoveNext() public void RecordDeliveryFailure() { + _cache.RecordDeliveryFailure(_cursor); } public void RecordDeliverySuccess() => _cache.RecordDeliverySuccess(_cursor); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 3298f22bf93..76d6e413485 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -497,6 +497,41 @@ public void Cache_DeliveredThroughScansIntermediatePartitionRecordsWithoutRedeli Assert.Equal(10, progress.SafeSequenceToken?.SequenceNumber); } + [Fact] + public void Cache_DeliveryFailureRewindsToFirstPendingRecord() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamA, 3, "a-3"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.True(cursor.MoveNext()); + Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + + cursor.RecordDeliveryFailure(); + + Assert.Null(progress.SafeSequenceToken); + Assert.True(cursor.MoveNext()); + Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + progress.RecordDeliverySuccess(); + Assert.True(cursor.MoveNext()); + Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + } + [Fact] public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetry() { From a731f4607f0e9b708666eb76f91600c62ba7c813 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 21:28:19 -0700 Subject: [PATCH 24/90] fix(streaming): retain from checkpoint eligibility --- .../MySQL-Streaming.sql | 19 ++- .../PostgreSQL-Streaming.sql | 16 ++- .../SQLServer-Streaming.sql | 119 ++++++++++++------ .../Streaming/AdoNetStreamPartitionTests.cs | 55 +++++++- .../Streaming/AdoNetStreamSchemaTests.cs | 5 + 5 files changed, 171 insertions(+), 43 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index bfbe6b9c9d9..e5f664dea3d 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -76,6 +76,7 @@ CREATE TABLE OrleansStreamMessage StreamIdBytes LONGBLOB NOT NULL, StreamNamespaceLength INT NOT NULL, CreatedOn DATETIME(6) NOT NULL, + CheckpointedOn DATETIME(6) NULL, Payload LONGBLOB NOT NULL, PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) @@ -270,6 +271,13 @@ BEGIN AND ProviderId = _ProviderId AND QueueId = _QueueId; + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, _Now) + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND MessageId <= _Checkpoint; + SELECT OwnerEpoch INTO _OwnerEpoch FROM OrleansStreamPartition @@ -330,6 +338,15 @@ BEGIN SET _Updated = ROW_COUNT() = 1; + IF _Updated THEN + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, UTC_TIMESTAMP(6)) + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND MessageId <= _Checkpoint; + END IF; + SELECT OwnerEpoch, Checkpoint INTO _CurrentOwnerEpoch, _CurrentCheckpoint FROM OrleansStreamPartition @@ -446,7 +463,7 @@ BEGIN ( _Checkpoint IS NOT NULL AND MessageId <= _Checkpoint - AND CreatedOn < DATE_SUB(_Now, INTERVAL _RetentionPeriodSeconds SECOND) + AND CheckpointedOn < DATE_SUB(_Now, INTERVAL _RetentionPeriodSeconds SECOND) ) OR ( diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 576bb868eb1..6b8ad117c25 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -63,6 +63,7 @@ CREATE TABLE OrleansStreamMessage StreamIdBytes BYTEA NOT NULL, StreamNamespaceLength INT NOT NULL, CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + CheckpointedOn TIMESTAMP(6) WITHOUT TIME ZONE NULL, Payload BYTEA NOT NULL, CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY @@ -246,6 +247,13 @@ BEGIN AND P.QueueId = _QueueId RETURNING P.OwnerEpoch INTO _OwnerEpoch; + UPDATE OrleansStreamMessage AS M + SET CheckpointedOn = COALESCE(M.CheckpointedOn, _Now) + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND M.MessageId <= _Checkpoint; + RETURN QUERY SELECT _ServiceId, @@ -298,6 +306,12 @@ BEGIN IF FOUND THEN _Updated := TRUE; + UPDATE OrleansStreamMessage AS M + SET CheckpointedOn = COALESCE(M.CheckpointedOn, CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND M.MessageId <= _Checkpoint; ELSE SELECT P.OwnerEpoch, P.Checkpoint INTO _CurrentOwnerEpoch, _CurrentCheckpoint @@ -407,7 +421,7 @@ BEGIN ( _Checkpoint IS NOT NULL AND M.MessageId <= _Checkpoint - AND M.CreatedOn < _Now - make_interval(secs => _RetentionPeriodSeconds) + AND M.CheckpointedOn < _Now - make_interval(secs => _RetentionPeriodSeconds) ) OR ( diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index aac7578a96f..8702fa68e01 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -61,6 +61,7 @@ CREATE TABLE OrleansStreamMessage StreamIdBytes VARBINARY(MAX) NOT NULL, StreamNamespaceLength INT NOT NULL, CreatedOn DATETIME2(7) NOT NULL, + CheckpointedOn DATETIME2(7) NULL, Payload VARBINARY(MAX) NOT NULL, CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY CLUSTERED @@ -360,6 +361,13 @@ BEGIN AND ProviderId = @ProviderId AND QueueId = @QueueId; + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, @Now) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND MessageId <= @Checkpoint; + SELECT @OwnerEpoch = OwnerEpoch FROM OrleansStreamPartition WHERE ServiceId = @ServiceId @@ -400,6 +408,7 @@ CREATE PROCEDURE AdvanceStreamCheckpoint AS BEGIN SET NOCOUNT ON; + SET XACT_ABORT ON; DECLARE @Result TABLE ( @@ -410,44 +419,76 @@ BEGIN [Checkpoint] BIGINT NULL, Updated BIT NOT NULL ); + DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @StartedTransaction BIT = 0; - UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) - SET - [Checkpoint] = @Checkpoint, - ModifiedOn = SYSUTCDATETIME() - OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.OwnerEpoch, - Inserted.[Checkpoint], - CAST(1 AS BIT) - INTO @Result - WHERE ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND OwnerEpoch = @OwnerEpoch - AND ([Checkpoint] IS NULL OR [Checkpoint] < @Checkpoint) - AND @Checkpoint < NextMessageId; - - IF EXISTS (SELECT 1 FROM @Result) - BEGIN - SELECT ServiceId, ProviderId, QueueId, OwnerEpoch, [Checkpoint], Updated - FROM @Result; - RETURN; - END; - - SELECT - ServiceId, - ProviderId, - QueueId, - OwnerEpoch, - [Checkpoint], - CAST(0 AS BIT) AS Updated - FROM OrleansStreamPartition - WHERE ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId; + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + [Checkpoint] = @Checkpoint, + ModifiedOn = @Now + OUTPUT + Inserted.ServiceId, + Inserted.ProviderId, + Inserted.QueueId, + Inserted.OwnerEpoch, + Inserted.[Checkpoint], + CAST(1 AS BIT) + INTO @Result + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND OwnerEpoch = @OwnerEpoch + AND ([Checkpoint] IS NULL OR [Checkpoint] < @Checkpoint) + AND @Checkpoint < NextMessageId; + + IF EXISTS (SELECT 1 FROM @Result) + BEGIN + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, @Now) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND MessageId <= @Checkpoint; + END; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + + IF EXISTS (SELECT 1 FROM @Result) + BEGIN + SELECT ServiceId, ProviderId, QueueId, OwnerEpoch, [Checkpoint], Updated + FROM @Result; + RETURN; + END; + + SELECT + ServiceId, + ProviderId, + QueueId, + OwnerEpoch, + [Checkpoint], + CAST(0 AS BIT) AS Updated + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + THROW; + END CATCH; END; GO @@ -521,7 +562,7 @@ BEGIN ( SELECT TOP (@CleanupBatchSize) MessageId - FROM OrleansStreamMessage WITH (UPDLOCK, READPAST, ROWLOCK) + FROM OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId @@ -530,7 +571,7 @@ BEGIN ( @Checkpoint IS NOT NULL AND MessageId <= @Checkpoint - AND CreatedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) + AND CheckpointedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) ) OR ( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index 7f848fc76c6..22d6613d6e4 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -148,6 +148,21 @@ UPDATE OrleansStreamMessage AddParameter(command, "QueueId", queueId); }); + private Task AgeCheckpointedMessagesAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamMessage + SET CheckpointedOn = @CheckpointedOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CheckpointedOn", DateTime.UtcNow.AddDays(-2)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + private Task MakeCleanupDueAsync(string serviceId, string providerId, string queueId) => _storage.ExecuteAsync( """ @@ -660,7 +675,7 @@ public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentio var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); - await AgePartitionMessagesAsync(serviceId, providerId, queueId); + await AgeCheckpointedMessagesAsync(serviceId, providerId, queueId); var result = await _queries.CleanupStreamMessagesAsync( serviceId, providerId, queueId, @@ -682,6 +697,42 @@ public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentio Assert.Empty(remaining); } + [Fact] + public async Task CleanupStreamMessages_RetentionStartsWhenCheckpointAdvances() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var appended = await AppendAsync(serviceId, providerId, queueId); + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + await _queries.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + state.OwnerEpoch, + appended.MessageId); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + var cleanup = await _queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(cleanup.Ran); + Assert.Equal(0, cleanup.DeletedCount); + Assert.Single(await _queries.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + afterMessageId: 0, + maxCount: 100)); + } + [Fact] public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics() { @@ -731,7 +782,7 @@ public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); - await AgePartitionMessagesAsync(serviceId, providerId, queueId); + await AgeCheckpointedMessagesAsync(serviceId, providerId, queueId); var first = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); Assert.True(first.Ran); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 0fc9d66b876..0ef94223ca7 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -23,6 +23,10 @@ public void SchemaDefinesVersionedStreamPartitions(string provider) Assert.Contains("CREATE TABLE OrleansStreamMessage", script, StringComparison.Ordinal); Assert.Contains("StreamIdBytes", script, StringComparison.Ordinal); Assert.Contains("StreamNamespaceLength INT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn = COALESCE", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn <", script, StringComparison.Ordinal); + Assert.Contains("CreatedOn <", script, StringComparison.Ordinal); Assert.Contains("Payload", script, StringComparison.Ordinal); Assert.Contains("'StreamSchemaVersionKey', '2'", script, StringComparison.Ordinal); @@ -46,6 +50,7 @@ public void SqlServerAppendUsesUpdateOutputWhileHoldingPartitionLock() Assert.Contains("UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK)", script, StringComparison.Ordinal); Assert.Contains("OUTPUT Inserted.NextMessageId - 1", script, StringComparison.Ordinal); Assert.Contains("@LockOwner = 'Transaction'", script, StringComparison.Ordinal); + Assert.Contains("READPAST, READCOMMITTEDLOCK", script, StringComparison.Ordinal); } [Fact] From effe067adfb9c2a23f38a0079d16d1d652928c1c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 22:03:19 -0700 Subject: [PATCH 25/90] fix(streaming): cancel Kinesis initialization attempts --- .../Streams/KinesisPooledAdapterReceiver.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index df361637d6f..d547f7ea8fd 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -203,16 +203,19 @@ private Task EnsureInitialized(CancellationToken cancellationToken) if (_initializationTask is null || _initializationTask.IsCompleted) { - _initializationTask = InitializeCore(); + _initializationTask = InitializeCore(cancellationToken); } return _initializationTask.WaitAsync(cancellationToken); } } - private async Task InitializeCore() + private async Task InitializeCore(CancellationToken initializationToken) { - var lifecycleToken = _lifecycleCancellation.Token; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + _lifecycleCancellation.Token, + initializationToken); + var lifecycleToken = cancellation.Token; if (_inner is null) { var checkpointer = await _checkpointerFactory.Create(_partition, lifecycleToken); From 8a29fbcaa048feebb876fc0d65c31fa28264c5e0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 22:44:38 -0700 Subject: [PATCH 26/90] perf(streaming): limit checkpoint eligibility updates --- src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql | 6 ++++-- .../Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql | 6 ++++-- src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql | 6 ++++-- .../Streaming/AdoNetStreamSchemaTests.cs | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index e5f664dea3d..d947b31b071 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -276,7 +276,8 @@ BEGIN WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId - AND MessageId <= _Checkpoint; + AND MessageId <= _Checkpoint + AND CheckpointedOn IS NULL; SELECT OwnerEpoch INTO _OwnerEpoch @@ -344,7 +345,8 @@ BEGIN WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId - AND MessageId <= _Checkpoint; + AND MessageId <= _Checkpoint + AND CheckpointedOn IS NULL; END IF; SELECT OwnerEpoch, Checkpoint diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 6b8ad117c25..190626fcaf4 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -252,7 +252,8 @@ BEGIN WHERE M.ServiceId = _ServiceId AND M.ProviderId = _ProviderId AND M.QueueId = _QueueId - AND M.MessageId <= _Checkpoint; + AND M.MessageId <= _Checkpoint + AND M.CheckpointedOn IS NULL; RETURN QUERY SELECT @@ -311,7 +312,8 @@ BEGIN WHERE M.ServiceId = _ServiceId AND M.ProviderId = _ProviderId AND M.QueueId = _QueueId - AND M.MessageId <= _Checkpoint; + AND M.MessageId <= _Checkpoint + AND M.CheckpointedOn IS NULL; ELSE SELECT P.OwnerEpoch, P.Checkpoint INTO _CurrentOwnerEpoch, _CurrentCheckpoint diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index 8702fa68e01..8131efd03af 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -366,7 +366,8 @@ BEGIN WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId - AND MessageId <= @Checkpoint; + AND MessageId <= @Checkpoint + AND CheckpointedOn IS NULL; SELECT @OwnerEpoch = OwnerEpoch FROM OrleansStreamPartition @@ -455,7 +456,8 @@ BEGIN WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId - AND MessageId <= @Checkpoint; + AND MessageId <= @Checkpoint + AND CheckpointedOn IS NULL; END; IF @StartedTransaction = 1 diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 0ef94223ca7..5a51bd00bab 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -25,6 +25,7 @@ public void SchemaDefinesVersionedStreamPartitions(string provider) Assert.Contains("StreamNamespaceLength INT NOT NULL", script, StringComparison.Ordinal); Assert.Contains("CheckpointedOn", script, StringComparison.Ordinal); Assert.Contains("CheckpointedOn = COALESCE", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn IS NULL", script, StringComparison.Ordinal); Assert.Contains("CheckpointedOn <", script, StringComparison.Ordinal); Assert.Contains("CreatedOn <", script, StringComparison.Ordinal); Assert.Contains("Payload", script, StringComparison.Ordinal); From b3aa49e076c648fdaa58b4ffc12c7b56ca19b2a0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 08:18:24 -0700 Subject: [PATCH 27/90] fix(streaming): preserve legacy resume boundaries --- .../PersistentStreams/PersistentStreamPullingAgent.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 1b60032b8d6..10db304fb0f 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -483,7 +483,8 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { - cursorStartToken = requestedHandshakeToken is DeliveryToken + var isDeliveryToken = requestedHandshakeToken is DeliveryToken; + cursorStartToken = isDeliveryToken ? cacheToken ?? consumerData.PendingStartToken ?? requestedToken : requestedToken; consumerData.SafeDisposeCursor(logger); From aaf3de5d394877aa532839215742506947b1a32c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 09:24:44 -0700 Subject: [PATCH 28/90] fix(streaming): address Kinesis cache review Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../Streams/KinesisRecoverableStream.cs | 8 ++++++-- .../Common/PooledCache/CachedMessageBlock.cs | 4 ++-- .../KinesisBatchContainerTests.cs | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs index 10ffa2f5fe1..c11ff68d18c 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -11,11 +11,15 @@ internal sealed class KinesisCacheRecord( Record record, long sequenceNumber) { + private byte[]? _rawPayload; + public Record Record { get; } = record; public long SequenceNumber { get; } = sequenceNumber; public KinesisBatchContainer.Body? Body { get; set; } + + public byte[] RawPayload => _rawPayload ??= Record.Data.ToArray(); } internal sealed class KinesisRecoverableStreamSource( @@ -166,7 +170,7 @@ internal sealed class KinesisRecoverableStreamDataAdapter( { public StreamPosition GetStreamPosition(KinesisCacheRecord queueMessage) { - queueMessage.Body ??= serializer.Deserialize(queueMessage.Record.Data.ToArray())!; + queueMessage.Body ??= serializer.Deserialize(queueMessage.RawPayload)!; return new( queueMessage.Body.StreamId, new KinesisSequenceToken( @@ -181,7 +185,7 @@ public CachedMessage FromQueueMessage( DateTime dequeueTimeUtc, Func> getSegment) { - var payload = queueMessage.Record.Data.ToArray(); + var payload = queueMessage.RawPayload; var size = SegmentBuilder.CalculateAppendSize(queueMessage.Record.SequenceNumber) + SegmentBuilder.CalculateAppendSize(payload); var segment = getSegment(size); diff --git a/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs b/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs index be211ad8f99..3ad0e9c50eb 100644 --- a/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs +++ b/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs @@ -177,10 +177,10 @@ public StreamSequenceToken GetOldestSequenceToken(ICacheDataAdapter dataAdapter) } /// - /// Gets the index of the first message in this block that has a sequence token at or before the provided token + /// Gets the index of the newest message in this block whose sequence token is less than or equal to the provided token. /// /// The sequence token. - /// The index of the first message in this block that has a sequence token equal to or before the provided token. + /// The index of the newest message whose sequence token is less than or equal to the provided token. public int GetIndexOfFirstMessageLessThanOrEqualTo(StreamSequenceToken token) { for (int i = writeIndex - 1; i >= readIndex; i--) diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index 9493349d596..3222da5bfd0 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -144,6 +144,8 @@ public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering( var adapter = new KinesisRecoverableStreamDataAdapter(serializer); var position = adapter.GetStreamPosition(queueMessage); + var rawPayload = queueMessage.RawPayload; + record.Data = null!; var cached = adapter.FromQueueMessage( position, queueMessage, @@ -151,6 +153,7 @@ public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering( size => new byte[size]); Assert.Equal(streamId, cached.StreamId); + Assert.Same(rawPayload, queueMessage.RawPayload); Assert.Equal(record.SequenceNumber, adapter.GetOffset(ref cached)); Assert.True(adapter.Compare( ref cached, From b238866a6cb88b0ab140f1d57f7bb96785c2bc41 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 09:53:38 -0700 Subject: [PATCH 29/90] fix(streaming): retry canceled initialization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../RecoverableStreamReceiver.cs | 39 +++++++++--- .../RecoverableStreamReceiverTests.cs | 63 ++++++++++++++++++- 2 files changed, 92 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 4461cfecd3b..c636a769ea4 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -22,6 +22,7 @@ public sealed class RecoverableStreamReceiver : IQueueAdapterRece private readonly object _lifecycleLock = new(); private readonly CancellationTokenSource _lifecycleCancellation = new(); private Task? _initializeTask; + private CancellationToken _initializeTaskOwnerToken; private int _running; private int _shutdown; @@ -72,22 +73,44 @@ public async Task Initialize(TimeSpan timeout) public Task Initialize(CancellationToken cancellationToken) => EnsureInitialized(cancellationToken); - private Task EnsureInitialized(CancellationToken cancellationToken) + private async Task EnsureInitialized(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - lock (_lifecycleLock) + while (true) { - if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) + Task initializeTask; + CancellationToken initializeTaskOwnerToken; + lock (_lifecycleLock) { - return Task.CompletedTask; + if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) + { + return; + } + + if (_initializeTask is null || _initializeTask.IsCompleted) + { + _initializeTaskOwnerToken = cancellationToken; + _initializeTask = InitializeCore(cancellationToken); + } + + initializeTask = _initializeTask; + initializeTaskOwnerToken = _initializeTaskOwnerToken; } - if (_initializeTask is null || _initializeTask.IsCompleted) + try { - _initializeTask = InitializeCore(cancellationToken); + await initializeTask.WaitAsync(cancellationToken); + return; + } + catch (OperationCanceledException) + when (!cancellationToken.IsCancellationRequested + && Volatile.Read(ref _shutdown) == 0 + && initializeTaskOwnerToken.IsCancellationRequested + && initializeTask.IsCanceled) + { + // The caller which started this shared initialization canceled it. Once that + // task has settled, loop and create a fresh attempt for this still-active caller. } - - return _initializeTask.WaitAsync(cancellationToken); } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 76d6e413485..09a1dc1d529 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -70,6 +70,12 @@ private sealed class BlockingInitializationCheckpointer : IStreamQueueCheckpoint public TaskCompletionSource FirstLoadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource FirstLoadCancellationObserved { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowFirstLoadToComplete { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken FirstLoadCancellation { get; private set; } public int LoadCount { get; private set; } @@ -85,7 +91,12 @@ public async Task Load(CancellationToken cancellationToken) { FirstLoadCancellation = cancellationToken; FirstLoadStarted.TrySetResult(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + using var registration = cancellationToken.Register( + static state => ((TaskCompletionSource)state!).TrySetResult(), + FirstLoadCancellationObserved); + await FirstLoadCancellationObserved.Task; + await AllowFirstLoadToComplete.Task; + cancellationToken.ThrowIfCancellationRequested(); } return string.Empty; @@ -103,6 +114,34 @@ public Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class IndependentlyCanceledCheckpointer : IStreamQueueCheckpointer + { + public int LoadCount { get; private set; } + + public bool CheckpointExists => false; + + public Task Load() => Load(CancellationToken.None); + + public async Task Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + LoadCount++; + await Task.Yield(); + throw new OperationCanceledException(new CancellationToken(canceled: true)); + } + + public void Update(string offset, DateTime utcNow) { } + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => cancellationToken.ThrowIfCancellationRequested(); + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + [Fact] public async Task Receiver_RestartRedeliversInclusiveBatchWhenCheckpointDidNotAdvance() { @@ -547,12 +586,32 @@ public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetr await Assert.ThrowsAnyAsync(() => initialization); Assert.True(checkpointer.FirstLoadCancellation.IsCancellationRequested); - Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + var retry = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + Assert.Equal(1, checkpointer.LoadCount); + checkpointer.AllowFirstLoadToComplete.TrySetResult(); + + Assert.Empty(await retry); Assert.Equal(2, checkpointer.LoadCount); Assert.Equal(1, source.InitializeCount); await receiver.Shutdown(TimeSpan.FromSeconds(5)); } + [Fact] + public async Task Receiver_IndependentInitializationCancellationIsNotRetried() + { + var source = new TestSource([]); + var checkpointer = new IndependentlyCanceledCheckpointer(); + var receiver = CreateReceiver(source, checkpointer); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(5))); + + Assert.Equal(1, checkpointer.LoadCount); + Assert.Equal(0, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + [Fact] public void Cache_QuietCursorAdvancesAcrossUnrelatedRecords() { From fdfbf39098571f1517dd86bcb755b5e9be2acebd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 12:16:03 -0700 Subject: [PATCH 30/90] fix(streaming): close retained recovery gaps --- .../Streams/KinesisPooledAdapterReceiver.cs | 40 +++++++--- .../AdoNetRecoverableStream.cs | 18 ++++- .../AdoNetStreamPartitionState.cs | 1 + .../MySQL-Streaming.sql | 38 +++++++--- .../PostgreSQL-Streaming.sql | 41 +++++++--- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 4 +- .../SQLServer-Streaming.sql | 41 +++++++--- .../Storage/RelationalOrleansQueries.cs | 2 + .../Streaming/AdoNetRecoverableStreamTests.cs | 27 +++++++ .../Streaming/AdoNetStreamPartitionTests.cs | 41 ++++++++++ .../Streaming/AdoNetStreamSchemaTests.cs | 51 +++++++++++++ .../RedisStreamBatchContainerTests.cs | 14 ++++ .../KinesisRuntimeTests.cs | 74 +++++++++++++++++++ .../PersistentStreamPullingAgentTests.cs | 10 +++ 14 files changed, 355 insertions(+), 47 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index d547f7ea8fd..01fd05d57e5 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -24,6 +24,7 @@ internal sealed class KinesisPooledAdapterReceiver : IQueueAdapterReceiver, IQue private readonly CancellationTokenSource _lifecycleCancellation = new(); private RecoverableStreamReceiver? _inner; private Task? _initializationTask; + private CancellationToken _initializationOwnerToken; private int _initialized; private int _shutdown; @@ -191,22 +192,43 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) => _inner?.UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); - private Task EnsureInitialized(CancellationToken cancellationToken) + private async Task EnsureInitialized(CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - lock (_lifecycleLock) + while (true) { - if (Volatile.Read(ref _shutdown) != 0 || Volatile.Read(ref _initialized) != 0) + cancellationToken.ThrowIfCancellationRequested(); + Task initializationTask; + CancellationToken initializationOwnerToken; + lock (_lifecycleLock) { - return Task.CompletedTask; + if (Volatile.Read(ref _shutdown) != 0 || Volatile.Read(ref _initialized) != 0) + { + return; + } + + if (_initializationTask is null || _initializationTask.IsCompleted) + { + _initializationOwnerToken = cancellationToken; + _initializationTask = InitializeCore(cancellationToken); + } + + initializationTask = _initializationTask; + initializationOwnerToken = _initializationOwnerToken; } - if (_initializationTask is null || _initializationTask.IsCompleted) + try { - _initializationTask = InitializeCore(cancellationToken); + await initializationTask.WaitAsync(cancellationToken); + return; + } + catch (OperationCanceledException) when ( + !cancellationToken.IsCancellationRequested + && initializationOwnerToken.IsCancellationRequested + && initializationTask.IsCanceled) + { + // The caller which owned the shared initialization attempt canceled. + // An unaffected caller retries after that attempt has settled. } - - return _initializationTask.WaitAsync(cancellationToken); } } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index 3d206164333..82f71385a1d 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -146,16 +146,26 @@ public Task Shutdown(CancellationToken cancellationToken) private void ThrowIfRetentionGap(AdoNetStreamPartitionState state) { - if (state.Checkpoint is { } checkpoint - && state.EarliestMessageId is { } earliest - && checkpoint < earliest - 1) + if (HasRetentionGap(state)) { throw new DataNotAvailableException( $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' has a retention gap: " - + $"checkpoint {checkpoint}, earliest retained record {earliest}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); + + $"checkpoint {state.Checkpoint}, earliest retained record {state.EarliestMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}, " + + $"next message id {state.NextMessageId}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); } } + internal static bool HasRetentionGap(AdoNetStreamPartitionState state) + { + if (state.Checkpoint is not { } checkpoint) + { + return false; + } + + var earliestAvailablePosition = state.EarliestMessageId ?? state.NextMessageId; + return checkpoint < earliestAvailablePosition - 1; + } + [LoggerMessage( Level = LogLevel.Warning, Message = "Hard stream retention deleted {DeletedCount} records for {ServiceId}/{ProviderId}/{QueueId} from {DeletedFrom} through {DeletedThrough}, crossing checkpoint {Checkpoint}.")] diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs index 0d8e44f7933..a84356007e9 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs @@ -8,6 +8,7 @@ internal record AdoNetStreamPartitionState( string ProviderId, string QueueId, long OwnerEpoch, + long NextMessageId, long? Checkpoint, long? EarliestMessageId, long? TailMessageId); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index d947b31b071..a45b76994ac 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -95,7 +95,7 @@ CREATE PROCEDURE AppendStreamMessage IN _ManageTransaction BOOLEAN ) BEGIN - DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _Now DATETIME(6); DECLARE _MessageId BIGINT; DECLARE EXIT HANDLER FOR SQLEXCEPTION @@ -130,9 +130,9 @@ BEGIN 1, NULL, 0, - _Now, - _Now, - _Now + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6) ) ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; @@ -144,6 +144,8 @@ BEGIN AND QueueId = _QueueId FOR UPDATE; + SET _Now = UTC_TIMESTAMP(6); + UPDATE OrleansStreamPartition SET NextMessageId = _MessageId + 1, @@ -195,7 +197,7 @@ CREATE PROCEDURE AcquireStreamPartition IN _ManageTransaction BOOLEAN ) BEGIN - DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _Now DATETIME(6); DECLARE _NextMessageId BIGINT; DECLARE _Checkpoint BIGINT; DECLARE _OwnerEpoch BIGINT; @@ -234,9 +236,9 @@ BEGIN 1, NULL, 0, - _Now, - _Now, - _Now + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6) ) ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; @@ -248,6 +250,8 @@ BEGIN AND QueueId = _QueueId FOR UPDATE; + SET _Now = UTC_TIMESTAMP(6); + SELECT MIN(MessageId), MAX(MessageId) INTO _EarliestMessageId, _TailMessageId FROM OrleansStreamMessage @@ -295,6 +299,7 @@ BEGIN _ProviderId AS ProviderId, _QueueId AS QueueId, _OwnerEpoch AS OwnerEpoch, + _NextMessageId AS NextMessageId, _Checkpoint AS Checkpoint, _EarliestMessageId AS EarliestMessageId, _TailMessageId AS TailMessageId; @@ -310,6 +315,7 @@ CREATE PROCEDURE AdvanceStreamCheckpoint IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6); DECLARE _CurrentOwnerEpoch BIGINT; DECLARE _CurrentCheckpoint BIGINT; DECLARE _Updated BOOLEAN DEFAULT FALSE; @@ -326,10 +332,20 @@ BEGIN START TRANSACTION; END IF; + SELECT OwnerEpoch, Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; + + SET _Now = UTC_TIMESTAMP(6); + UPDATE OrleansStreamPartition SET Checkpoint = _Checkpoint, - ModifiedOn = UTC_TIMESTAMP(6) + ModifiedOn = _Now WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId @@ -341,7 +357,7 @@ BEGIN IF _Updated THEN UPDATE OrleansStreamMessage - SET CheckpointedOn = COALESCE(CheckpointedOn, UTC_TIMESTAMP(6)) + SET CheckpointedOn = COALESCE(CheckpointedOn, _Now) WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId @@ -534,5 +550,5 @@ VALUES ('AcquireStreamPartitionKey', 'CALL AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow, TRUE)'), ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), ('AdvanceStreamCheckpointKey', 'CALL AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint, TRUE)'), - ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint'), ('CleanupStreamMessagesKey', 'CALL CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize, TRUE)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 190626fcaf4..454e8de03a9 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -95,7 +95,7 @@ LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _Now TIMESTAMP(6) WITHOUT TIME ZONE; _MessageId BIGINT; BEGIN INSERT INTO OrleansStreamPartition @@ -118,21 +118,23 @@ BEGIN 1, NULL, 0, - _Now, - _Now, - _Now + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC' ) ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; UPDATE OrleansStreamPartition AS P SET NextMessageId = P.NextMessageId + 1, - ModifiedOn = _Now + ModifiedOn = clock_timestamp() AT TIME ZONE 'UTC' WHERE P.ServiceId = _ServiceId AND P.ProviderId = _ProviderId AND P.QueueId = _QueueId RETURNING P.NextMessageId - 1 INTO _MessageId; + _Now := clock_timestamp() AT TIME ZONE 'UTC'; + INSERT INTO OrleansStreamMessage ( ServiceId, @@ -174,6 +176,7 @@ RETURNS TABLE ProviderId VARCHAR(150), QueueId VARCHAR(150), OwnerEpoch BIGINT, + NextMessageId BIGINT, Checkpoint BIGINT, EarliestMessageId BIGINT, TailMessageId BIGINT @@ -182,7 +185,7 @@ LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _Now TIMESTAMP(6) WITHOUT TIME ZONE; _NextMessageId BIGINT; _Checkpoint BIGINT; _OwnerEpoch BIGINT; @@ -209,9 +212,9 @@ BEGIN 1, NULL, 0, - _Now, - _Now, - _Now + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC' ) ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; @@ -223,6 +226,8 @@ BEGIN AND P.QueueId = _QueueId FOR UPDATE; + _Now := clock_timestamp() AT TIME ZONE 'UTC'; + SELECT MIN(M.MessageId), MAX(M.MessageId) INTO _EarliestMessageId, _TailMessageId FROM OrleansStreamMessage AS M @@ -261,6 +266,7 @@ BEGIN _ProviderId, _QueueId, _OwnerEpoch, + _NextMessageId, _Checkpoint, _EarliestMessageId, _TailMessageId; @@ -288,14 +294,25 @@ LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE + _Now TIMESTAMP(6) WITHOUT TIME ZONE; _CurrentOwnerEpoch BIGINT; _CurrentCheckpoint BIGINT; _Updated BOOLEAN := FALSE; BEGIN + SELECT P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + FOR UPDATE; + + _Now := clock_timestamp() AT TIME ZONE 'UTC'; + UPDATE OrleansStreamPartition AS P SET Checkpoint = _Checkpoint, - ModifiedOn = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' + ModifiedOn = _Now WHERE P.ServiceId = _ServiceId AND P.ProviderId = _ProviderId AND P.QueueId = _QueueId @@ -308,7 +325,7 @@ BEGIN IF FOUND THEN _Updated := TRUE; UPDATE OrleansStreamMessage AS M - SET CheckpointedOn = COALESCE(M.CheckpointedOn, CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + SET CheckpointedOn = COALESCE(M.CheckpointedOn, _Now) WHERE M.ServiceId = _ServiceId AND M.ProviderId = _ProviderId AND M.QueueId = _QueueId @@ -487,5 +504,5 @@ VALUES ('AcquireStreamPartitionKey', 'SELECT * FROM AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow)'), ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), ('AdvanceStreamCheckpointKey', 'SELECT * FROM AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint)'), - ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.Checkpoint'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint'), ('CleanupStreamMessagesKey', 'SELECT * FROM CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index d3169b67b81..3d77f2d7bbb 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -58,10 +58,12 @@ The provider stores each queue as an immutable, ordered stream partition. Its st - `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. Fractional seconds round upward. - `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. Fractional cleanup intervals round upward. -The partitioned stream provider is rewindable while requested records remain retained. Inclusive subscription start positions remain pending until the corresponding record is delivered or intentionally filtered. The queue checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. It resumes strictly after that durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. +The partitioned stream provider is rewindable while requested records remain retained. Inclusive subscription start positions remain pending until the corresponding record is delivered or intentionally filtered. The queue checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. It resumes strictly after that durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. Partition state retains the next message identifier, so recovery detects a hard-retention gap even when the purge leaves no records. Partition acquisition is cancellation-aware. A receiver whose acquisition command is still completing retains its queue reservation, so a late database result settles before a replacement receiver acquires a newer ownership epoch. +Message creation and checkpoint-eligibility timestamps are sampled after the partition lock is acquired. Lock contention therefore does not consume the configured retention window. + ## Alpha schema upgrade The current streaming scripts use schema version 2 and are intentionally incompatible with the former queue, visibility-timeout, confirmation, and dead-letter schema. The provider fails during initialization when it detects old or mixed streaming query keys. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index 8131efd03af..6a2135b932e 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -87,7 +87,8 @@ BEGIN SET XACT_ABORT ON; DECLARE @StartedTransaction BIT = 0; - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Now DATETIME2(7); + DECLARE @LockedNextMessageId BIGINT; DECLARE @Allocated TABLE (MessageId BIGINT NOT NULL); BEGIN TRY @@ -166,13 +167,21 @@ BEGIN 1, NULL, 0, - @Now, - @Now, - @Now + SYSUTCDATETIME(), + SYSUTCDATETIME(), + SYSUTCDATETIME() ); END; END; + SELECT @LockedNextMessageId = NextMessageId + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + SET @Now = SYSUTCDATETIME(); + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) SET NextMessageId = NextMessageId + 1, @@ -238,7 +247,7 @@ BEGIN SET XACT_ABORT ON; DECLARE @StartedTransaction BIT = 0; - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Now DATETIME2(7); DECLARE @NextMessageId BIGINT; DECLARE @Checkpoint BIGINT; DECLARE @OwnerEpoch BIGINT; @@ -321,9 +330,9 @@ BEGIN 1, NULL, 0, - @Now, - @Now, - @Now + SYSUTCDATETIME(), + SYSUTCDATETIME(), + SYSUTCDATETIME() ); END; END; @@ -336,6 +345,8 @@ BEGIN AND ProviderId = @ProviderId AND QueueId = @QueueId; + SET @Now = SYSUTCDATETIME(); + SELECT @EarliestMessageId = MIN(MessageId), @TailMessageId = MAX(MessageId) @@ -385,6 +396,7 @@ BEGIN @ProviderId AS ProviderId, @QueueId AS QueueId, @OwnerEpoch AS OwnerEpoch, + @NextMessageId AS NextMessageId, @Checkpoint AS [Checkpoint], @EarliestMessageId AS EarliestMessageId, @TailMessageId AS TailMessageId; @@ -420,7 +432,8 @@ BEGIN [Checkpoint] BIGINT NULL, Updated BIT NOT NULL ); - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Now DATETIME2(7); + DECLARE @LockedCheckpoint BIGINT; DECLARE @StartedTransaction BIT = 0; BEGIN TRY @@ -430,6 +443,14 @@ BEGIN SET @StartedTransaction = 1; END; + SELECT @LockedCheckpoint = [Checkpoint] + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + SET @Now = SYSUTCDATETIME(); + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) SET [Checkpoint] = @Checkpoint, @@ -645,6 +666,6 @@ VALUES ('AcquireStreamPartitionKey', 'EXECUTE AcquireStreamPartition @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @StartFromNow = @StartFromNow'), ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId OFFSET 0 ROWS FETCH NEXT @MaxCount ROWS ONLY'), ('AdvanceStreamCheckpointKey', 'EXECUTE AdvanceStreamCheckpoint @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @OwnerEpoch = @OwnerEpoch, @Checkpoint = @Checkpoint'), - ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.[Checkpoint] AS [Checkpoint], MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.[Checkpoint]'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.[Checkpoint] AS [Checkpoint], MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.[Checkpoint]'), ('CleanupStreamMessagesKey', 'EXECUTE CleanupStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @RetentionPeriodSeconds = @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds = @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds = @CleanupIntervalSeconds, @CleanupBatchSize = @CleanupBatchSize'); GO diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index 7b0527c0dbd..9408540925f 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -519,6 +519,7 @@ internal Task AcquireStreamPartitionAsync( (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], (string)record[nameof(AdoNetStreamPartitionState.QueueId)], (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + (long)record[nameof(AdoNetStreamPartitionState.NextMessageId)], GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), @@ -630,6 +631,7 @@ internal Task> ReadStreamMessagesAsync( (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], (string)record[nameof(AdoNetStreamPartitionState.QueueId)], (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + (long)record[nameof(AdoNetStreamPartitionState.NextMessageId)], GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index c950fd71ced..78f78044969 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -207,6 +207,31 @@ public async Task Read_UsesDistinctRoundedRetentionParameters() Assert.Equal(9, parameters[nameof(DbStoredQueries.Columns.CleanupBatchSize)]); } + [Theory] + [InlineData(null, 1L, null, false)] + [InlineData(0L, 1L, null, false)] + [InlineData(3L, 4L, null, false)] + [InlineData(0L, 4L, null, true)] + [InlineData(0L, 4L, 2L, true)] + public void HasRetentionGap_UsesNextMessageIdWhenRetainedHistoryIsEmpty( + long? checkpoint, + long nextMessageId, + long? earliestMessageId, + bool expected) + { + var state = new AdoNetStreamPartitionState( + "service", + "provider", + "queue", + OwnerEpoch: 1, + NextMessageId: nextMessageId, + Checkpoint: checkpoint, + EarliestMessageId: earliestMessageId, + TailMessageId: null); + + Assert.Equal(expected, AdoNetRecoverableStream.HasRetentionGap(state)); + } + private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage) { var queryValues = typeof(DbStoredQueries) @@ -236,6 +261,7 @@ public void CompleteAcquisition(long ownerEpoch) "provider", "queue", ownerEpoch, + NextMessageId: 1, Checkpoint: 0, EarliestMessageId: null, TailMessageId: null)); @@ -258,6 +284,7 @@ public async Task> ReadAsync( [nameof(AdoNetStreamPartitionState.ProviderId)] = state.ProviderId, [nameof(AdoNetStreamPartitionState.QueueId)] = state.QueueId, [nameof(AdoNetStreamPartitionState.OwnerEpoch)] = state.OwnerEpoch, + [nameof(AdoNetStreamPartitionState.NextMessageId)] = state.NextMessageId, [nameof(AdoNetStreamPartitionState.Checkpoint)] = state.Checkpoint, [nameof(AdoNetStreamPartitionState.EarliestMessageId)] = state.EarliestMessageId, [nameof(AdoNetStreamPartitionState.TailMessageId)] = state.TailMessageId, diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index 22d6613d6e4..0d70fd3105c 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -650,6 +650,7 @@ public async Task GetStreamPartitionBounds_ReflectsCurrentState() Assert.NotNull(bounds); Assert.Equal(state.OwnerEpoch, bounds!.OwnerEpoch); + Assert.Equal(second.MessageId + 1, bounds.NextMessageId); Assert.Equal(second.MessageId, bounds.Checkpoint); Assert.Equal(1, bounds.EarliestMessageId); Assert.Equal(second.MessageId, bounds.TailMessageId); @@ -766,6 +767,46 @@ public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics( Assert.Null(result.Checkpoint); } + [Fact] + public async Task FullyPurgedUncheckpointedHistory_RemainsDetectableAsRetentionGap() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var initial = await _queries.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + startFromNow: false); + Assert.Equal(0, initial.Checkpoint); + Assert.Equal(1, initial.NextMessageId); + + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + var cleanup = await _queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: 120, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + Assert.Equal(3, cleanup.HardDeletedCount); + + var bounds = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + Assert.NotNull(bounds); + Assert.Equal(0, bounds.Checkpoint); + Assert.Equal(4, bounds.NextMessageId); + Assert.Null(bounds.EarliestMessageId); + Assert.Null(bounds.TailMessageId); + Assert.True(AdoNetRecoverableStream.HasRetentionGap(bounds)); + } + [Fact] public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() { diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 5a51bd00bab..9833cc282b4 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -37,6 +37,7 @@ public void SchemaDefinesVersionedStreamPartitions(string provider) Assert.Contains("'AdvanceStreamCheckpointKey'", script, StringComparison.Ordinal); Assert.Contains("'GetStreamPartitionBoundsKey'", script, StringComparison.Ordinal); Assert.Contains("'CleanupStreamMessagesKey'", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId", GetStoredQuery(script, "GetStreamPartitionBoundsKey"), StringComparison.Ordinal); Assert.DoesNotContain("CREATE TABLE OrleansStreamDeadLetter", script, StringComparison.Ordinal); Assert.DoesNotContain("CREATE TABLE OrleansStreamControl", script, StringComparison.Ordinal); @@ -86,6 +87,56 @@ public void MySqlScriptDoesNotProduceWhitespaceOnlyBatches() Assert.DoesNotContain(batches, string.IsNullOrWhiteSpace); } + [Theory] + [InlineData("SQLServer", "SELECT @LockedNextMessageId", "SET @Now = SYSUTCDATETIME()", "INSERT INTO OrleansStreamMessage")] + [InlineData("PostgreSQL", "RETURNING P.NextMessageId - 1 INTO _MessageId", "_Now := clock_timestamp()", "INSERT INTO OrleansStreamMessage")] + [InlineData("MySQL", "FOR UPDATE;", "SET _Now = UTC_TIMESTAMP(6)", "INSERT INTO OrleansStreamMessage")] + public void AppendSamplesMessageTimestampAfterPartitionLock( + string provider, + string lockMarker, + string timestampMarker, + string messageInsertMarker) + { + var script = ReadScript(provider); + + AssertOrder(script, lockMarker, timestampMarker, messageInsertMarker); + } + + [Theory] + [InlineData("SQLServer", "SELECT @LockedCheckpoint", "SET @Now = SYSUTCDATETIME()", "SET CheckpointedOn")] + [InlineData("PostgreSQL", "FOR UPDATE;", "_Now := clock_timestamp()", "SET CheckpointedOn")] + [InlineData("MySQL", "FOR UPDATE;", "SET _Now = UTC_TIMESTAMP(6)", "SET CheckpointedOn")] + public void CheckpointSamplesEligibilityTimestampAfterPartitionLock( + string provider, + string lockMarker, + string timestampMarker, + string eligibilityMarker) + { + var script = ReadScript(provider); + var checkpointProcedure = script[script.IndexOf("AdvanceStreamCheckpoint", StringComparison.Ordinal)..]; + + AssertOrder(checkpointProcedure, lockMarker, timestampMarker, eligibilityMarker); + } + private static string ReadScript(string provider) => File.ReadAllText(Path.Combine(AppContext.BaseDirectory, $"{provider}-Streaming.sql")); + + private static string GetStoredQuery(string script, string queryKey) + { + var start = script.IndexOf($"('{queryKey}'", StringComparison.Ordinal); + Assert.True(start >= 0); + var end = script.IndexOfAny(['\r', '\n'], start); + return script[start..end]; + } + + private static void AssertOrder(string text, params string[] markers) + { + var previous = -1; + foreach (var marker in markers) + { + var current = text.IndexOf(marker, previous + 1, StringComparison.Ordinal); + Assert.True(current > previous, $"Expected '{marker}' after index {previous}."); + previous = current; + } + } } diff --git a/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs index 1a96e4081f4..a79178f63d9 100644 --- a/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs @@ -58,4 +58,18 @@ public void RedisStreamSequenceToken_CompareTo_UsesRedisSequenceNumberBeforeEven Assert.True(sameEntryNextEvent.CompareTo(nextEntrySameMillisecond) < 0); Assert.True(nextEntrySameMillisecond.CompareTo(nextMillisecond) < 0); } + + [Fact] + public void CreateSequenceTokenForEventPreservesRedisPosition() + { + var batchToken = new RedisStreamSequenceToken("100-7", 100, 7, 0); + + var eventToken = batchToken.CreateSequenceTokenForEvent(2); + + Assert.Equal("100-7", eventToken.EntryId); + Assert.Equal(100, eventToken.SequenceNumber); + Assert.Equal(7, eventToken.RedisSequenceNumber); + Assert.Equal(2, eventToken.EventIndex); + Assert.True(batchToken.CompareTo(eventToken) < 0); + } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index f38499ca770..4fc1da9bcf8 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -458,6 +458,80 @@ await client.Received(1).GetRecordsAsync( TestContext.Current.CancellationToken); } + [Fact] + public async Task ConcurrentInitializationRetriesWhenOwningCallerCancels() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + var firstAttemptStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var createCount = 0; + checkpointerFactory.Create("shard-1", Arg.Any()) + .Returns(async call => + { + var token = call.Arg(); + if (Interlocked.Increment(ref createCount) == 1) + { + firstAttemptStarted.SetResult(token); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + } + + return checkpointer; + }); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-1", + Records = [], + })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + using var ownerCancellation = new CancellationTokenSource(); + + var owner = receiver.GetQueueMessagesAsync(10, ownerCancellation.Token); + var unaffected = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + var firstAttemptToken = await firstAttemptStarted.Task; + ownerCancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => owner); + Assert.Empty(await unaffected); + + Assert.True(firstAttemptToken.IsCancellationRequested); + Assert.Equal(2, createCount); + await client.Received(1).GetShardIteratorAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task ConcurrentInitializationPropagatesProviderCancellationWhenOwnerIsActive() + { + var client = Substitute.For(); + var checkpointerFactory = Substitute.For(); + var createCount = 0; + checkpointerFactory.Create("shard-1", Arg.Any()) + .Returns(_ => + { + Interlocked.Increment(ref createCount); + return Task.FromException>( + new OperationCanceledException("provider canceled independently")); + }); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal(1, createCount); + await client.DidNotReceive().GetShardIteratorAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + [Fact] public async Task ReceiverReadForwardsCancellationToken() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 263a15348de..73bf679883c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4310,6 +4310,16 @@ public void EventSequenceTokens_RejectDerivedTokensSymmetrically() Assert.True(v1DerivedPeer.Equals(v1Derived)); Assert.Equal(0, v1Derived.CompareTo(v1DerivedPeer)); Assert.Equal(v1Derived.GetHashCode(), v1DerivedPeer.GetHashCode()); + + var v2EventToken = Assert.IsType( + ((DerivedEventSequenceToken)derivedToken).CreateSequenceTokenForEvent(3)); + Assert.Equal(10, v2EventToken.SequenceNumber); + Assert.Equal(3, v2EventToken.EventIndex); + + var v1EventToken = Assert.IsType( + ((DerivedEventSequenceTokenV1)v1Derived).CreateSequenceTokenForEvent(3)); + Assert.Equal(10, v1EventToken.SequenceNumber); + Assert.Equal(3, v1EventToken.EventIndex); } [TestSuite("BVT")] From 0246f0d64753cfe3433b5c005aa5e06008f37314 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 12:43:30 -0700 Subject: [PATCH 31/90] test(streaming): require persisted ADO.NET messages --- .../Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 99a560f1c59..94218ce2575 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -152,6 +152,7 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() TestContext.Current.CancellationToken)) .OrderBy(static message => message.MessageId) .ToList(); + Assert.Equal(3, stored.Count); for (var i = 0; i < stored.Count; i++) { var item = stored[i]; From 8cbf0718978edb1c9a1bb06d9c88e2b934616b07 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 02:38:36 -0700 Subject: [PATCH 32/90] fix(streaming): retry receiver creation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../QueueAdapterReceiverRegistry.cs | 16 +++++++++++-- .../RecoverableStreamReceiverTests.cs | 24 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs index 2eea3a72380..d3e098d450e 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -37,12 +37,24 @@ public IReadOnlyDictionary Receivers /// Gets or creates the coordinator for a queue. /// public TReceiver GetOrCreate(QueueId queueId) - => _receivers.GetOrAdd( + { + var receiver = _receivers.GetOrAdd( queueId, static (id, factory) => new( () => factory(id), LazyThreadSafetyMode.ExecutionAndPublication), - _factory).Value; + _factory); + try + { + return receiver.Value; + } + catch + { + ((ICollection>>)_receivers) + .Remove(new(queueId, receiver)); + throw; + } + } /// /// Removes a receiver if it is still the registered instance for the queue. diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 09a1dc1d529..4999e3fa07c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -719,6 +719,30 @@ public async Task Registry_ConcurrentRequestsReturnSameWinningInstance() Assert.Same(results[0], Assert.Single(registry.Receivers).Value); } + [Fact] + public void Registry_FactoryFailureAllowsLaterCreationRetry() + { + var queue = QueueId.GetQueueId("queue", 0, 0); + var attempts = 0; + var registry = new QueueAdapterReceiverRegistry(_ => + { + if (Interlocked.Increment(ref attempts) == 1) + { + throw new InvalidOperationException("creation failed"); + } + + return new TestCombinedReceiver(); + }); + + Assert.Throws(() => registry.GetOrCreate(queue)); + Assert.Empty(registry.Receivers); + + var receiver = registry.GetOrCreate(queue); + + Assert.Equal(2, attempts); + Assert.Same(receiver, Assert.Single(registry.Receivers).Value); + } + [Fact] public void Registry_RemoveAllowsFreshReceiverForReassignedQueue() { From 0ef3c8a4e6e565284d8a9dfcdf3a9aec2d0bbea8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 03:32:45 -0700 Subject: [PATCH 33/90] test(streaming): flow test cancellation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../PersistentStreamPullingAgentTests.cs | 32 ++++++++++++++----- .../RecoverableStreamReceiverTests.cs | 4 +-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 73bf679883c..cf0a8a3a962 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4372,7 +4372,9 @@ public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparis Assert.Empty(queueCache.DeliveryProgressTokens); timeProvider.Advance(TimeSpan.FromTicks(1)); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(providerEarliest, Assert.Single(queueCache.DeliveryProgressTokens)); Assert.Equal(timeProvider.GetUtcNow().UtcDateTime, Assert.Single(queueCache.DeliveryProgressUtcTimes)); @@ -4570,7 +4572,9 @@ public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() queueCache.ClearDeliveryProgress(); timeProvider.Advance(options.DeliveryProgressUpdateInterval); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(deliveredToken, Assert.Single(queueCache.DeliveryProgressTokens)); await testAccessor.Shutdown(); @@ -4631,7 +4635,9 @@ .. Enumerable.Range(2, 8) await testAccessor.RunConsumerCursor(consumerData); Assert.Equal(10, consumerData.LastSafePartitionToken?.SequenceNumber); timeProvider.Advance(options.DeliveryProgressUpdateInterval); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(10, Assert.Single(queueCache.DeliveryProgressTokens)?.SequenceNumber); await testAccessor.Shutdown(); } @@ -4720,7 +4726,9 @@ public async Task DeliveryProgress_QuietSubscriptionAdvancesAcrossBusyStreamBeyo Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 2)); await testAccessor.GetPubSubCache(); timeProvider.Advance(options.DeliveryProgressUpdateInterval); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); Assert.NotNull(checkpoint); @@ -4825,7 +4833,9 @@ public async Task DeliveryProgress_IncludesFilteredBatches() Assert.Equal(filteredToken, consumerData.LastProcessedToken); timeProvider.Advance(options.DeliveryProgressUpdateInterval); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(filteredToken, Assert.Single(queueCache.DeliveryProgressTokens)); await testAccessor.Shutdown(); @@ -4886,14 +4896,20 @@ public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() queueCache.ClearDeliveryProgress(); Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); - await slowConsumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); - await fastConsumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await slowConsumer.Delivered.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + await fastConsumer.Delivered.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); await testAccessor.GetPubSubCache(); Assert.Equal(previousToken, slowConsumerData.LastProcessedToken); Assert.Equal(currentToken, fastConsumerData.LastProcessedToken); timeProvider.Advance(options.DeliveryProgressUpdateInterval); - await queueCache.DeliveryProgressUpdated.WaitAsync(TimeSpan.FromSeconds(5)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); Assert.Equal(previousToken, Assert.Single(queueCache.DeliveryProgressTokens)); slowConsumer.ReleaseDelivery(); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 4999e3fa07c..643e2b16023 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -604,8 +604,8 @@ public async Task Receiver_IndependentInitializationCancellationIsNotRetried() var receiver = CreateReceiver(source, checkpointer); await Assert.ThrowsAnyAsync( - () => receiver.GetQueueMessagesAsync(10, CancellationToken.None) - .WaitAsync(TimeSpan.FromSeconds(5))); + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); Assert.Equal(1, checkpointer.LoadCount); Assert.Equal(0, source.InitializeCount); From b09a062d177bfebc6694d6575e181e26d05aa02e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 04:40:13 -0700 Subject: [PATCH 34/90] test(streaming): align provider tests with main Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../RelationalOrleansQueriesUnitTests.cs | 363 +----------------- .../RelationalStorageExtensionsTests.cs | 15 +- .../Streaming/AdoNetRecoverableStreamTests.cs | 6 +- .../Streaming/AdoNetStreamPartitionTests.cs | 147 +++++-- .../EventHubCheckpointerTests.cs | 15 +- .../KinesisRuntimeTests.cs | 16 +- 6 files changed, 175 insertions(+), 387 deletions(-) diff --git a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs index fbf6c2fae5d..7a99f6dc105 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs @@ -53,12 +53,13 @@ public sealed class RelationalOrleansQueriesUnitTests private static readonly string[] StreamingQueryKeys = [ - "QueueStreamMessageKey", - "GetStreamMessagesKey", - "ConfirmStreamMessagesKey", - "FailStreamMessageKey", - "EvictStreamMessagesKey", - "EvictStreamDeadLettersKey", + "StreamSchemaVersionKey", + "AppendStreamMessageKey", + "AcquireStreamPartitionKey", + "ReadStreamMessagesKey", + "AdvanceStreamCheckpointKey", + "GetStreamPartitionBoundsKey", + "CleanupStreamMessagesKey", ]; private static readonly string[] DirectoryQueryKeys = @@ -324,186 +325,6 @@ public async Task MembershipMutation_ReturnsResultAndCapturesAllParameters() storage.VerifyComplete(); } - [Fact] - public async Task GetStreamMessages_ReturnsRowsSortedByQueueAndSequence() - { - var now = new DateTime(2026, 8, 27, 16, 0, 0, DateTimeKind.Utc); - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("GetStreamMessagesKey"), - CreateTable( - StreamMessageColumns, - ["service-e", "provider-a", "queue-a", 20L, 2, now, now.AddHours(1), now.AddMinutes(-2), now.AddMinutes(-1), new byte[] { 2, 0 }], - ["service-e", "provider-a", "queue-a", 3L, 1, now, now.AddHours(2), now.AddMinutes(-4), now.AddMinutes(-3), new byte[] { 0, 3 }])); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.GetStreamMessagesAsync("service-e", "provider-a", "queue-a", 25, 4, 30, 60, 90, 10); - - Assert.Equal([3L, 20L], result.Select(message => message.MessageId)); - Assert.Equal([0, 3], result[0].Payload); - Assert.Equal(2, result[1].Dequeued); - AssertParameters( - AssertOperationCall(storage, Sql("GetStreamMessagesKey")), - ("ServiceId", "service-e"), - ("ProviderId", "provider-a"), - ("QueueId", "queue-a"), - ("MaxCount", 25), - ("MaxAttempts", 4), - ("VisibilityTimeout", 30), - ("RemovalTimeout", 60), - ("EvictionInterval", 90), - ("EvictionBatchSize", 10)); - storage.VerifyComplete(); - } - - [Fact] - public async Task GetStreamMessages_ReturnsEmptyWhenNoRowsExist() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead(Sql("GetStreamMessagesKey")); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.GetStreamMessagesAsync("service-f", "provider-b", "queue-b", 5, 2, 10, 20, 30, 4); - - Assert.Empty(result); - AssertParameters( - AssertOperationCall(storage, Sql("GetStreamMessagesKey")), - ("ServiceId", "service-f"), - ("ProviderId", "provider-b"), - ("QueueId", "queue-b"), - ("MaxCount", 5), - ("MaxAttempts", 2), - ("VisibilityTimeout", 10), - ("RemovalTimeout", 20), - ("EvictionInterval", 30), - ("EvictionBatchSize", 4)); - storage.VerifyComplete(); - } - - [Fact] - public async Task ConfirmStreamMessages_EmptySet_DoesNotExecuteMutation() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.ConfirmStreamMessagesAsync("service-g", "provider-c", "queue-c", []); - - Assert.Empty(result); - Assert.Single(storage.Calls); - storage.VerifyComplete(); - } - - [Fact] - public async Task ConfirmStreamMessages_CapturesReceiptParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable( - StreamConfirmationColumns, - ["service-g", "provider-c", "queue-c", 11L], - ["service-g", "provider-c", "queue-c", 12L])); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.ConfirmStreamMessagesAsync( - "service-g", - "provider-c", - "queue-c", - [new(11, 3), new(12, 4)]); - - Assert.Equal([11L, 12L], result.Select(ack => ack.MessageId)); - Assert.All(result, ack => Assert.Equal("queue-c", ack.QueueId)); - AssertParameters( - AssertOperationCall(storage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-g"), - ("ProviderId", "provider-c"), - ("QueueId", "queue-c"), - ("Items", "11:3|12:4")); - storage.VerifyComplete(); - - var singletonStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable(StreamConfirmationColumns, ["service-g", "provider-c", "queue-c", 13L])); - var singletonQueries = await StreamingQueries.CreateInstance(singletonStorage, TestContext.Current.CancellationToken); - - var singletonResult = await singletonQueries.ConfirmStreamMessagesAsync( - "service-g", - "provider-c", - "queue-c", - [new(13, 5)]); - - Assert.Equal([13L], singletonResult.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(singletonStorage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-g"), - ("ProviderId", "provider-c"), - ("QueueId", "queue-c"), - ("Items", "13:5")); - singletonStorage.VerifyComplete(); - } - - [Fact] - public async Task ReleaseStreamMessages_EmptySet_DoesNotExecuteMutation() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.ReleaseStreamMessagesAsync("service-h", "provider-d", "queue-d", []); - - Assert.Empty(result); - Assert.Single(storage.Calls); - storage.VerifyComplete(); - } - - [Fact] - public async Task ReleaseStreamMessages_CapturesReceiptParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable( - StreamConfirmationColumns, - ["service-h", "provider-d", "queue-d", 21L], - ["service-h", "provider-d", "queue-d", 22L])); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.ReleaseStreamMessagesAsync( - "service-h", - "provider-d", - "queue-d", - [new(21, 5), new(22, 6)]); - - Assert.Equal([21L, 22L], result.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(storage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-h"), - ("ProviderId", "provider-d"), - ("QueueId", "queue-d"), - ("Items", "21:-5|22:-6")); - storage.VerifyComplete(); - - var singletonStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable(StreamConfirmationColumns, ["service-h", "provider-d", "queue-d", 23L])); - var singletonQueries = await StreamingQueries.CreateInstance(singletonStorage, TestContext.Current.CancellationToken); - - var singletonResult = await singletonQueries.ReleaseStreamMessagesAsync( - "service-h", - "provider-d", - "queue-d", - [new(23, 7)]); - - Assert.Equal([23L], singletonResult.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(singletonStorage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-h"), - ("ProviderId", "provider-d"), - ("QueueId", "queue-d"), - ("Items", "23:-7")); - singletonStorage.VerifyComplete(); - } [Fact] public async Task GrainDirectoryLookup_ReturnsSingleEntry() @@ -593,12 +414,18 @@ public async Task Operation_PropagatesStorageErrorAcrossProductionCopies() membershipStorage.VerifyComplete(); var streamingStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectReadException(Sql("GetStreamMessagesKey"), expected); + .ExpectReadException(Sql("ReadStreamMessagesKey"), expected); var streamingQueries = await StreamingQueries.CreateInstance(streamingStorage, TestContext.Current.CancellationToken); var streamingError = await Assert.ThrowsAsync( - () => streamingQueries.GetStreamMessagesAsync("service-error", "provider-error", "queue-error", 5, 2, 10, 20, 30, 4)); + () => streamingQueries.ReadStreamMessagesAsync( + "service-error", + "provider-error", + "queue-error", + afterMessageId: 0, + maxCount: 5, + TestContext.Current.CancellationToken)); Assert.Same(expected, streamingError); - AssertOperationCall(streamingStorage, Sql("GetStreamMessagesKey")); + AssertOperationCall(streamingStorage, Sql("ReadStreamMessagesKey")); streamingStorage.VerifyComplete(); var directoryStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), DirectoryQueryKeys) @@ -695,40 +522,6 @@ public async Task UnregisterGrainActivationsAsync_ReturnsCountAndCapturesParamet storage.VerifyComplete(); } - [Fact] - public async Task QueueStreamMessageAsync_ReturnsAcknowledgementAndCapturesParameters() - { - byte[] payload = [0, 1, 127, 255]; - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("QueueStreamMessageKey"), - CreateTable( - StreamConfirmationColumns, - ["service-queue", "provider-queue", "queue-queue", 9_876_543_210L])); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var result = await queries.QueueStreamMessageAsync( - "service-queue", - "provider-queue", - "queue-queue", - payload, - 3_600); - - Assert.Equal( - new AdoNetStreamMessageAck("service-queue", "provider-queue", "queue-queue", 9_876_543_210L), - result); - var call = AssertOperationCall(storage, Sql("QueueStreamMessageKey")); - AssertParameters( - call, - ("ServiceId", "service-queue"), - ("ProviderId", "provider-queue"), - ("QueueId", "queue-queue"), - ("Payload", payload), - ("ExpiryTimeout", 3_600)); - Assert.Equal(payload, Assert.IsType(Parameter(call, "Payload").Value)); - storage.VerifyComplete(); - } - [Fact] public async Task UpsertReminderRowAsync_ReturnsVersionAndCapturesParameters() { @@ -1000,57 +793,6 @@ public async Task InsertMembershipVersionRowAsync_ReturnsResultAndCapturesDeploy storage.VerifyComplete(); } - [Fact] - public async Task FailStreamMessageAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("FailStreamMessageKey"), affectedRows: 1); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - await queries.FailStreamMessageAsync( - "service-fail", - "provider-fail", - "queue-fail", - 4_294_967_297L, - 8, - 7_200); - - AssertParameters( - AssertOperationCall(storage, Sql("FailStreamMessageKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-fail"), - ("ProviderId", "provider-fail"), - ("QueueId", "queue-fail"), - ("MessageId", 4_294_967_297L), - ("MaxAttempts", 8), - ("RemovalTimeout", 7_200)); - storage.VerifyComplete(); - } - - [Theory] - [InlineData(0, "serviceId")] - [InlineData(1, "providerId")] - [InlineData(2, "queueId")] - public async Task QueueStreamMessageAsync_WithNullRequiredArgument_ThrowsArgumentNullException( - int nullIndex, - string expectedParameterName) - { - var arguments = new string?[] { "service-null", "provider-null", "queue-null" }; - arguments[nullIndex] = null; - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - var exception = await Assert.ThrowsAsync( - () => queries.QueueStreamMessageAsync( - arguments[0]!, - arguments[1]!, - arguments[2]!, - [1, 2, 3], - 60)); - - Assert.Equal(expectedParameterName, exception.ParamName); - AssertOnlyQueryLoadCall(storage); - storage.VerifyComplete(); - } [Theory] [InlineData(0, "clusterId")] @@ -1148,53 +890,6 @@ public async Task UnregisterGrainActivationsAsync_WithNullRequiredArgument_Throw storage.VerifyComplete(); } - [Fact] - public async Task EvictStreamMessagesAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("EvictStreamMessagesKey"), affectedRows: 37); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - await queries.EvictStreamMessagesAsync( - "service-evict", - "provider-evict", - "queue-evict", - 37, - 8, - 7_200); - - AssertParameters( - AssertOperationCall(storage, Sql("EvictStreamMessagesKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-evict"), - ("ProviderId", "provider-evict"), - ("QueueId", "queue-evict"), - ("MaxCount", 37), - ("MaxAttempts", 8), - ("RemovalTimeout", 7_200)); - storage.VerifyComplete(); - } - - [Fact] - public async Task EvictStreamDeadLettersAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("EvictStreamDeadLettersKey"), affectedRows: 43); - var queries = await StreamingQueries.CreateInstance(storage, TestContext.Current.CancellationToken); - - await queries.EvictStreamDeadLettersAsync( - "service-dead-letter", - "provider-dead-letter", - "queue-dead-letter", - 43); - - AssertParameters( - AssertOperationCall(storage, Sql("EvictStreamDeadLettersKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-dead-letter"), - ("ProviderId", "provider-dead-letter"), - ("QueueId", "queue-dead-letter"), - ("MaxCount", 43)); - storage.VerifyComplete(); - } [Theory] [InlineData(0)] @@ -1628,28 +1323,6 @@ private static DataTable CreateMembershipCleanupRow(int version, SiloStatus stat ], [start, 11_222, 14, "10.4.5.6", "silo-cleanup", "host-cleanup", (int)status, 30_004, heartbeat, suspectTimes, (long)version]); - private static readonly (string Name, Type Type)[] StreamMessageColumns = - [ - ("ServiceId", typeof(string)), - ("ProviderId", typeof(string)), - ("QueueId", typeof(string)), - ("MessageId", typeof(long)), - ("Dequeued", typeof(int)), - ("VisibleOn", typeof(DateTime)), - ("ExpiresOn", typeof(DateTime)), - ("CreatedOn", typeof(DateTime)), - ("ModifiedOn", typeof(DateTime)), - ("Payload", typeof(byte[])), - ]; - - private static readonly (string Name, Type Type)[] StreamConfirmationColumns = - [ - ("ServiceId", typeof(string)), - ("ProviderId", typeof(string)), - ("QueueId", typeof(string)), - ("MessageId", typeof(long)), - ]; - private static readonly (string Name, Type Type)[] DirectoryEntryColumns = [ ("ClusterId", typeof(string)), @@ -1664,7 +1337,9 @@ private static ScriptedRelationalStorage ExpectQueryLoad( IEnumerable keys) => storage.ExpectRead( GetQueriesSql, - CreateQueryTable(keys.Select(key => (key, Sql(key))).ToArray())); + CreateQueryTable(keys + .Select(key => (key, key == "StreamSchemaVersionKey" ? "2" : Sql(key))) + .ToArray())); private static DataTable CreateQueryTable(params (string Key, string Query)[] queries) => CreateTable( diff --git a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs index c9c70b11baf..cd28314f071 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs @@ -25,7 +25,8 @@ record => record.GetInt32(0) * 2, "@tenant", "north", size: 32, - dbType: DbType.AnsiString)); + dbType: DbType.AnsiString), + TestContext.Current.CancellationToken); Assert.Equal([84], results); var call = Assert.Single(storage.Calls); @@ -70,7 +71,11 @@ public async Task ReadAsync_FlowsAcrossResultSetsInOrder() CreateTable(("Value", typeof(int), 10), ("Value", typeof(int), 20)), CreateTable(("Value", typeof(int), 30))); - var results = await storage.ReadAsync(Sql, record => record.GetInt32(0), parameterProvider: null); + var results = await storage.ReadAsync( + Sql, + record => record.GetInt32(0), + parameterProvider: null, + TestContext.Current.CancellationToken); Assert.Equal([10, 20, 30], results); Assert.Single(storage.Calls); @@ -133,7 +138,11 @@ public async Task ReadAsync_RejectsNullSelector() var storage = new ScriptedRelationalStorage(); var exception = await Assert.ThrowsAsync( - () => storage.ReadAsync(Sql, (Func)null!, parameterProvider: null)); + () => storage.ReadAsync( + Sql, + (Func)null!, + parameterProvider: null, + TestContext.Current.CancellationToken)); Assert.Equal("selector", exception.ParamName); Assert.Empty(storage.Calls); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 78f78044969..3ca139422d6 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -109,7 +109,7 @@ public async Task Load_PropagatesCancellationAndDiscardsLateAcquisition() await Assert.ThrowsAnyAsync(() => loadTask); Assert.True(source.AcquisitionCompletion.IsCompletedSuccessfully); await Assert.ThrowsAsync( - () => source.Update("1", "1", CancellationToken.None).AsTask()); + () => source.Update("1", "1", TestContext.Current.CancellationToken).AsTask()); } [Theory] @@ -137,7 +137,7 @@ public async Task ShutdownNotification_WaitsForAnyAcquisitionCompletionBeforeAll acquisition.SetException(new InvalidOperationException("late failure")); break; case AcquisitionCompletionKind.Canceled: - acquisition.SetCanceled(); + acquisition.SetCanceled(TestContext.Current.CancellationToken); break; } @@ -198,7 +198,7 @@ public async Task Read_UsesDistinctRoundedRetentionParameters() CreateQueries(storage), NullLogger.Instance); - Assert.Empty(await source.Read(10, CancellationToken.None)); + Assert.Empty(await source.Read(10, TestContext.Current.CancellationToken)); var parameters = storage.Parameters[nameof(DbStoredQueries.CleanupStreamMessagesKey)]; Assert.Equal(3, parameters[nameof(DbStoredQueries.Columns.RetentionPeriodSeconds)]); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index 0d70fd3105c..0589d25d496 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -1,15 +1,15 @@ using System.Data; using System.Data.Common; using System.Globalization; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; using MySql.Data.MySqlClient; using Npgsql; using Orleans.Configuration; using Orleans.Runtime; -using Orleans.Streams; using Orleans.Streaming.AdoNet; using Orleans.Streaming.AdoNet.Storage; +using Orleans.Streams; using UnitTests.General; using static System.String; @@ -91,7 +91,7 @@ public abstract class AdoNetStreamPartitionTests(string invariant) : IAsyncLifet private const string TestDatabaseName = "OrleansStreamTest"; private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; + private CancelableRelationalQueries _queries = null!; public async ValueTask InitializeAsync() { @@ -99,7 +99,7 @@ public async ValueTask InitializeAsync() Assert.SkipWhen(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = new(await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString)); } public ValueTask DisposeAsync() => ValueTask.CompletedTask; @@ -553,7 +553,7 @@ public async Task RecoverableStreamUpdate_ReturnsPersistedStateWhenExpectedVersi providerId, queueId, new AdoNetStreamOptions(), - _queries, + _queries.Inner, NullLogger.Instance); var loaded = await store.Load(CancellationToken.None); Assert.Equal("0", loaded.Checkpoint); @@ -584,14 +584,14 @@ public async Task RecoverableStreamUpdate_ThrowsWhenPartitionOwnershipIsLost() providerId, queueId, new AdoNetStreamOptions(), - _queries, + _queries.Inner, NullLogger.Instance); - var loaded = await store.Load(CancellationToken.None); + var loaded = await store.Load(TestContext.Current.CancellationToken); var newOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); Assert.NotEqual(long.Parse(loaded.Version, CultureInfo.InvariantCulture), newOwner.OwnerEpoch); var exception = await Assert.ThrowsAsync( - () => store.Update("1", loaded.Version, CancellationToken.None).AsTask()); + () => store.Update("1", loaded.Version, TestContext.Current.CancellationToken).AsTask()); Assert.Contains("ownership was lost", exception.Message); } @@ -906,7 +906,8 @@ public async Task CreateInstance_MissingStreamPartitionQueryKeys_FailsExplicitly { await _storage.ExecuteAsync( "DELETE FROM OrleansQuery WHERE QueryKey IN ('AppendStreamMessageKey', 'StreamSchemaVersionKey')", - command => { }); + command => { }, + cancellationToken: TestContext.Current.CancellationToken); var exception = await Assert.ThrowsAsync(() => RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); @@ -920,7 +921,8 @@ public async Task CreateInstance_MixedLegacyAndStreamPartitionQueryKeys_FailsExp { await _storage.ExecuteAsync( "INSERT INTO OrleansQuery (QueryKey, QueryText) VALUES ('QueueStreamMessageKey', 'legacy')", - command => { }); + command => { }, + cancellationToken: TestContext.Current.CancellationToken); var exception = await Assert.ThrowsAsync(() => RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); @@ -948,8 +950,8 @@ protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoe var payload = RandomPayload(); await using var firstConnection = new SqlConnection(_storage.ConnectionString); - await firstConnection.OpenAsync(); - await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(); + await firstConnection.OpenAsync(TestContext.Current.CancellationToken); + await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(TestContext.Current.CancellationToken); await using var firstCommand = CreateAppendCommand(firstConnection, firstTransaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); var firstMessageId = await ReadMessageId(firstCommand); Assert.Equal(1, firstMessageId); @@ -958,7 +960,7 @@ protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoe // transaction still holds the partition-row lock. await using (var secondConnection = new SqlConnection(_storage.ConnectionString)) { - await secondConnection.OpenAsync(); + await secondConnection.OpenAsync(TestContext.Current.CancellationToken); await using var secondCommand = secondConnection.CreateCommand(); secondCommand.CommandType = CommandType.Text; secondCommand.CommandText = "SET LOCK_TIMEOUT 0; EXECUTE AppendStreamMessage @ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload;"; @@ -969,14 +971,15 @@ protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoe secondCommand.Parameters.AddWithValue("StreamNamespaceLength", nsLength); secondCommand.Parameters.AddWithValue("Payload", payload); - var exception = await Assert.ThrowsAsync(() => secondCommand.ExecuteReaderAsync()); + var exception = await Assert.ThrowsAsync( + () => secondCommand.ExecuteReaderAsync(TestContext.Current.CancellationToken)); Assert.Equal(51000, exception.Number); Assert.Contains("initialization lock", exception.Message, StringComparison.Ordinal); } // Rolling back must not burn the allocated identifier: it is reused by the next append, // and no message row for it was left behind. - await firstTransaction.RollbackAsync(); + await firstTransaction.RollbackAsync(TestContext.Current.CancellationToken); var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); Assert.Equal(1, afterRollback.MessageId); @@ -1002,8 +1005,8 @@ protected async Task VerifySqlServerReadExcludesUncommittedInFlightAppend() var payload = RandomPayload(); await using var connection = new SqlConnection(_storage.ConnectionString); - await connection.OpenAsync(); - await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); var inFlightMessageId = await ReadMessageId(command); Assert.Equal(committed.MessageId + 1, inFlightMessageId); @@ -1011,7 +1014,7 @@ protected async Task VerifySqlServerReadExcludesUncommittedInFlightAppend() var duringAppend = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); Assert.Equal([committed.MessageId], duringAppend.Select(m => m.MessageId)); - await transaction.CommitAsync(); + await transaction.CommitAsync(TestContext.Current.CancellationToken); var afterCommit = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); Assert.Equal([committed.MessageId, inFlightMessageId], afterCommit.Select(m => m.MessageId)); @@ -1027,16 +1030,16 @@ protected async Task VerifySqlServerPartitionsAreIndependent() var payload = RandomPayload(); await using var connection = new SqlConnection(_storage.ConnectionString); - await connection.OpenAsync(); - await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, firstQueueId, streamIdBytes, nsLength, payload); Assert.Equal(1, await ReadMessageId(command)); var independent = await _queries.AppendStreamMessageAsync(serviceId, providerId, secondQueueId, streamIdBytes, nsLength, payload) - .WaitAsync(TimeSpan.FromSeconds(5)); + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, independent.MessageId); - await transaction.RollbackAsync(); + await transaction.RollbackAsync(TestContext.Current.CancellationToken); } protected async Task VerifyMySqlRollbackRestoresAllocation() @@ -1048,8 +1051,8 @@ protected async Task VerifyMySqlRollbackRestoresAllocation() var payload = RandomPayload(); await using var connection = new MySqlConnection(_storage.ConnectionString); - await connection.OpenAsync(); - await using var transaction = await connection.BeginTransactionAsync(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); await using (var command = connection.CreateCommand()) { command.Transaction = transaction; @@ -1063,7 +1066,7 @@ protected async Task VerifyMySqlRollbackRestoresAllocation() Assert.Equal(1, await ReadMessageId(command)); } - await transaction.RollbackAsync(); + await transaction.RollbackAsync(TestContext.Current.CancellationToken); var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); Assert.Equal(1, afterRollback.MessageId); @@ -1078,8 +1081,8 @@ protected async Task VerifyPostgreSqlRollbackRestoresAllocation() var payload = RandomPayload(); await using var connection = new NpgsqlConnection(_storage.ConnectionString); - await connection.OpenAsync(); - await using var transaction = await connection.BeginTransactionAsync(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); await using (var command = connection.CreateCommand()) { command.Transaction = transaction; @@ -1093,7 +1096,7 @@ protected async Task VerifyPostgreSqlRollbackRestoresAllocation() Assert.Equal(1, await ReadMessageId(command)); } - await transaction.RollbackAsync(); + await transaction.RollbackAsync(TestContext.Current.CancellationToken); var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); Assert.Equal(1, afterRollback.MessageId); @@ -1124,10 +1127,94 @@ private static SqlCommand CreateAppendCommand( private static async Task ReadMessageId(DbCommand command) { - await using var reader = await command.ExecuteReaderAsync(); - Assert.True(await reader.ReadAsync()); + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + Assert.True(await reader.ReadAsync(TestContext.Current.CancellationToken)); return reader.GetInt64(reader.GetOrdinal(nameof(AdoNetStreamMessage.MessageId))); } + private sealed class CancelableRelationalQueries(RelationalOrleansQueries inner) + { + public RelationalOrleansQueries Inner { get; } = inner; + + public Task AppendStreamMessageAsync( + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) + => Inner.AppendStreamMessageAsync( + serviceId, + providerId, + queueId, + streamIdBytes, + streamNamespaceLength, + payload); + + public Task AcquireStreamPartitionAsync( + string serviceId, + string providerId, + string queueId, + bool startFromNow) + => Inner.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + startFromNow, + TestContext.Current.CancellationToken); + + public Task> ReadStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + long afterMessageId, + int maxCount) + => Inner.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + afterMessageId, + maxCount, + TestContext.Current.CancellationToken); + + public Task AdvanceStreamCheckpointAsync( + string serviceId, + string providerId, + string queueId, + long ownerEpoch, + long checkpoint) + => Inner.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + ownerEpoch, + checkpoint, + TestContext.Current.CancellationToken); + + public Task GetStreamPartitionBoundsAsync( + string serviceId, + string providerId, + string queueId) + => Inner.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + + public Task CleanupStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + int retentionPeriodSeconds, + int? maximumRetentionPeriodSeconds, + int cleanupIntervalSeconds, + int cleanupBatchSize) + => Inner.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds, + maximumRetentionPeriodSeconds, + cleanupIntervalSeconds, + cleanupBatchSize, + TestContext.Current.CancellationToken); + } + #endregion SQL Server } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index dd9d01b5c20..fa88e00a54d 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -566,7 +566,10 @@ public async Task Update_WhenOffsetDoesNotAdvance_DoesNotChangeCheckpoint(string var checkpointer = CreateUninitializedCheckpointer(); SetPersistedOffset(checkpointer, "20"); - checkpointer.Update(candidate, new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + candidate, + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); await checkpointer.FlushAsync(TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); @@ -581,7 +584,10 @@ public void Update_WhenOffsetAdvances_TracksLatestCheckpoint() var checkpointer = CreateUninitializedCheckpointer(); SetPersistedOffset(checkpointer, "20"); - checkpointer.Update("21", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + "21", + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); Assert.Equal("21", GetLatestOffset(checkpointer)); @@ -594,7 +600,10 @@ public void Update_WithNoComparer_TracksOpaqueCheckpoint() var checkpointer = CreateUninitializedCheckpointer(useNumericComparer: false); ThrottleSaves(checkpointer); - checkpointer.Update("opaque-checkpoint", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + "opaque-checkpoint", + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); Assert.Equal("opaque-checkpoint", GetLatestOffset(checkpointer)); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index 4fc1da9bcf8..fd2e787df3e 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -268,8 +268,12 @@ public async Task ReceiverAssignsMonotonicallyIncreasingLocalOrdinalsAcrossReads var receiver = CreateReceiver(client, checkpointerFactory, timeProvider); await receiver.Initialize(TimeSpan.FromSeconds(5)); - var firstBatch = (await receiver.GetQueueMessagesAsync(10)).Cast().ToArray(); - var secondBatch = (await receiver.GetQueueMessagesAsync(10)).Cast().ToArray(); + var firstBatch = (await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)).Cast().ToArray(); + var secondBatch = (await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)).Cast().ToArray(); Assert.Equal([0L, 1L], firstBatch.Select(container => container.Token.SequenceNumber)); Assert.Equal([2L], secondBatch.Select(container => container.Token.SequenceNumber)); @@ -297,7 +301,9 @@ public async Task MessagesDeliveredCommitsNumericallyHighestShardSequence() new Amazon.Kinesis.Model.Record { SequenceNumber = "42", Data = new MemoryStream() }, sequenceId: 1); - await receiver.MessagesDeliveredAsync([hugeButReadFirst, smallButReadSecond]); + await receiver.MessagesDeliveredAsync( + [hugeButReadFirst, smallButReadSecond], + TestContext.Current.CancellationToken); checkpointer.Received(1).Update( "170141183460469231731687303715884105727", @@ -319,7 +325,9 @@ public async Task MessagesDeliveredWithEmptyListDoesNotUpdateCheckpointOrThrow() var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); await receiver.Initialize(TimeSpan.FromSeconds(5)); - await receiver.MessagesDeliveredAsync(Array.Empty()); + await receiver.MessagesDeliveredAsync( + Array.Empty(), + TestContext.Current.CancellationToken); checkpointer.DidNotReceive().Update(Arg.Any(), Arg.Any(), Arg.Any()); } From 490c2483de7b8b7993b05cf5d33fd38df89ee78e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 04:17:32 -0700 Subject: [PATCH 35/90] fix(streaming): close recoverable stream review gaps Reuse pooled cache blocks transactionally, preserve shutdown safety, handle unlimited flow control, and align ADO.NET retention, checkpoint, locking, rewind, benchmark, Unicode, API, test, and documentation contracts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec261a38-b57f-43b7-b524-02dfb8d049b6 --- .../docs/streaming/stream-providers.md | 4 +- .../AdoNetBatchContainer.cs | 2 +- .../AdoNetQueueAdapter.cs | 4 +- .../MySQL-Streaming.sql | 39 ++++++------- .../PostgreSQL-Streaming.sql | 5 +- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 2 +- .../SQLServer-Streaming.sql | 3 +- .../Common/PooledCache/FixedSizeBuffer.cs | 9 +++ .../RecoverableStreamQueueCache.cs | 11 +++- .../PersistentStreamPullingAgent.cs | 20 ++++++- .../StreamPartitionReadCheckpointBenchmark.cs | 27 +++++---- .../AdoNetQueueAdapterFactoryTests.cs | 2 +- .../Streaming/AdoNetQueueAdapterTests.cs | 2 +- .../Streaming/AdoNetStreamSchemaTests.cs | 48 +++++++++++++++- .../StreamingTests/ClientStreamTestRunner.cs | 3 +- .../PersistentStreamPullingAgentTests.cs | 43 ++++++++++++++ .../RecoverableStreamReceiverTests.cs | 57 ++++++++++++++++++- 17 files changed, 233 insertions(+), 48 deletions(-) diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 1e0308d4ee6..234324eb6b8 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -18,7 +18,7 @@ A stream provider connects the Orleans streaming API to a transport and defines | Azure Event Hubs | [`Microsoft.Orleans.Streaming.EventHubs`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.EventHubs) | Stable | Yes, within Event Hubs retention | Yes | Event Hubs namespace, hub, consumer group, and checkpoint storage | | Amazon Kinesis | [`Microsoft.Orleans.Streaming.Kinesis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Kinesis) | Stable | Yes, within Kinesis retention | Yes | Kinesis data stream, AWS credentials, region, and durable checkpoint storage | | Amazon SQS | [`Microsoft.Orleans.Streaming.SQS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.SQS) | Stable | Yes, within SQS retention | No | AWS account, queue permissions, region/endpoint configuration | -| ADO.NET partitioned stream | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational stream partitions | Yes, while records remain retained | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | +| ADO.NET partitioned stream | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational stream partitions | No; resumes after its durable safe checkpoint | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | | NATS JetStream | [`Microsoft.Orleans.Streaming.NATS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.NATS) | **Alpha** | Configurable; file storage is the default | No | NATS server with JetStream and sufficient storage; subject/stream administration | | Redis Streams | [`Microsoft.Orleans.Streaming.Redis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Redis) | **Alpha** | Configurable through Redis persistence and stream retention | Yes, while entries remain | Redis deployment, persistence/HA policy, and retention sizing | @@ -70,7 +70,7 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S ## ADO.NET partitioned stream provider (alpha) -Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline is rewindable while records remain retained and resumes strictly after the earliest safe scan/delivery checkpoint. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. retains checkpointed records for one day by default. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Retention and cleanup intervals round fractional seconds upward. Cleanup is bounded by . can fault one failing subscription while preserving the multicast partition record. +Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline resumes strictly after the earliest safe scan/delivery checkpoint, redelivering uncheckpointed records after recovery. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. retains checkpointed records for one day by default before cleanup. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Retention and cleanup intervals round fractional seconds upward. Cleanup is bounded by and deletes a contiguous eligible prefix. can fault one failing subscription while preserving the multicast partition record. The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs index 46d9aaacb33..bb2854ac3c1 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs @@ -5,7 +5,7 @@ namespace Orleans.Streaming.AdoNet; /// /// /// This class uses binary serialization to prioritize database storage performance and payload size. -/// Its identifies a record position in the rewindable partition history. +/// Its identifies a record position in the partition history. /// [GenerateSerializer] [Alias("Orleans.Streaming.AdoNet.AdoNetBatchContainer")] diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index f9decb20b7a..3bdedc728fa 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -44,9 +44,9 @@ public AdoNetQueueAdapter( public string Name { get; } /// - /// The ADO.NET partitioned stream provider supports replay from cached and durable checkpoints. + /// The ADO.NET partitioned stream provider resumes from its durable partition checkpoint. /// - public bool IsRewindable => true; + public bool IsRewindable => false; /// /// The ADO.NET provider works both ways. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index a45b76994ac..8ac7c2f7226 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -54,9 +54,9 @@ DROP PROCEDURE ValidateOrleansStreamingSchemaUpgrade; CREATE TABLE OrleansStreamPartition ( - ServiceId VARCHAR(150) NOT NULL, - ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, + ServiceId NVARCHAR(150) NOT NULL, + ProviderId NVARCHAR(150) NOT NULL, + QueueId NVARCHAR(150) NOT NULL, NextMessageId BIGINT NOT NULL, Checkpoint BIGINT NULL, OwnerEpoch BIGINT NOT NULL, @@ -69,9 +69,9 @@ CREATE TABLE OrleansStreamPartition CREATE TABLE OrleansStreamMessage ( - ServiceId VARCHAR(150) NOT NULL, - ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, + ServiceId NVARCHAR(150) NOT NULL, + ProviderId NVARCHAR(150) NOT NULL, + QueueId NVARCHAR(150) NOT NULL, MessageId BIGINT NOT NULL, StreamIdBytes LONGBLOB NOT NULL, StreamNamespaceLength INT NOT NULL, @@ -86,9 +86,9 @@ DELIMITER $$ CREATE PROCEDURE AppendStreamMessage ( - IN _ServiceId VARCHAR(150), - IN _ProviderId VARCHAR(150), - IN _QueueId VARCHAR(150), + IN _ServiceId NVARCHAR(150), + IN _ProviderId NVARCHAR(150), + IN _QueueId NVARCHAR(150), IN _StreamIdBytes LONGBLOB, IN _StreamNamespaceLength INT, IN _Payload LONGBLOB, @@ -190,9 +190,9 @@ END$$ CREATE PROCEDURE AcquireStreamPartition ( - IN _ServiceId VARCHAR(150), - IN _ProviderId VARCHAR(150), - IN _QueueId VARCHAR(150), + IN _ServiceId NVARCHAR(150), + IN _ProviderId NVARCHAR(150), + IN _QueueId NVARCHAR(150), IN _StartFromNow BOOLEAN, IN _ManageTransaction BOOLEAN ) @@ -307,9 +307,9 @@ END$$ CREATE PROCEDURE AdvanceStreamCheckpoint ( - IN _ServiceId VARCHAR(150), - IN _ProviderId VARCHAR(150), - IN _QueueId VARCHAR(150), + IN _ServiceId NVARCHAR(150), + IN _ProviderId NVARCHAR(150), + IN _QueueId NVARCHAR(150), IN _OwnerEpoch BIGINT, IN _Checkpoint BIGINT, IN _ManageTransaction BOOLEAN @@ -361,6 +361,7 @@ BEGIN WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId + AND (_CurrentCheckpoint IS NULL OR MessageId > _CurrentCheckpoint) AND MessageId <= _Checkpoint AND CheckpointedOn IS NULL; END IF; @@ -390,9 +391,9 @@ END$$ CREATE PROCEDURE CleanupStreamMessages ( - IN _ServiceId VARCHAR(150), - IN _ProviderId VARCHAR(150), - IN _QueueId VARCHAR(150), + IN _ServiceId NVARCHAR(150), + IN _ProviderId NVARCHAR(150), + IN _QueueId NVARCHAR(150), IN _RetentionPeriodSeconds INT, IN _MaximumRetentionPeriodSeconds INT, IN _CleanupIntervalSeconds INT, @@ -491,7 +492,7 @@ BEGIN ) ORDER BY MessageId LIMIT _CleanupBatchSize - FOR UPDATE SKIP LOCKED; + FOR UPDATE; SELECT COUNT(*), diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 454e8de03a9..8b1e66e8d50 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -297,6 +297,7 @@ DECLARE _Now TIMESTAMP(6) WITHOUT TIME ZONE; _CurrentOwnerEpoch BIGINT; _CurrentCheckpoint BIGINT; + _PreviousCheckpoint BIGINT; _Updated BOOLEAN := FALSE; BEGIN SELECT P.OwnerEpoch, P.Checkpoint @@ -306,6 +307,7 @@ BEGIN AND P.ProviderId = _ProviderId AND P.QueueId = _QueueId FOR UPDATE; + _PreviousCheckpoint := _CurrentCheckpoint; _Now := clock_timestamp() AT TIME ZONE 'UTC'; @@ -329,6 +331,7 @@ BEGIN WHERE M.ServiceId = _ServiceId AND M.ProviderId = _ProviderId AND M.QueueId = _QueueId + AND (_PreviousCheckpoint IS NULL OR M.MessageId > _PreviousCheckpoint) AND M.MessageId <= _Checkpoint AND M.CheckpointedOn IS NULL; ELSE @@ -449,7 +452,7 @@ BEGIN ) ) ORDER BY M.MessageId - FOR UPDATE SKIP LOCKED + FOR UPDATE LIMIT _CleanupBatchSize ), Deleted AS diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index 3d77f2d7bbb..312110269bd 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -58,7 +58,7 @@ The provider stores each queue as an immutable, ordered stream partition. Its st - `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. Fractional seconds round upward. - `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. Fractional cleanup intervals round upward. -The partitioned stream provider is rewindable while requested records remain retained. Inclusive subscription start positions remain pending until the corresponding record is delivered or intentionally filtered. The queue checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. It resumes strictly after that durable, ownership-fenced checkpoint and can redeliver records after a crash without skipping uncheckpointed data. Partition state retains the next message identifier, so recovery detects a hard-retention gap even when the purge leaves no records. +The partitioned stream provider resumes strictly after the durable, ownership-fenced queue checkpoint and can redeliver records after a crash without skipping uncheckpointed data. The checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. Subscription starts attach to the live partition position. Partition state retains the next message identifier, so recovery detects a hard-retention gap even when the purge leaves no records. Partition acquisition is cancellation-aware. A receiver whose acquisition command is still completing retains its queue reservation, so a late database result settles before a replacement receiver acquires a newer ownership epoch. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index 6a2135b932e..73ed997bb5f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -477,6 +477,7 @@ BEGIN WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + AND (@LockedCheckpoint IS NULL OR MessageId > @LockedCheckpoint) AND MessageId <= @Checkpoint AND CheckpointedOn IS NULL; END; @@ -585,7 +586,7 @@ BEGIN ( SELECT TOP (@CleanupBatchSize) MessageId - FROM OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) + FROM OrleansStreamMessage WITH (UPDLOCK, READCOMMITTEDLOCK, ROWLOCK) WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId diff --git a/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs b/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs index 0bed302edd7..4cc4db692fb 100644 --- a/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs +++ b/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs @@ -22,6 +22,8 @@ public class FixedSizeBuffer : PooledResource /// public object Id => buffer; + internal int Position => count; + /// /// Manages access to a fixed size byte buffer. /// @@ -56,6 +58,13 @@ public bool TryGetSegment(int size, out ArraySegment value) return true; } + internal void ResetTo(int position) + { + ArgumentOutOfRangeException.ThrowIfNegative(position); + ArgumentOutOfRangeException.ThrowIfGreaterThan(position, count); + count = position; + } + /// public override void OnResetState() { diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index ddb5136f565..11aa7fe2492 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -103,7 +103,9 @@ public IReadOnlyList Add( var positions = new List(messages.Count); var cachedMessages = new List(messages.Count); var allocatedBuffers = new List(); - FixedSizeBuffer? batchBuffer = null; + var initialBuffer = _currentBuffer; + var initialBufferPosition = initialBuffer?.Position ?? 0; + var batchBuffer = initialBuffer; try { foreach (var message in messages) @@ -115,6 +117,7 @@ public IReadOnlyList Add( } catch { + initialBuffer?.ResetTo(initialBufferPosition); foreach (var buffer in allocatedBuffers) { buffer.Dispose(); @@ -164,7 +167,11 @@ public int GetMaxAddCount() if (_flowController is not null) { - result = Math.Min(result, _flowController.GetMaxAddCount()); + var flowControlLimit = _flowController.GetMaxAddCount(); + if (flowControlLimit >= 0) + { + result = Math.Min(result, flowControlLimit); + } } return result; diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 10db304fb0f..310a0ceb17a 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -49,6 +49,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task _activePumpTask = Task.CompletedTask; private AdmissionGate _workAdmission = new(); private Task? _shutdownTask; + private int _hasUnprocessedRead; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -187,6 +188,7 @@ public async Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; + Volatile.Write(ref _hasUnprocessedRead, 0); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -296,7 +298,7 @@ private async Task ShutdownCore(CancellationToken cancellationToken) await drainTask; // All accepted work has finished progress bookkeeping and released its batch/registration pins. - if (!hasPendingRegistrations) + if (!hasPendingRegistrations && Volatile.Read(ref _hasUnprocessedRead) == 0) { NotifyDeliveryProgress(); } @@ -1021,6 +1023,16 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList? multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount, cancellationToken); + if (IsShutdown || cancellationToken.IsCancellationRequested) + { + if (multiBatch is { Count: > 0 }) + { + Volatile.Write(ref _hasUnprocessedRead, 1); + } + return false; + } + } + // Receivers built against older Orleans versions can still return null. if (multiBatch is null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback. @@ -1048,6 +1060,12 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? foreach (var group in availableMessages.GroupBy(container => container.StreamId)) { + if (IsShutdown || cancellationToken.IsCancellationRequested) + { + Volatile.Write(ref _hasUnprocessedRead, 1); + return false; + } + var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); StreamSequenceToken startToken = group.First().SequenceToken; if (pubSubCache.TryGetValue(streamId, out var streamData)) diff --git a/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs index dfef147dbf8..23af845f961 100644 --- a/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs @@ -108,18 +108,23 @@ await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => [BenchmarkCategory("Checkpoint")] public async Task AdvanceEpochFencedCheckpoints() { - await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => + await Parallel.ForAsync(0, Math.Min(PartitionCount, OperationsPerInvoke), async (partition, cancellationToken) => { - var partition = i % PartitionCount; - var checkpoint = (i / PartitionCount) + 1L; - var result = await _queries.AdvanceStreamCheckpointAsync( - "ServiceId-0", - "ProviderId-0", - _partitionIds[partition], - _ownerEpochs[partition], - checkpoint); - if (result is not null) + for (var i = partition; i < OperationsPerInvoke; i += PartitionCount) { + var checkpoint = (i / PartitionCount) + 1L; + var result = await _queries.AdvanceStreamCheckpointAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + _ownerEpochs[partition], + checkpoint, + cancellationToken); + if (result is not { Updated: true }) + { + throw new InvalidOperationException($"Checkpoint {checkpoint} did not advance partition {partition}."); + } + _consumer.Consume(result); } }); @@ -226,7 +231,7 @@ WITH Ranked AS FROM OrleansStreamMessage ) UPDATE Message - SET CreatedOn = DATEADD(DAY, -2, SYSUTCDATETIME()) + SET CheckpointedOn = DATEADD(DAY, -2, SYSUTCDATETIME()) FROM OrleansStreamMessage AS Message INNER JOIN Ranked ON Ranked.ServiceId = Message.ServiceId diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index 10009d375b1..958e506541f 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -104,7 +104,7 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() Assert.NotNull(adapter); Assert.IsType(adapter); Assert.Equal(name, adapter.Name); - Assert.True(adapter.IsRewindable); + Assert.False(adapter.IsRewindable); Assert.Equal(StreamProviderDirection.ReadWrite, adapter.Direction); var queueId = factory.GetStreamQueueMapper().GetAllQueues().First(); Assert.Same( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 94218ce2575..0859640ff2a 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -102,7 +102,7 @@ public void AdoNetQueueAdapter_Constructs() // assert Assert.Equal(name, adapter.Name); - Assert.True(adapter.IsRewindable); + Assert.False(adapter.IsRewindable); Assert.Equal(StreamProviderDirection.ReadWrite, adapter.Direction); } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index 9833cc282b4..e2e5cd61348 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -52,7 +52,6 @@ public void SqlServerAppendUsesUpdateOutputWhileHoldingPartitionLock() Assert.Contains("UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK)", script, StringComparison.Ordinal); Assert.Contains("OUTPUT Inserted.NextMessageId - 1", script, StringComparison.Ordinal); Assert.Contains("@LockOwner = 'Transaction'", script, StringComparison.Ordinal); - Assert.Contains("READPAST, READCOMMITTEDLOCK", script, StringComparison.Ordinal); } [Fact] @@ -77,6 +76,18 @@ public void MySqlAppendUsesSelectForUpdateThenUpdate() Assert.DoesNotContain("@@session.in_transaction", script, StringComparison.Ordinal); } + [Fact] + public void MySqlSchemaUsesUnicodePartitionIdentifiers() + { + var script = ReadScript("MySQL"); + + Assert.Contains("ServiceId NVARCHAR(150) NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("IN _ServiceId NVARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("ServiceId VARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("ProviderId VARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("QueueId VARCHAR(150)", script, StringComparison.Ordinal); + } + [Fact] public void MySqlScriptDoesNotProduceWhitespaceOnlyBatches() { @@ -118,6 +129,32 @@ public void CheckpointSamplesEligibilityTimestampAfterPartitionLock( AssertOrder(checkpointProcedure, lockMarker, timestampMarker, eligibilityMarker); } + [Theory] + [InlineData("SQLServer", "AND (@LockedCheckpoint IS NULL OR MessageId > @LockedCheckpoint)")] + [InlineData("PostgreSQL", "AND (_PreviousCheckpoint IS NULL OR M.MessageId > _PreviousCheckpoint)")] + [InlineData("MySQL", "AND (_CurrentCheckpoint IS NULL OR MessageId > _CurrentCheckpoint)")] + public void CheckpointEligibilityUpdateStartsAfterPreviousCheckpoint(string provider, string lowerBound) + { + var checkpointProcedure = GetProcedure(ReadScript(provider), "AdvanceStreamCheckpoint", "CleanupStreamMessages"); + + Assert.Contains(lowerBound, checkpointProcedure, StringComparison.Ordinal); + } + + [Theory] + [InlineData("SQLServer", "READPAST")] + [InlineData("PostgreSQL", "SKIP LOCKED")] + [InlineData("MySQL", "SKIP LOCKED")] + public void CleanupWaitsForLeadingEligibleRows(string provider, string skipLockedMarker) + { + var script = ReadScript(provider); + var cleanupStart = script.IndexOf("CleanupStreamMessages", StringComparison.Ordinal); + Assert.True(cleanupStart >= 0); + var cleanupProcedure = script[cleanupStart..]; + + Assert.Contains("ORDER BY MessageId", cleanupProcedure, StringComparison.Ordinal); + Assert.DoesNotContain(skipLockedMarker, cleanupProcedure, StringComparison.Ordinal); + } + private static string ReadScript(string provider) => File.ReadAllText(Path.Combine(AppContext.BaseDirectory, $"{provider}-Streaming.sql")); @@ -129,6 +166,15 @@ private static string GetStoredQuery(string script, string queryKey) return script[start..end]; } + private static string GetProcedure(string script, string startMarker, string endMarker) + { + var start = script.IndexOf(startMarker, StringComparison.Ordinal); + Assert.True(start >= 0); + var end = script.IndexOf(endMarker, start + startMarker.Length, StringComparison.Ordinal); + Assert.True(end > start); + return script[start..end]; + } + private static void AssertOrder(string text, params string[] markers) { var previous = -1; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs index 1dca6abd49a..1995ad4ff02 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs @@ -52,7 +52,6 @@ public async Task StreamConsumerOnDroppedClientTest( bool waitForRetryTimeouts = false, CancellationToken cancellationToken = default) { - var hasDeliveryFailureCounter = getDeliveryFailureCount is not null; getDeliveryFailureCount ??= DefaultDeliveryFailureCount; Guid streamGuid = Guid.NewGuid(); @@ -76,7 +75,7 @@ public async Task StreamConsumerOnDroppedClientTest( await ProduceEventsToClient(streamProviderName, streamGuid, streamNamespace, 10, eventCount, cancellationToken); // Wait for the dropped client's subscription to be removed after delivery fails. - if (waitForRetryTimeouts && hasDeliveryFailureCounter) + if (waitForRetryTimeouts) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_timeout); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index cf0a8a3a962..7890d72202b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -3809,6 +3809,49 @@ public async Task Shutdown_DrainsRegistrationPinBeforeReceiverCleanup(string out } } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_DoesNotCheckpointBatchReturnedAfterShutdownStarts() + { + var queueReadStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var queueReadReleased = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(async _ => + { + queueReadStarted.TrySetResult(); + return await queueReadReleased.Task; + }); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = Substitute.For(); + queueCache.GetMaxAddCount().Returns(1); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(queueId).Returns(queueCache); + var agent = CreateAgent(pubSub: null, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + + var pumpTask = testAccessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + await queueReadStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var shutdownTask = testAccessor.Shutdown(); + queueReadReleased.SetResult( + [ + new GeneratedBatchContainer(streamId, 1, new EventSequenceTokenV2(1)), + ]); + + await shutdownTask; + await pumpTask; + + queueCache.DidNotReceive().AddToCache(Arg.Any>()); + queueCache.DidNotReceive().UpdateDeliveryProgress( + Arg.Any(), + Arg.Any()); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 643e2b16023..46339c1d69c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -351,6 +351,21 @@ [new TestQueueMessage(streamId, 11, "payload")], Assert.Equal(0, cache.ItemCount); } + [Fact] + public void Cache_UnlimitedFlowControlUsesConfiguredReadLimit() + { + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance, + flowController: new FixedFlowController(QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG)); + + Assert.Equal(100, cache.GetMaxAddCount()); + Assert.False(cache.IsUnderPressure()); + } + [Fact] public void Cache_DisposeDrainsMessagesReturnsBuffersAndAllocatesFreshBufferIfReused() { @@ -407,6 +422,38 @@ public void Cache_FailedPackingReturnsUncommittedPooledBuffers() Assert.Equal(2, bufferPool.FreeCount); } + [Fact] + public void Cache_ReusesCurrentBufferAcrossAddsAndRollsBackFailedPacking() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var first = new TestQueueMessage(streamId, 1, "a"); + var failed = new TestQueueMessage(streamId, 2, new string('b', 10)); + var final = new TestQueueMessage(streamId, 3, new string('c', 10)); + var bufferPool = new TrackingBufferPool( + GetPackedSize(first) + GetPackedSize(failed) + GetPackedSize(final) - 1); + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + _ = cache.Add([first], DateTime.UnixEpoch); + adapter.ThrowAfterPackingSequenceNumber = 2; + + Assert.Throws(() => cache.Add([failed], DateTime.UnixEpoch)); + + adapter.ThrowAfterPackingSequenceNumber = null; + _ = cache.Add([final], DateTime.UnixEpoch); + + Assert.Equal(2, cache.ItemCount); + Assert.Equal(1, bufferPool.AllocateCount); + + static int GetPackedSize(TestQueueMessage message) => + SegmentBuilder.CalculateAppendSize(message.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + + SegmentBuilder.CalculateAppendSize(message.Payload); + } + [Fact] public void Cache_AddsRawRecordsInOrderAndDecodesLazily() { @@ -780,6 +827,7 @@ private sealed class TestDataAdapter : IRecoverableStreamDataAdapter maxAddCount; } - private sealed class TrackingBufferPool : IObjectPool + private sealed class TrackingBufferPool(int bufferSize = 4 * 1024) : IObjectPool { public int AllocateCount { get; private set; } @@ -1068,7 +1121,7 @@ private sealed class TrackingBufferPool : IObjectPool public FixedSizeBuffer Allocate() { AllocateCount++; - return new(4 * 1024) { Pool = this }; + return new(bufferSize) { Pool = this }; } public void Free(FixedSizeBuffer resource) From 7a79b256770be349b6e30c86ffe1dcc725332595 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 06:02:01 -0700 Subject: [PATCH 36/90] fix(streaming): use utf8mb4 partition identifiers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../MySQL-Streaming.sql | 36 +++++++++---------- .../Streaming/AdoNetStreamSchemaTests.cs | 10 +++--- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 8ac7c2f7226..3adeea780a4 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -54,9 +54,9 @@ DROP PROCEDURE ValidateOrleansStreamingSchemaUpgrade; CREATE TABLE OrleansStreamPartition ( - ServiceId NVARCHAR(150) NOT NULL, - ProviderId NVARCHAR(150) NOT NULL, - QueueId NVARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, NextMessageId BIGINT NOT NULL, Checkpoint BIGINT NULL, OwnerEpoch BIGINT NOT NULL, @@ -69,9 +69,9 @@ CREATE TABLE OrleansStreamPartition CREATE TABLE OrleansStreamMessage ( - ServiceId NVARCHAR(150) NOT NULL, - ProviderId NVARCHAR(150) NOT NULL, - QueueId NVARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MessageId BIGINT NOT NULL, StreamIdBytes LONGBLOB NOT NULL, StreamNamespaceLength INT NOT NULL, @@ -86,9 +86,9 @@ DELIMITER $$ CREATE PROCEDURE AppendStreamMessage ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, IN _StreamIdBytes LONGBLOB, IN _StreamNamespaceLength INT, IN _Payload LONGBLOB, @@ -190,9 +190,9 @@ END$$ CREATE PROCEDURE AcquireStreamPartition ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, IN _StartFromNow BOOLEAN, IN _ManageTransaction BOOLEAN ) @@ -307,9 +307,9 @@ END$$ CREATE PROCEDURE AdvanceStreamCheckpoint ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, IN _OwnerEpoch BIGINT, IN _Checkpoint BIGINT, IN _ManageTransaction BOOLEAN @@ -391,9 +391,9 @@ END$$ CREATE PROCEDURE CleanupStreamMessages ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, IN _RetentionPeriodSeconds INT, IN _MaximumRetentionPeriodSeconds INT, IN _CleanupIntervalSeconds INT, diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs index e2e5cd61348..50624918403 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -81,11 +81,11 @@ public void MySqlSchemaUsesUnicodePartitionIdentifiers() { var script = ReadScript("MySQL"); - Assert.Contains("ServiceId NVARCHAR(150) NOT NULL", script, StringComparison.Ordinal); - Assert.Contains("IN _ServiceId NVARCHAR(150)", script, StringComparison.Ordinal); - Assert.DoesNotContain("ServiceId VARCHAR(150)", script, StringComparison.Ordinal); - Assert.DoesNotContain("ProviderId VARCHAR(150)", script, StringComparison.Ordinal); - Assert.DoesNotContain("QueueId VARCHAR(150)", script, StringComparison.Ordinal); + Assert.Contains("ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", script, StringComparison.Ordinal); + Assert.DoesNotContain("ServiceId NVARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("ProviderId NVARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("QueueId NVARCHAR(150)", script, StringComparison.Ordinal); } [Fact] From bfe386a138e973e926afe21b6a3608c7b6d63433 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 29 Aug 2026 03:07:58 -0700 Subject: [PATCH 37/90] fix(streaming): dispose Kinesis lifecycle cancellation Dispose the pooled receiver lifecycle CancellationTokenSource after shutdown work completes and verify it is canceled and disposed on both target frameworks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec261a38-b57f-43b7-b524-02dfb8d049b6 --- .../Streams/KinesisPooledAdapterReceiver.cs | 3 +++ .../Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index 01fd05d57e5..b826b376d67 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -28,6 +28,8 @@ internal sealed class KinesisPooledAdapterReceiver : IQueueAdapterReceiver, IQue private int _initialized; private int _shutdown; + internal CancellationToken LifecycleCancellationToken => _lifecycleCancellation.Token; + public KinesisPooledAdapterReceiver( IAmazonKinesis client, string streamName, @@ -171,6 +173,7 @@ public async Task Shutdown(TimeSpan timeout) } finally { + _lifecycleCancellation.Dispose(); _onShutdown?.Invoke(this); } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index fd2e787df3e..c01807adf32 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -114,7 +114,7 @@ public void QueueAdapterFactoryIsRewindable() } [Fact] - public async Task PooledReceiver_ReadsAfterSuccessfulInitialization() + public async Task PooledReceiver_ReadsAndDisposesLifecycleCancellationOnShutdown() { var client = Substitute.For(); client.GetShardIteratorAsync( @@ -152,6 +152,7 @@ public async Task PooledReceiver_ReadsAfterSuccessfulInitialization() topologyMonitor, TimeSpan.Zero, timeProvider); + var lifecycleCancellationToken = receiver.LifecycleCancellationToken; await receiver.Initialize(TimeSpan.FromSeconds(5)); Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); @@ -160,6 +161,9 @@ await client.Received(1).GetRecordsAsync( Arg.Any(), Arg.Any()); await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.True(lifecycleCancellationToken.IsCancellationRequested); + Assert.Throws(() => _ = receiver.LifecycleCancellationToken); } [Fact] From fc3c7d2474e1ff97d7b137a0c0df6600215efd56 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 30 Aug 2026 02:40:29 -0700 Subject: [PATCH 38/90] fix(streaming): emit unregister after clearing pubsub Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../PubSub/PubSubRendezvousGrain.cs | 5 +++++ .../PubSubRendezvousGrainTests.cs | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs index 0655a251c8b..0c8cbb4901c 100644 --- a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs +++ b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs @@ -605,6 +605,11 @@ public async Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId st if (await TryClearState(cancellationToken)) { + if (numRemoved != 0) + { + StreamingEvents.EmitSubscriptionUnregistered(streamId.ProviderName, streamId.StreamId, subscriptionId.Guid, GrainContext.Address.SiloAddress); + } + // If state was cleared expedite Deactivation DeactivateOnIdle(); } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs index af2b8c020ce..4aa3ef4c17f 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs @@ -703,6 +703,27 @@ public async Task FutureReplacementVersionUsesFreshValidation() Assert.Equal(SiloStatus.Active, statuses[replacementSilo]); } + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task UnregisterLastConsumerEmitsDiagnosticAfterClearingState() + { + var streamId = new QualifiedStreamId("ProviderName", StreamId.Create("StreamNamespace", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var pubSubGrain = this.fixture.GrainFactory.GetGrain(streamId.ToString()); + using var observer = StreamingDiagnosticObserver.Create(); + + await pubSubGrain.RegisterConsumer(subscriptionId, streamId, default, null!); + await pubSubGrain.UnregisterConsumer(subscriptionId, streamId); + + var unregistered = await observer.WaitForSubscriptionUnregisteredAsync( + streamId.StreamId, + subscriptionId.Guid, + streamId.ProviderName, + TestContext.Current.CancellationToken); + Assert.Equal(streamId.StreamId, unregistered.StreamId); + Assert.Equal(subscriptionId.Guid, unregistered.SubscriptionId); + Assert.Equal(0, await pubSubGrain.ConsumerCount(streamId)); + } + /// /// This test fails because the producer must be grain reference which is not implied by the IStreamProducerExtension in the producer management calls. /// TODO: Fix rendezvous implementation. From 53b7c54a254022a6b9905a01981b13ed53e91593 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 3 Sep 2026 08:26:15 -0700 Subject: [PATCH 39/90] fix(streaming): reconcile recovery with cursor start positions Integrate retained-log scan and delivery progress with earliest-available subscriptions, preserve legacy resume boundaries, release purged payload references, and fault ADO.NET sources when hard retention crosses unread or unadmitted records. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec261a38-b57f-43b7-b524-02dfb8d049b6 --- .../Streams/KinesisPooledAdapterReceiver.cs | 5 + .../AdoNetQueueAdapterReceiver.cs | 5 + .../AdoNetRecoverableStream.cs | 42 +++++++ .../Common/PooledCache/PooledQueueCache.cs | 28 ++--- .../RecoverableStreamQueueCache.cs | 15 +++ .../RecoverableStreamReceiver.cs | 6 + .../PersistentStreamPullingAgent.cs | 100 +++++++++------ .../QueueStreamDataStructures.cs | 1 + .../Streaming/AdoNetRecoverableStreamTests.cs | 114 +++++++++++++++++- .../Streams/CachedMessageBlockTests.cs | 23 ++++ .../PersistentStreamPullingAgentTests.cs | 81 ++++++++++++- .../PubSubRendezvousGrainTests.cs | 2 +- .../RecoverableStreamReceiverTests.cs | 54 +++++++++ 13 files changed, 415 insertions(+), 61 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index b826b376d67..2302b0b5f2a 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -190,6 +190,11 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _cache.GetCacheCursor(streamId, token); + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => _cache.GetCacheCursorAtPosition(streamId, startPosition); + public bool IsUnderPressure() => _cache.IsUnderPressure(); public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 33e7c149b96..a63a29deffb 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -126,6 +126,11 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _inner.GetCacheCursor(streamId, token); + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => _inner.GetCacheCursorAtPosition(streamId, startPosition); + public bool IsUnderPressure() => _inner.IsUnderPressure(); public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index 82f71385a1d..4823a2793d6 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -14,6 +14,8 @@ internal sealed partial class AdoNetRecoverableStream( private AdoNetStreamPartitionState? _partition; private long _readOffset; private Task? _acquisitionTask; + private DataNotAvailableException? _retentionFailure; + private long? _pendingHardDeletedThrough; internal Task AcquisitionCompletion => Volatile.Read(ref _acquisitionTask) ?? Task.CompletedTask; @@ -96,6 +98,11 @@ public async Task> Read( int maxCount, CancellationToken cancellationToken) { + if (Volatile.Read(ref _retentionFailure) is { } retentionFailure) + { + throw retentionFailure; + } + var messages = await queries.ReadStreamMessagesAsync( serviceId, providerId, @@ -128,6 +135,21 @@ options.MaximumRetentionPeriod is { } maximum cleanup.Checkpoint); } + var readThrough = messages.Count > 0 ? messages[^1].MessageId : _readOffset; + if (cleanup.HardDeletedThroughMessageId is { } hardDeletedThrough + && hardDeletedThrough > readThrough) + { + var failure = new DataNotAvailableException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' lost unread retained records after " + + $"message {readThrough}: hard retention deleted through message {hardDeletedThrough}."); + var retainedFailure = Interlocked.CompareExchange(ref _retentionFailure, failure, comparand: null) ?? failure; + throw retainedFailure; + } + + _pendingHardDeletedThrough = cleanup.HardDeletedThroughMessageId is { } deletedThrough + && deletedThrough > _readOffset + ? deletedThrough + : null; return messages as IReadOnlyList ?? messages.ToList(); } @@ -137,6 +159,26 @@ public void MessagesAdded(IReadOnlyList messages) { _readOffset = messages[^1].MessageId; } + + if (_pendingHardDeletedThrough is { } hardDeletedThrough + && hardDeletedThrough <= _readOffset) + { + _pendingHardDeletedThrough = null; + } + } + + public void MessagesAddFailed(IReadOnlyList messages) + { + if (_pendingHardDeletedThrough is not { } hardDeletedThrough) + { + return; + } + + var failure = new DataNotAvailableException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' could not retain hard-deleted records " + + $"through message {hardDeletedThrough} because cache admission failed before the read offset advanced."); + Interlocked.CompareExchange(ref _retentionFailure, failure, comparand: null); + _pendingHardDeletedThrough = null; } public Task Shutdown(CancellationToken cancellationToken) diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index fe4e074cfaf..bcaf86bd35a 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -405,17 +405,14 @@ private bool TryGetCacheMiss( private void SetCursorAtEarliestAvailable(Cursor cursor) { - for (var node = messageBlocks.Last; node is not null; node = node.Previous) + if (messageBlocks.Last is { } oldestBlock) { - if (node.Value.TryFindFirstMessage(cursor.StreamId, cacheDataAdapter, out var index)) - { - cursor.State = CursorStates.EarliestAvailableSet; - cursor.CurrentBlock = node; - cursor.Index = index; - cursor.SequenceToken = node.Value.GetSequenceToken(index, cacheDataAdapter); - cursor.BlockGeneration = node.Value.Generation; - return; - } + cursor.State = CursorStates.EarliestAvailableSet; + cursor.CurrentBlock = oldestBlock; + cursor.Index = oldestBlock.Value.OldestMessageIndex; + cursor.SequenceToken = oldestBlock.Value.GetOldestSequenceToken(cacheDataAdapter); + cursor.BlockGeneration = oldestBlock.Value.Generation; + return; } SetWaitingAtCurrentEnd(cursor); @@ -439,7 +436,7 @@ private static void SetWaitingAfter(Cursor cursor, LinkedListNode? node; int startIndex; @@ -459,13 +456,12 @@ private bool TrySetCursorAtFirstMatchingMessageAfterAnchor(Cursor cursor) while (node is not null) { if (!node.Value.IsEmpty - && startIndex < node.Value.WriteIndex - && node.Value.TryFindNextMessage(startIndex, cursor.StreamId, cacheDataAdapter, out var index)) + && startIndex < node.Value.WriteIndex) { cursor.State = CursorStates.EarliestAvailableSet; cursor.CurrentBlock = node; - cursor.Index = index; - cursor.SequenceToken = node.Value.GetSequenceToken(index, cacheDataAdapter); + cursor.Index = startIndex; + cursor.SequenceToken = node.Value.GetSequenceToken(startIndex, cacheDataAdapter); cursor.BlockGeneration = node.Value.Generation; return true; } @@ -547,7 +543,7 @@ public QueueCacheCursorMoveResult TryGetNextMessageWithResult(object cursorObj, } if (cursor.State == CursorStates.EarliestAvailableWaiting - && !TrySetCursorAtFirstMatchingMessageAfterAnchor(cursor)) + && !TrySetCursorAfterAnchor(cursor)) { return QueueCacheCursorMoveResult.NoData; } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 11aa7fe2492..19ef94ea2d5 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -206,6 +206,12 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => new Cursor(_cache, streamId, token); + /// + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => new Cursor(_cache, streamId, startPosition); + /// public bool IsUnderPressure() => GetMaxAddCount() <= 0; @@ -272,6 +278,15 @@ public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? to _cursor = cache.GetCursor(streamId, token); } + public Cursor( + PooledQueueCache cache, + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + { + _cache = cache; + _cursor = cache.GetCursorAtPosition(streamId, startPosition); + } + public void Dispose() { } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index c636a769ea4..7b664a3ad95 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -286,6 +286,12 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _cache.GetCacheCursor(streamId, token); + /// + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => _cache.GetCacheCursorAtPosition(streamId, startPosition); + /// public bool IsUnderPressure() => _cache.IsUnderPressure(); diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 310a0ceb17a..5673ff533c1 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -59,11 +59,10 @@ internal interface ITestAccessor Task ReadFromQueueWithCancellation(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount, CancellationToken cancellationToken); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); + IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData); Task RunConsumerCursor(StreamConsumerData consumerData); Task> GetPubSubCache(); Task AddSubscriber(StreamConsumerData consumerData); - Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); - Task RunConsumerCursor(StreamConsumerData consumerData); Task RunQueuePump(QueueId myQueueId, CancellationToken cancellationToken); Task Shutdown(); } @@ -97,7 +96,7 @@ internal PersistentStreamPullingAgent( if (options.DeliveryProgressUpdateInterval <= TimeSpan.Zero) { throw new ArgumentOutOfRangeException( - nameof(options.DeliveryProgressUpdateInterval), + nameof(options), options.DeliveryProgressUpdateInterval, "The delivery progress update interval must be greater than zero."); } @@ -140,7 +139,10 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke }).Unwrap(); Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) - => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); + => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken, ++consumerData.HandshakeRequestId)).Unwrap(); + + IQueueCacheCursor ITestAccessor.GetRecoveryCursor(StreamConsumerData consumerData) + => GetRecoveryCursor(consumerData); Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) => this.RunOrQueueTask(() => RunConsumerCursor(consumerData)); @@ -152,12 +154,6 @@ Task ITestAccessor.AddSubscriber(StreamConsumerData consumerData) => this.RunOrQueueTask(() => AddSubscriber_Impl( consumerData.SubscriptionId, consumerData.StreamId, default, consumerData.FilterData, null)); - Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) - => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken, ++consumerData.HandshakeRequestId)).Unwrap(); - - Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) - => this.RunOrQueueTask(() => RunConsumerCursor(consumerData)); - Task ITestAccessor.RunQueuePump(QueueId myQueueId, CancellationToken cancellationToken) => this.RunOrQueueTask(() => RunQueuePump(myQueueId, cancellationToken)); @@ -685,24 +681,7 @@ private IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData) { if (consumerData.LastProcessedToken is { } lastProcessedToken) { - var result = queueCache!.TryGetCacheCursor(consumerData.StreamId, lastProcessedToken); - if (result.Kind == QueueCacheCursorResultKind.CacheMiss) - { - return GetCacheCursorOrThrow(consumerData.StreamId, null); - } - - if (result.Kind != QueueCacheCursorResultKind.Success) - { - throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."); - } - - var cursor = result.Cursor!; - if (!TryAdvanceRecoveryCursor(cursor, lastProcessedToken, out consumerData.PendingBatch)) - { - return GetCacheCursorOrThrow(consumerData.StreamId, null); - } - - return cursor; + return GetCursorAfterProcessedToken(consumerData, lastProcessedToken); } if (consumerData.LastToken is StartPositionToken startPositionToken) @@ -713,6 +692,12 @@ private IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData) if (consumerData.LastToken is StartToken or DeliveryToken && consumerData.LastToken.Token is { } handshakeSequenceToken) { + if (consumerData.LastToken is DeliveryToken + || SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) + { + return GetCursorAfterProcessedToken(consumerData, handshakeSequenceToken); + } + var result = queueCache!.TryGetCacheCursor(consumerData.StreamId, handshakeSequenceToken); if (result.Kind == QueueCacheCursorResultKind.CacheMiss) { @@ -724,17 +709,7 @@ private IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData) throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."); } - var cursor = result.Cursor!; - if (consumerData.LastToken is DeliveryToken - || SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) - { - if (!TryAdvanceRecoveryCursor(cursor, handshakeSequenceToken, out consumerData.PendingBatch)) - { - return GetCacheCursorOrThrow(consumerData.StreamId, null); - } - } - - return cursor; + return result.Cursor!; } return GetCacheCursorOrThrow(consumerData.StreamId, null); @@ -844,6 +819,48 @@ private static bool TryAdvanceRecoveryCursor( } } + private IQueueCacheCursor GetCursorAfterProcessedToken( + StreamConsumerData consumerData, + StreamSequenceToken processedToken) + { + var restartToken = consumerData.LastSafePartitionToken ?? processedToken; + IQueueCacheCursor cursor; + try + { + cursor = queueCache!.GetCacheCursor(consumerData.StreamId, restartToken); + } + catch (QueueCacheMissException) + { + try + { + cursor = queueCache!.GetCacheCursorAtPosition( + consumerData.StreamId, + StreamSubscriptionStartPosition.EarliestAvailable); + } + catch (NotSupportedException) + { + cursor = queueCache!.GetCacheCursor(consumerData.StreamId, null); + } + } + + if (cursor is IQueueCacheCursorProgress progressCursor) + { + progressCursor.SetDeliveredThrough(processedToken); + } + else + { + if (!Equals(restartToken, processedToken)) + { + cursor.Dispose(); + cursor = queueCache.GetCacheCursor(consumerData.StreamId, processedToken); + } + + cursor.MoveNext(); + } + + return cursor; + } + private IQueueCacheCursor GetCacheMissRecoveryCursor(StreamConsumerData consumerData) { var result = queueCache!.TryGetCacheCursorAtPosition( @@ -858,7 +875,10 @@ private IQueueCacheCursor GetCacheMissRecoveryCursor(StreamConsumerData consumer }; } - public Task RemoveSubscriber(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken) + public Task RemoveSubscriber( + GuidId subscriptionId, + QualifiedStreamId streamId, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); RemoveSubscriber_Impl(subscriptionId, streamId); diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index 3a39f2da9f6..f5574b52e07 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -38,6 +38,7 @@ internal sealed class StreamConsumerData public IBatchContainer? PendingBatch; [NonSerialized] public bool StartPositionIsProviderDefault; + [NonSerialized] public StreamSequenceToken? CursorStartToken; /// diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 3ca139422d6..78f4c547ba2 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -207,6 +207,105 @@ public async Task Read_UsesDistinctRoundedRetentionParameters() Assert.Equal(9, parameters[nameof(DbStoredQueries.Columns.CleanupBatchSize)]); } + [Fact] + public async Task Read_HardRetentionPastReturnedBatchPermanentlyFaultsSource() + { + var storage = new CapturingRelationalStorage + { + ReadRecords = + [ + Record( + (nameof(AdoNetStreamMessage.ServiceId), "service"), + (nameof(AdoNetStreamMessage.ProviderId), "provider"), + (nameof(AdoNetStreamMessage.QueueId), "queue"), + (nameof(AdoNetStreamMessage.MessageId), 10L), + (nameof(AdoNetStreamMessage.StreamIdBytes), new byte[] { 1 }), + (nameof(AdoNetStreamMessage.StreamNamespaceLength), 0), + (nameof(AdoNetStreamMessage.CreatedOn), DateTime.UtcNow), + (nameof(AdoNetStreamMessage.Payload), new byte[] { 2 })), + ], + CleanupRecords = + [ + Record( + (nameof(AdoNetStreamCleanupResult.Ran), true), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 100), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), 100L), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 100), + (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), 1L), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), 100L), + (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), 101L), + (nameof(AdoNetStreamCleanupResult.TailMessageId), 101L)), + ], + }; + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions { MaxMessagesPerRead = 10 }, + CreateQueries(storage), + NullLogger.Instance); + + var first = await Assert.ThrowsAsync( + () => source.Read(10, TestContext.Current.CancellationToken)); + var readsAfterFailure = storage.ReadCallCount; + var second = await Assert.ThrowsAsync( + () => source.Read(10, TestContext.Current.CancellationToken)); + + Assert.Same(first, second); + Assert.Contains("hard retention deleted through message 100", first.Message); + Assert.Equal(readsAfterFailure, storage.ReadCallCount); + } + + [Fact] + public async Task MessagesAddFailed_AfterHardDeletionPermanentlyFaultsSource() + { + var storage = new CapturingRelationalStorage + { + ReadRecords = + [ + Record( + (nameof(AdoNetStreamMessage.ServiceId), "service"), + (nameof(AdoNetStreamMessage.ProviderId), "provider"), + (nameof(AdoNetStreamMessage.QueueId), "queue"), + (nameof(AdoNetStreamMessage.MessageId), 10L), + (nameof(AdoNetStreamMessage.StreamIdBytes), new byte[] { 1 }), + (nameof(AdoNetStreamMessage.StreamNamespaceLength), 0), + (nameof(AdoNetStreamMessage.CreatedOn), DateTime.UtcNow), + (nameof(AdoNetStreamMessage.Payload), new byte[] { 2 })), + ], + CleanupRecords = + [ + Record( + (nameof(AdoNetStreamCleanupResult.Ran), true), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 10), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), 10L), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 10), + (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), 1L), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), 10L), + (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), 11L), + (nameof(AdoNetStreamCleanupResult.TailMessageId), 11L)), + ], + }; + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions { MaxMessagesPerRead = 10 }, + CreateQueries(storage), + NullLogger.Instance); + var messages = await source.Read(10, TestContext.Current.CancellationToken); + + source.MessagesAddFailed(messages); + var readsAfterFailure = storage.ReadCallCount; + var failure = await Assert.ThrowsAsync( + () => source.Read(10, TestContext.Current.CancellationToken)); + + Assert.Contains("cache admission failed", failure.Message); + Assert.Equal(readsAfterFailure, storage.ReadCallCount); + } + [Theory] [InlineData(null, 1L, null, false)] [InlineData(0L, 1L, null, false)] @@ -241,6 +340,9 @@ private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage return new RelationalOrleansQueries(storage, new DbStoredQueries(queryValues)); } + private static IDataRecord Record(params (string Name, object? Value)[] values) + => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); + private sealed class BlockingRelationalStorage : IRelationalStorage { private readonly TaskCompletionSource acquisition = @@ -345,6 +447,12 @@ private sealed class CapturingRelationalStorage : IRelationalStorage public Dictionary> Parameters { get; } = []; + public IReadOnlyList ReadRecords { get; init; } = []; + + public IReadOnlyList? CleanupRecords { get; init; } + + public int ReadCallCount { get; private set; } + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; public string ConnectionString => string.Empty; @@ -356,6 +464,7 @@ public async Task> ReadAsync( CommandBehavior commandBehavior = CommandBehavior.Default, CancellationToken cancellationToken = default) { + ReadCallCount++; CapturedCancellationToken = cancellationToken; using var command = new SqlCommand(); parameterProvider?.Invoke(command); @@ -364,7 +473,7 @@ public async Task> ReadAsync( parameter.Value is DBNull ? null : parameter.Value); var records = query switch { - nameof(DbStoredQueries.ReadStreamMessagesKey) => Array.Empty(), + nameof(DbStoredQueries.ReadStreamMessagesKey) => ReadRecords, nameof(DbStoredQueries.AdvanceStreamCheckpointKey) => [ Record( @@ -375,6 +484,7 @@ public async Task> ReadAsync( (nameof(AdoNetStreamCheckpointUpdate.Checkpoint), 1L), (nameof(AdoNetStreamCheckpointUpdate.Updated), true)), ], + nameof(DbStoredQueries.CleanupStreamMessagesKey) when CleanupRecords is not null => CleanupRecords, nameof(DbStoredQueries.CleanupStreamMessagesKey) => [ Record( @@ -405,8 +515,6 @@ public Task ExecuteAsync( CommandBehavior commandBehavior = CommandBehavior.Default, CancellationToken cancellationToken = default) => throw new NotSupportedException(); - private static IDataRecord Record(params (string Name, object? Value)[] values) - => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); } private sealed class ReservedReceiver : IQueueAdapterReceiver, IQueueCache diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs index 2464d88326d..7ac2fc09bfc 100644 --- a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs @@ -1,6 +1,7 @@ using Orleans.Providers.Streams.Common; using Orleans.Runtime; using Orleans.Streams; +using System.Reflection; using Xunit; namespace UnitTests.OrleansRuntime.Streams @@ -100,6 +101,28 @@ public void Add1Remove1Test() RemoveAndCheck(block, 0, 0); } + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void RemoveAndResetClearPayloadReferences() + { + var pool = new MyTestPooled(); + var block = pool.Allocate(); + var firstPayload = new byte[1024]; + var secondPayload = new byte[1024]; + block.Add(new CachedMessage { Segment = new ArraySegment(firstPayload) }); + block.Add(new CachedMessage { Segment = new ArraySegment(secondPayload) }); + var messages = Assert.IsType( + typeof(CachedMessageBlock) + .GetField("cachedMessages", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(block)); + + Assert.True(block.Remove()); + Assert.Null(messages[0].Segment.Array); + Assert.Same(secondPayload, messages[1].Segment.Array); + + block.Dispose(); + Assert.Null(messages[1].Segment.Array); + } + [Fact, TestCategory("BVT"), TestCategory("Streaming")] public void Add2Remove1UntilFull() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 7890d72202b..b73de00dcf5 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -459,6 +459,41 @@ static async Task WaitForInactive( } } + private sealed class CacheMissRecoveryQueueCache(StreamSequenceToken purgedToken) : IQueueCache + { + private readonly List messages = []; + + public int GetMaxAddCount() => 1000; + + public void AddToCache(IList messages) => this.messages.AddRange(messages); + + public bool TryPurgeFromCache(out IList purgedItems) + { + purgedItems = null!; + return false; + } + + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + { + if (Equals(token, purgedToken)) + { + throw new QueueCacheMissException("The safe restart token was purged."); + } + + return new ScriptedQueueCursor(messages, streamId, token); + } + + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + { + Assert.Equal(StreamSubscriptionStartPosition.EarliestAvailable, startPosition); + return new ScriptedQueueCursor(messages, streamId, token: null); + } + + public bool IsUnderPressure() => false; + } + private static PersistentStreamPullingAgent CreateAgent( IStreamPubSub? pubSub, QueueId queueId, @@ -1988,6 +2023,50 @@ public async Task ReattachmentDeliveryTokenOverridesProviderEarliest() await accessor.Shutdown(); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task RecoveryCursorSeedsAcknowledgedProgressWithoutPendingDelivery() + { + var quietStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var busyStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var quietQualifiedId = new QualifiedStreamId("provider", quietStreamId); + var safeToken = new EventSequenceTokenV2(1); + var processedToken = new EventSequenceTokenV2(2); + var scannedToken = new EventSequenceTokenV2(3); + var nextQuietToken = new EventSequenceTokenV2(4); + var queueCache = new CacheMissRecoveryQueueCache(safeToken); + queueCache.AddToCache( + [ + new TestBatchContainer(quietStreamId, processedToken), + new TestBatchContainer(busyStreamId, scannedToken), + new TestBatchContainer(quietStreamId, nextQuietToken), + ]); + var (accessor, _, streamData) = await CreateInitializedAgentWithStream( + quietQualifiedId, + processedToken, + queueCache, + new StreamPullingAgentOptions()); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + quietQualifiedId, + new ImmediateConsumer(), + filterData: null, + now: DateTime.UtcNow); + consumerData.LastProcessedToken = processedToken; + consumerData.LastSafePartitionToken = safeToken; + + using var cursor = accessor.GetRecoveryCursor(consumerData); + + Assert.True(cursor.MoveNext()); + var current = cursor.GetCurrent(out _); + Assert.NotNull(current); + Assert.Equal(nextQuietToken, current.SequenceToken); + Assert.Equal(scannedToken, Assert.IsAssignableFrom(cursor).SafeSequenceToken); + await accessor.Shutdown(); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4997,7 +5076,7 @@ public void DeliveryProgress_RejectsNonPositiveUpdateInterval() var exception = Assert.Throws(() => CreateAgent(pubSub: null, queueId, options: options)); - Assert.Equal(nameof(options.DeliveryProgressUpdateInterval), exception.ParamName); + Assert.Equal(nameof(options), exception.ParamName); } public enum SubscriptionStartTokenSource diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs index 4aa3ef4c17f..6a03cf63b03 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs @@ -709,7 +709,7 @@ public async Task UnregisterLastConsumerEmitsDiagnosticAfterClearingState() var streamId = new QualifiedStreamId("ProviderName", StreamId.Create("StreamNamespace", Guid.NewGuid())); var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); var pubSubGrain = this.fixture.GrainFactory.GetGrain(streamId.ToString()); - using var observer = StreamingDiagnosticObserver.Create(); + using var observer = StreamingDiagnosticObserver.Create(fixture.HostedCluster); await pubSubGrain.RegisterConsumer(subscriptionId, streamId, default, null!); await pubSubGrain.UnregisterConsumer(subscriptionId, streamId); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 46339c1d69c..2147c856235 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -324,6 +324,60 @@ public void Cache_CursorCreatedWhileEmptyReadsFirstLaterRecord() Assert.Equal("payload", Assert.IsType(cursor.GetCurrent(out _)).Payload); } + [Fact] + public void Cache_EarliestAvailableReturnsOldestMatchingRecord() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var otherStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + _ = cache.Add( + [ + new TestQueueMessage(otherStreamId, 1, "other"), + new TestQueueMessage(streamId, 2, "first"), + new TestQueueMessage(streamId, 3, "second"), + ], + DateTime.UnixEpoch); + + using var cursor = cache.GetCacheCursorAtPosition( + streamId, + StreamSubscriptionStartPosition.EarliestAvailable); + + Assert.True(cursor.MoveNext()); + Assert.Equal("first", Assert.IsType(cursor.GetCurrent(out _)).Payload); + Assert.Equal(1, Assert.IsAssignableFrom(cursor).SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public void Cache_EarliestAvailableQuietStreamScansUnrelatedRecords() + { + var quietStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var busyStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + _ = cache.Add( + [ + new TestQueueMessage(busyStreamId, 1, "first"), + new TestQueueMessage(busyStreamId, 2, "second"), + ], + DateTime.UnixEpoch); + + using var cursor = cache.GetCacheCursorAtPosition( + quietStreamId, + StreamSubscriptionStartPosition.EarliestAvailable); + + Assert.False(cursor.MoveNext()); + Assert.Equal(2, Assert.IsAssignableFrom(cursor).SafeSequenceToken?.SequenceNumber); + } + [Fact] public void CachePressure_BlocksUnsafeTimePurgeUntilDeliveryProgressAdvances() { From d8121797eea8f3f440fad4b8459365fe50216e90 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 3 Sep 2026 15:33:10 -0700 Subject: [PATCH 40/90] fix(streaming): align recovery with cancellation contracts Preserve batches returned during shutdown, apply cancellation at subscription registration boundaries, and update cancellation-aware stream test doubles. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../PersistentStreamPullingAgent.cs | 22 +------ .../AdoNetQueueAdapterFactoryTests.cs | 2 +- .../PersistentStreamPullingAgentTests.cs | 64 +++++++++---------- .../PubSubRendezvousGrainTests.cs | 11 +++- 4 files changed, 43 insertions(+), 56 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 5673ff533c1..17763fb0715 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -49,7 +49,6 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task _activePumpTask = Task.CompletedTask; private AdmissionGate _workAdmission = new(); private Task? _shutdownTask; - private int _hasUnprocessedRead; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -184,7 +183,6 @@ public async Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; - Volatile.Write(ref _hasUnprocessedRead, 0); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -294,7 +292,7 @@ private async Task ShutdownCore(CancellationToken cancellationToken) await drainTask; // All accepted work has finished progress bookkeeping and released its batch/registration pins. - if (!hasPendingRegistrations && Volatile.Read(ref _hasUnprocessedRead) == 0) + if (!hasPendingRegistrations) { NotifyDeliveryProgress(); } @@ -1043,16 +1041,6 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList? multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount, cancellationToken); - if (IsShutdown || cancellationToken.IsCancellationRequested) - { - if (multiBatch is { Count: > 0 }) - { - Volatile.Write(ref _hasUnprocessedRead, 1); - } - return false; - } - } - // Receivers built against older Orleans versions can still return null. if (multiBatch is null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback. @@ -1080,12 +1068,6 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? foreach (var group in availableMessages.GroupBy(container => container.StreamId)) { - if (IsShutdown || cancellationToken.IsCancellationRequested) - { - Volatile.Write(ref _hasUnprocessedRead, 1); - return false; - } - var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); StreamSequenceToken startToken = group.First().SequenceToken; if (pubSubCache.TryGetValue(streamId, out var streamData)) @@ -1373,7 +1355,7 @@ void FailRegistration(Exception exception) async Task SubscribeWithIsolation(PubSubSubscriptionState item) { - if (IsShutdown) + if (IsShutdown || cancellationToken.IsCancellationRequested) { return; } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index 958e506541f..08d5bb1efb9 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -111,7 +111,7 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() adapter.CreateReceiver(queueId), factory.GetQueueAdapterCache().CreateQueueCache(queueId)); var firstReceiver = adapter.CreateReceiver(queueId); - await firstReceiver.Shutdown(TimeSpan.FromSeconds(5)); + await firstReceiver.Shutdown(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var reassignedReceiver = adapter.CreateReceiver(queueId); Assert.NotSame(firstReceiver, reassignedReceiver); Assert.Same( diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index b73de00dcf5..09f44a68988 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -2569,23 +2569,23 @@ private sealed class ImmediateConsumer : IStreamConsumerExtension { public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); - public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); - public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken) + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) { Delivered.TrySetResult(true); return Task.FromResult(null); } - public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; - public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToken cancellationToken) => Task.CompletedTask; - public Task GetSequenceToken(GuidId subscriptionId) => Task.FromResult(null); + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(null); } [TestSuite("BVT")] @@ -3508,9 +3508,9 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => IStreamFailureHandler? failureHandler = null) { var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - pubSub.FaultSubscription(Arg.Any(), Arg.Any()) + pubSub.FaultSubscription(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(true)); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); @@ -3892,7 +3892,7 @@ public async Task Shutdown_DrainsRegistrationPinBeforeReceiverCleanup(string out [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_DoesNotCheckpointBatchReturnedAfterShutdownStarts() + public async Task Shutdown_CachesBatchReturnedAfterShutdownStarts() { var queueReadStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var queueReadReleased = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); @@ -3905,7 +3905,7 @@ public async Task Shutdown_DoesNotCheckpointBatchReturnedAfterShutdownStarts() queueReadStarted.TrySetResult(); return await queueReadReleased.Task; }); - receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var queueCache = Substitute.For(); queueCache.GetMaxAddCount().Returns(1); var queueAdapterCache = Substitute.For(); @@ -3925,10 +3925,7 @@ public async Task Shutdown_DoesNotCheckpointBatchReturnedAfterShutdownStarts() await shutdownTask; await pumpTask; - queueCache.DidNotReceive().AddToCache(Arg.Any>()); - queueCache.DidNotReceive().UpdateDeliveryProgress( - Arg.Any(), - Arg.Any()); + queueCache.Received(1).AddToCache(Arg.Is>(messages => messages.Count == 1)); } [TestSuite("BVT")] @@ -4459,7 +4456,7 @@ public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparis queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -4523,7 +4520,7 @@ public async Task DeliveryProgress_InclusiveStartRemainsUnsafeBeforeFirstBatch( queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( pubSub, @@ -4588,7 +4585,7 @@ public async Task Handshake_DeliveryTokenStartsAfterConfirmedRecord() .Returns(Task.FromResult>( [new TestBatchContainer(streamId, new EventSequenceTokenV2(12))])); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -4633,7 +4630,7 @@ public async Task ReadFromQueue_WakesIdleCursorAtFirstPartitionRecord() var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -4672,7 +4669,7 @@ public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -4727,7 +4724,7 @@ .. Enumerable.Range(2, 8) var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -4799,7 +4796,7 @@ public async Task DeliveryProgress_QuietSubscriptionAdvancesAcrossBusyStreamBeyo var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( pubSub, @@ -4873,7 +4870,7 @@ public async Task DeliveryProgress_PendingRegistrationBlocksPeriodicUpdate() { var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default).ReturnsForAnyArgs(_ => registration.Task); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken).ReturnsForAnyArgs(_ => registration.Task); var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); var options = new StreamPullingAgentOptions(); var queueId = QueueId.GetQueueId("queue", 0u, 0u); @@ -4924,7 +4921,7 @@ public async Task DeliveryProgress_IncludesFilteredBatches() var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( pubSub, @@ -4979,12 +4976,12 @@ public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() var receiver = Substitute.For(); receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult>([new TestBatchContainer(streamId, currentToken)])); - receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var queueCache = new ScriptedQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default) + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -5049,7 +5046,7 @@ public async Task Shutdown_StopsPeriodicDeliveryProgressUpdates() var options = new StreamPullingAgentOptions(); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var receiver = Substitute.For(); - receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var queueCache = new RecordingQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); @@ -5101,30 +5098,33 @@ private sealed class StartingConsumer(StreamHandshakeToken? startToken) : IStrea QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, - StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) => throw new NotSupportedException(); public Task DeliverMutable( GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, - StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) => throw new NotSupportedException(); public Task DeliverBatch( GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, - StreamHandshakeToken? handshakeToken) + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) { DeliveredTokens.Add(item.SequenceToken); return Task.FromResult(null); } - public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; - public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToken cancellationToken) => Task.CompletedTask; - public Task GetSequenceToken(GuidId subscriptionId) + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(startToken); } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs index 6a03cf63b03..05a67d5f212 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs @@ -711,8 +711,13 @@ public async Task UnregisterLastConsumerEmitsDiagnosticAfterClearingState() var pubSubGrain = this.fixture.GrainFactory.GetGrain(streamId.ToString()); using var observer = StreamingDiagnosticObserver.Create(fixture.HostedCluster); - await pubSubGrain.RegisterConsumer(subscriptionId, streamId, default, null!); - await pubSubGrain.UnregisterConsumer(subscriptionId, streamId); + await pubSubGrain.RegisterConsumer( + subscriptionId, + streamId, + default, + null!, + TestContext.Current.CancellationToken); + await pubSubGrain.UnregisterConsumer(subscriptionId, streamId, TestContext.Current.CancellationToken); var unregistered = await observer.WaitForSubscriptionUnregisteredAsync( streamId.StreamId, @@ -721,7 +726,7 @@ public async Task UnregisterLastConsumerEmitsDiagnosticAfterClearingState() TestContext.Current.CancellationToken); Assert.Equal(streamId.StreamId, unregistered.StreamId); Assert.Equal(subscriptionId.Guid, unregistered.SubscriptionId); - Assert.Equal(0, await pubSubGrain.ConsumerCount(streamId)); + Assert.Equal(0, await pubSubGrain.ConsumerCount(streamId, TestContext.Current.CancellationToken)); } /// From 8466897f359c7d567d9b437bd28f8675aee035cc Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 3 Sep 2026 15:38:28 -0700 Subject: [PATCH 41/90] fix: forward remaining initialization cancellation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../Streaming/StreamPartitionReadCheckpointBenchmark.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs index 23af845f961..2cf9a31766d 100644 --- a/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs @@ -99,7 +99,8 @@ await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => "ProviderId-0", _partitionIds[i % PartitionCount], afterMessageId: 0, - BatchSize); + BatchSize, + cancellationToken); _consumer.Consume(messages); }); } From 6b32bbbf56d29d282c9f4afd6a98deeee581dfe6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 03:38:33 -0700 Subject: [PATCH 42/90] fix(streaming): validate measurable failure counts Skip the dropped-client delivery-failure assertion when the provider does not expose a real counter, and restore the ADO.NET message file's required final newline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs | 2 +- .../StreamingTests/ClientStreamTestRunner.cs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs index c5b2231ecf9..6b5295fc22e 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs @@ -23,4 +23,4 @@ public AdoNetStreamMessage() : this("", "", "", 0, [], 0, DateTime.MinValue, []) public StreamId StreamId => StreamId.Create( StreamIdBytes.AsSpan(0, StreamNamespaceLength), StreamIdBytes.AsSpan(StreamNamespaceLength)); -} \ No newline at end of file +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs index 1995ad4ff02..0238bb81039 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs @@ -12,7 +12,6 @@ namespace Tester.StreamingTests { public class ClientStreamTestRunner { - private static readonly Func> DefaultDeliveryFailureCount = () => Task.FromResult(0); private static readonly TimeSpan _timeout = TimeSpan.FromMinutes(3); private readonly TestCluster testHost; @@ -52,8 +51,6 @@ public async Task StreamConsumerOnDroppedClientTest( bool waitForRetryTimeouts = false, CancellationToken cancellationToken = default) { - getDeliveryFailureCount ??= DefaultDeliveryFailureCount; - Guid streamGuid = Guid.NewGuid(); var streamId = StreamId.Create(streamNamespace, streamGuid); int[] eventCount = { 0 }; @@ -82,8 +79,10 @@ public async Task StreamConsumerOnDroppedClientTest( await streamingObserver.WaitForSubscriptionUnregisteredAsync(streamId, droppedSubscriptionId, streamProviderName, cts.Token); } - int deliveryFailureCount = await getDeliveryFailureCount(); - Assert.Equal(0, deliveryFailureCount); + if (getDeliveryFailureCount is not null) + { + Assert.Equal(0, await getDeliveryFailureCount()); + } } private Task> SubscribeToStream(string streamProviderName, Guid streamGuid, string streamNamespace, From 40c0af0c185555fd45146ca14793e1df109c12f8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 10:50:48 -0700 Subject: [PATCH 43/90] fix(streaming): serialize adapter initialization Ensure Kinesis and ADO.NET adapter factories publish a single initialized resource set under concurrent CreateAdapter calls, preserving retry behavior after failures and handling ADO.NET initialization timeouts without over-releasing its semaphore. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec261a38-b57f-43b7-b524-02dfb8d049b6 --- .../Streams/KinesisAdapterFactory.cs | 37 +++++-- .../AdoNetQueueAdapterFactory.cs | 51 ++++++++-- .../Streaming/AdoNetRecoverableStreamTests.cs | 98 +++++++++++++++++++ .../KinesisRuntimeTests.cs | 62 ++++++++++++ 4 files changed, 230 insertions(+), 18 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 9eb953cc0c4..53b97d2b772 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -32,6 +32,7 @@ internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQue private readonly Func _queueMapperFactory; private readonly IAmazonKinesis _client; private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _initializationSemaphore = new(1); private HashRingBasedPartitionedStreamQueueMapper _streamQueueMapper = null!; private KinesisShardTopologyMonitor _topologyMonitor = null!; @@ -91,21 +92,39 @@ public async Task CreateAdapter() public async Task CreateAdapter(CancellationToken cancellationToken) { - if (_streamQueueMapper is null) + if (Volatile.Read(ref _streamQueueMapper) is not null) { - var kinesisStreams = await GetPartitionIdsAsync(cancellationToken); - _streamQueueMapper = _queueMapperFactory(kinesisStreams); - _topologyMonitor = new( + return this; + } + + await _initializationSemaphore.WaitAsync(cancellationToken); + try + { + if (_streamQueueMapper is not null) + { + return this; + } + + var partitions = await GetPartitionIdsAsync(cancellationToken); + var queueMapper = _queueMapperFactory(partitions); + var topologyMonitor = new KinesisShardTopologyMonitor( _client, _options.StreamName, - kinesisStreams, + partitions, _options.TopologyCheckInterval, _timeProvider, _loggerFactory.CreateLogger()); - _receivers = new QueueAdapterReceiverRegistry(MakeReceiver); - } + var receivers = new QueueAdapterReceiverRegistry(MakeReceiver); - return this; + _topologyMonitor = topologyMonitor; + _receivers = receivers; + Volatile.Write(ref _streamQueueMapper, queueMapper); + return this; + } + finally + { + _initializationSemaphore.Release(); + } } public IQueueAdapterCache GetQueueAdapterCache() @@ -218,7 +237,7 @@ internal static string GetRegionName(KinesisStreamOptions options) internal async Task GetPartitionIdsAsync() => await GetPartitionIdsAsync(CancellationToken.None); - internal async Task GetPartitionIdsAsync(CancellationToken cancellationToken) + internal virtual async Task GetPartitionIdsAsync(CancellationToken cancellationToken) => await GetPartitionIdsAsync(_client, _options.StreamName, cancellationToken); internal static Task GetPartitionIdsAsync(IAmazonKinesis client, string streamName) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index c3d5cb3d76c..d115402c49a 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -29,7 +29,7 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, private readonly AdoNetStreamQueueMapper _adoNetQueueMapper; private RelationalOrleansQueries? _queries; - private AdoNetQueueAdapter? _adapter; + private IQueueAdapter? _adapter; /// /// Unfortunate implementation detail to account for lack of async lifetime. @@ -40,15 +40,15 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, /// /// Ensures queries are loaded only once while allowing for recovery if the load fails. /// - private ValueTask GetQueriesAsync() + internal virtual ValueTask GetQueriesAsync() { // attempt fast path - return _queries is not null ? new(_queries) : new(CoreAsync()); + return Volatile.Read(ref _queries) is { } queries ? new(queries) : new(CoreAsync()); // slow path async Task CoreAsync() { - await _semaphore.WaitAsync(_streamOptions.InitializationTimeout, _lifetime.ApplicationStopping); + await WaitForInitializationLockAsync(); try { // attempt fast path again @@ -58,9 +58,11 @@ async Task CoreAsync() } // slow path - the member variable will only be set if the call succeeds - return _queries = await RelationalOrleansQueries + var result = await RelationalOrleansQueries .CreateInstance(_streamOptions.Invariant, _streamOptions.ConnectionString, _streamOptions.DataSource) .WaitAsync(_streamOptions.InitializationTimeout); + Volatile.Write(ref _queries, result); + return result; } finally { @@ -71,13 +73,44 @@ async Task CoreAsync() public async Task CreateAdapter() { + if (Volatile.Read(ref _adapter) is { } adapter) + { + return adapter; + } + var queries = await GetQueriesAsync(); - return _adapter ??= (AdoNetQueueAdapter)AdapterFactory( - _serviceProvider, - [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries]); + await WaitForInitializationLockAsync(); + try + { + if (_adapter is not null) + { + return _adapter; + } + + var result = await CreateAdapterCore(queries); + Volatile.Write(ref _adapter, result); + return result; + } + finally + { + _semaphore.Release(); + } } + private async Task WaitForInitializationLockAsync() + { + if (!await _semaphore.WaitAsync(_streamOptions.InitializationTimeout, _lifetime.ApplicationStopping)) + { + throw new TimeoutException($"Timed out waiting to initialize ADO.NET stream provider '{_name}'."); + } + } + + internal virtual ValueTask CreateAdapterCore(RelationalOrleansQueries queries) + => new((AdoNetQueueAdapter)AdapterFactory( + _serviceProvider, + [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries])); + public async Task GetDeliveryFailureHandler(QueueId queueId) { var queries = await GetQueriesAsync(); @@ -88,7 +121,7 @@ public async Task GetDeliveryFailureHandler(QueueId queue public IQueueAdapterCache GetQueueAdapterCache() => this; public IQueueCache CreateQueueCache(QueueId queueId) - => (_adapter ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) + => ((_adapter as IQueueAdapterCache) ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) .CreateQueueCache(queueId); public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 78f4c547ba2..c435e65a75e 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -2,11 +2,13 @@ using System.Reflection; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; using Orleans.Configuration; using Orleans.Providers.Streams.Common; using Orleans.Streaming.AdoNet; using Orleans.Streaming.AdoNet.Storage; using Orleans.Streams; +using Tester.AdoNet.Fakes; namespace Tester.AdoNet.Streaming; @@ -16,6 +18,63 @@ namespace Tester.AdoNet.Streaming; [TestArea("Streaming")] public class AdoNetRecoverableStreamTests { + [Fact] + public async Task ConcurrentCreateAdapterCallsConstructOnce() + { + const int callerCount = 8; + var lifetime = new FakeHostApplicationLifetime(); + var adapter = Substitute.For(); + var factory = new BlockingAdoNetQueueAdapterFactory( + CreateQueries(new CapturingRelationalStorage()), + adapter, + lifetime); + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var invoked = new CountdownEvent(callerCount); + var tasks = Enumerable.Range(0, callerCount).Select(async _ => + { + await start.Task; + var operation = factory.CreateAdapter(); + invoked.Signal(); + return await operation; + }).ToArray(); + + start.SetResult(); + await Task.Run( + () => invoked.Wait(TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + var adapterConstructionCount = factory.AdapterConstructionCount; + factory.CompleteAdapterConstruction(); + + var adapters = await Task.WhenAll(tasks); + Assert.Equal(1, adapterConstructionCount); + Assert.Equal(1, factory.AdapterConstructionCount); + Assert.All(adapters, result => Assert.Same(adapter, result)); + } + + [Fact] + public async Task TimedOutCreateAdapterDoesNotEnterCriticalSection() + { + var lifetime = new FakeHostApplicationLifetime(); + var adapter = Substitute.For(); + var factory = new BlockingAdoNetQueueAdapterFactory( + CreateQueries(new CapturingRelationalStorage()), + adapter, + lifetime, + TimeSpan.FromMilliseconds(50)); + + var first = factory.CreateAdapter(); + await factory.AdapterConstructionStarted.Task; + + await Assert.ThrowsAsync(() => factory.CreateAdapter()); + Assert.Equal(1, factory.AdapterConstructionCount); + + factory.CompleteAdapterConstruction(); + Assert.Same(adapter, await first); + Assert.Same(adapter, await factory.CreateAdapter()); + Assert.Equal(1, factory.AdapterConstructionCount); + } + [Fact] public void ResolveCheckpointUpdate_ReturnsAuthoritativeStateForExpectedVersionConflict() { @@ -343,6 +402,45 @@ private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage private static IDataRecord Record(params (string Name, object? Value)[] values) => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); + private sealed class BlockingAdoNetQueueAdapterFactory( + RelationalOrleansQueries queries, + IQueueAdapter adapter, + FakeHostApplicationLifetime lifetime, + TimeSpan? initializationTimeout = null) + : AdoNetQueueAdapterFactory( + "provider", + new AdoNetStreamOptions { InitializationTimeout = initializationTimeout ?? TimeSpan.FromSeconds(5) }, + new ClusterOptions(), + new SimpleQueueCacheOptions(), + new HashRingStreamQueueMapperOptions(), + NullLoggerFactory.Instance, + lifetime, + Substitute.For()) + { + private readonly TaskCompletionSource _adapterConstruction = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _adapterConstructionStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _adapterConstructionCount; + + public int AdapterConstructionCount => Volatile.Read(ref _adapterConstructionCount); + + public TaskCompletionSource AdapterConstructionStarted => _adapterConstructionStarted; + + public void CompleteAdapterConstruction() => _adapterConstruction.SetResult(); + + internal override ValueTask GetQueriesAsync() => new(queries); + + internal override async ValueTask CreateAdapterCore(RelationalOrleansQueries value) + { + Assert.Same(queries, value); + Interlocked.Increment(ref _adapterConstructionCount); + _adapterConstructionStarted.TrySetResult(); + await _adapterConstruction.Task; + return adapter; + } + } + private sealed class BlockingRelationalStorage : IRelationalStorage { private readonly TaskCompletionSource acquisition = diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index c01807adf32..998d925dac2 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -113,6 +113,37 @@ public void QueueAdapterFactoryIsRewindable() Assert.Equal(StreamProviderDirection.ReadWrite, factory.Direction); } + [Fact] + public async Task ConcurrentCreateAdapterCallsInitializeOnce() + { + const int callerCount = 8; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var factory = new BlockingKinesisAdapterFactory( + services.GetRequiredService>()); + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var invoked = new CountdownEvent(callerCount); + var tasks = Enumerable.Range(0, callerCount).Select(async _ => + { + await start.Task; + var operation = factory.CreateAdapter(TestContext.Current.CancellationToken); + invoked.Signal(); + return await operation; + }).ToArray(); + + start.SetResult(); + await Task.Run( + () => invoked.Wait(TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + var partitionDiscoveryCount = factory.PartitionDiscoveryCount; + factory.CompletePartitionDiscovery(); + + var adapters = await Task.WhenAll(tasks); + Assert.Equal(1, partitionDiscoveryCount); + Assert.Equal(1, factory.PartitionDiscoveryCount); + Assert.All(adapters, adapter => Assert.Same(factory, adapter)); + } + [Fact] public async Task PooledReceiver_ReadsAndDisposesLifecycleCancellationOnShutdown() { @@ -166,6 +197,37 @@ await client.Received(1).GetRecordsAsync( Assert.Throws(() => _ = receiver.LifecycleCancellationToken); } + private sealed class BlockingKinesisAdapterFactory(Serializer serializer) + : KinesisAdapterFactory( + "Kinesis", + new KinesisStreamOptions + { + StreamName = "stream", + Service = "http://localhost:4566", + AccessKey = "access-key", + SecretKey = "secret-key", + }, + new SimpleQueueCacheOptions(), + serializer, + checkpointerFactory: null, + NullLoggerFactory.Instance) + { + private readonly TaskCompletionSource _partitionDiscovery = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _partitionDiscoveryCount; + + public int PartitionDiscoveryCount => Volatile.Read(ref _partitionDiscoveryCount); + + public void CompletePartitionDiscovery() => _partitionDiscovery.SetResult(); + + internal override async Task GetPartitionIdsAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _partitionDiscoveryCount); + await _partitionDiscovery.Task.WaitAsync(cancellationToken); + return ["shard-1"]; + } + } + [Fact] public async Task InitialShardIteratorUsesTrimHorizonWhenNoCheckpointExists() { From 6a18ccbd1064e448a6863c0c8e1772292f772dd2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 16:08:54 -0700 Subject: [PATCH 44/90] fix(streaming): guard Kinesis receiver initialization Report the required CreateAdapter ordering before receiver or cache creation instead of dereferencing uninitialized factory state. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../Streams/KinesisAdapterFactory.cs | 9 ++++++++- .../KinesisRuntimeTests.cs | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 53b97d2b772..266924077b9 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -158,13 +158,20 @@ public IQueueCache CreateQueueCache(QueueId queueId) private KinesisPooledAdapterReceiver GetOrCreateReceiver(QueueId queueId) { + var receivers = Volatile.Read(ref _receivers); + if (receivers is null) + { + throw new InvalidOperationException( + $"The Kinesis stream provider '{Name}' must complete {nameof(CreateAdapter)} before creating a receiver or cache."); + } + if (_checkpointerFactory is null) { throw new OrleansConfigurationException( $"No {nameof(IStreamQueueCheckpointerFactory)} is configured for the Kinesis stream provider '{Name}'."); } - return _receivers.GetOrCreate(queueId); + return receivers.GetOrCreate(queueId); } private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index 998d925dac2..ae63c92b0e4 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -113,6 +113,26 @@ public void QueueAdapterFactoryIsRewindable() Assert.Equal(StreamProviderDirection.ReadWrite, factory.Direction); } + [Fact] + public void ReceiverAndCacheCreationRequireAdapterInitialization() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var factory = new KinesisAdapterFactory( + "Kinesis", + new KinesisStreamOptions(), + new SimpleQueueCacheOptions(), + services.GetRequiredService>(), + Substitute.For(), + NullLoggerFactory.Instance); + var queueId = QueueId.GetQueueId("queue", 0, 0); + + var receiverException = Assert.Throws(() => factory.CreateReceiver(queueId)); + var cacheException = Assert.Throws(() => factory.CreateQueueCache(queueId)); + + Assert.Contains(nameof(KinesisAdapterFactory.CreateAdapter), receiverException.Message, StringComparison.Ordinal); + Assert.Equal(receiverException.Message, cacheException.Message); + } + [Fact] public async Task ConcurrentCreateAdapterCallsInitializeOnce() { From c85ee97c222fdc7b7a11eb046e4733ae30310701 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 16:08:55 -0700 Subject: [PATCH 45/90] fix(packaging): acknowledge ADO.NET alpha breaks Record the approved removal of the obsolete visibility and dead-letter queue options for package compatibility validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../CompatibilitySuppressions.xml | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 src/AdoNet/Orleans.Streaming.AdoNet/CompatibilitySuppressions.xml diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/CompatibilitySuppressions.xml b/src/AdoNet/Orleans.Streaming.AdoNet/CompatibilitySuppressions.xml new file mode 100644 index 00000000000..d2e4dfb79a5 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/CompatibilitySuppressions.xml @@ -0,0 +1,172 @@ + + + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_DeadLetterEvictionTimeout + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_EvictionBatchSize + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_EvictionInterval + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_ExpiryTimeout + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_MaxAttempts + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_VisibilityTimeout + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_DeadLetterEvictionTimeout(System.TimeSpan) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_EvictionBatchSize(System.Int32) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_EvictionInterval(System.TimeSpan) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_ExpiryTimeout(System.TimeSpan) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_MaxAttempts(System.Int32) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_VisibilityTimeout(System.TimeSpan) + lib/net10.0/Orleans.Streaming.AdoNet.dll + lib/net10.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_DeadLetterEvictionTimeout + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_EvictionBatchSize + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_EvictionInterval + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_ExpiryTimeout + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_MaxAttempts + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.get_VisibilityTimeout + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_DeadLetterEvictionTimeout(System.TimeSpan) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_EvictionBatchSize(System.Int32) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_EvictionInterval(System.TimeSpan) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_ExpiryTimeout(System.TimeSpan) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_MaxAttempts(System.Int32) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + + CP0002 + M:Orleans.Configuration.AdoNetStreamOptions.set_VisibilityTimeout(System.TimeSpan) + lib/net8.0/Orleans.Streaming.AdoNet.dll + lib/net8.0/Orleans.Streaming.AdoNet.dll + true + + \ No newline at end of file From ef42dd055f63cfaccc3cba06e8e979a433f344a5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 17:29:44 -0700 Subject: [PATCH 46/90] style(streaming): sort cached block usings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../OrleansRuntime/Streams/CachedMessageBlockTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs index 7ac2fc09bfc..dde7bea5b1b 100644 --- a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/CachedMessageBlockTests.cs @@ -1,7 +1,7 @@ +using System.Reflection; using Orleans.Providers.Streams.Common; using Orleans.Runtime; using Orleans.Streams; -using System.Reflection; using Xunit; namespace UnitTests.OrleansRuntime.Streams From 7913ba5851b13e7a93e5a90f5cb8b07a0bdbf016 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 17:43:39 -0700 Subject: [PATCH 47/90] fix(streaming): publish Kinesis adapter state atomically Publish the queue mapper and receiver registry as one immutable state so concurrent receiver/cache creation cannot observe partially initialized adapter resources. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../Streams/KinesisAdapterFactory.cs | 48 ++++++++++--------- .../KinesisRuntimeTests.cs | 30 +++++++++++- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 266924077b9..54766720a12 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -34,9 +34,7 @@ internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQue private readonly TimeProvider _timeProvider; private readonly SemaphoreSlim _initializationSemaphore = new(1); - private HashRingBasedPartitionedStreamQueueMapper _streamQueueMapper = null!; - private KinesisShardTopologyMonitor _topologyMonitor = null!; - private QueueAdapterReceiverRegistry _receivers = null!; + private InitializationState? _initializationState; private int _disposed; public KinesisAdapterFactory( @@ -92,7 +90,7 @@ public async Task CreateAdapter() public async Task CreateAdapter(CancellationToken cancellationToken) { - if (Volatile.Read(ref _streamQueueMapper) is not null) + if (Volatile.Read(ref _initializationState) is not null) { return this; } @@ -100,7 +98,7 @@ public async Task CreateAdapter(CancellationToken cancellationTok await _initializationSemaphore.WaitAsync(cancellationToken); try { - if (_streamQueueMapper is not null) + if (_initializationState is not null) { return this; } @@ -114,11 +112,11 @@ public async Task CreateAdapter(CancellationToken cancellationTok _options.TopologyCheckInterval, _timeProvider, _loggerFactory.CreateLogger()); - var receivers = new QueueAdapterReceiverRegistry(MakeReceiver); + QueueAdapterReceiverRegistry? receivers = null; + receivers = new( + queueId => MakeReceiver(queueId, queueMapper, topologyMonitor, receivers!)); - _topologyMonitor = topologyMonitor; - _receivers = receivers; - Volatile.Write(ref _streamQueueMapper, queueMapper); + Volatile.Write(ref _initializationState, new(queueMapper, receivers)); return this; } finally @@ -131,7 +129,7 @@ public IQueueAdapterCache GetQueueAdapterCache() => this; public IStreamQueueMapper GetStreamQueueMapper() - => _streamQueueMapper; + => GetInitializationState().StreamQueueMapper; public Task GetDeliveryFailureHandler(QueueId queueId) => Task.FromResult(new NoOpStreamDeliveryFailureHandler(false)); @@ -158,25 +156,22 @@ public IQueueCache CreateQueueCache(QueueId queueId) private KinesisPooledAdapterReceiver GetOrCreateReceiver(QueueId queueId) { - var receivers = Volatile.Read(ref _receivers); - if (receivers is null) - { - throw new InvalidOperationException( - $"The Kinesis stream provider '{Name}' must complete {nameof(CreateAdapter)} before creating a receiver or cache."); - } - if (_checkpointerFactory is null) { throw new OrleansConfigurationException( $"No {nameof(IStreamQueueCheckpointerFactory)} is configured for the Kinesis stream provider '{Name}'."); } - return receivers.GetOrCreate(queueId); + return GetInitializationState().Receivers.GetOrCreate(queueId); } - private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) + private KinesisPooledAdapterReceiver MakeReceiver( + QueueId queueId, + HashRingBasedPartitionedStreamQueueMapper streamQueueMapper, + KinesisShardTopologyMonitor topologyMonitor, + QueueAdapterReceiverRegistry receivers) { - var partition = _streamQueueMapper.QueueToPartition(queueId); + var partition = streamQueueMapper.QueueToPartition(queueId); return new KinesisPooledAdapterReceiver( CreateClient(), _options.StreamName, @@ -185,13 +180,18 @@ private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) _cacheOptions, _serializer, _loggerFactory, - _topologyMonitor, + topologyMonitor, _options.GetRecordsInterval, _timeProvider, - receiver => _receivers.Remove(queueId, receiver) + receiver => receivers.Remove(queueId, receiver) ); } + private InitializationState GetInitializationState() + => Volatile.Read(ref _initializationState) + ?? throw new InvalidOperationException( + $"The Kinesis stream provider '{Name}' must complete {nameof(CreateAdapter)} before accessing initialized adapter state."); + internal IAmazonKinesis CreateClient() => CreateClient(_options); internal static IAmazonKinesis CreateClient(KinesisStreamOptions options) @@ -283,5 +283,9 @@ public void Dispose() _client.Dispose(); } } + + private sealed record InitializationState( + HashRingBasedPartitionedStreamQueueMapper StreamQueueMapper, + QueueAdapterReceiverRegistry Receivers); } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index ae63c92b0e4..2d5adc202b6 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -164,6 +164,25 @@ await Task.Run( Assert.All(adapters, adapter => Assert.Same(factory, adapter)); } + [Fact] + public async Task ReceiverAndCacheCreationRemainUnavailableDuringAdapterInitialization() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var factory = new BlockingKinesisAdapterFactory( + services.GetRequiredService>(), + Substitute.For()); + var initialization = factory.CreateAdapter(TestContext.Current.CancellationToken); + await factory.PartitionDiscoveryStarted.WaitAsync(TestContext.Current.CancellationToken); + var queueId = QueueId.GetQueueId("queue", 0, 0); + + Assert.Throws(() => factory.CreateReceiver(queueId)); + Assert.Throws(() => factory.CreateQueueCache(queueId)); + + factory.CompletePartitionDiscovery(); + Assert.Same(factory, await initialization); + Assert.NotNull(factory.GetStreamQueueMapper()); + } + [Fact] public async Task PooledReceiver_ReadsAndDisposesLifecycleCancellationOnShutdown() { @@ -217,7 +236,9 @@ await client.Received(1).GetRecordsAsync( Assert.Throws(() => _ = receiver.LifecycleCancellationToken); } - private sealed class BlockingKinesisAdapterFactory(Serializer serializer) + private sealed class BlockingKinesisAdapterFactory( + Serializer serializer, + IStreamQueueCheckpointerFactory? checkpointerFactory = null) : KinesisAdapterFactory( "Kinesis", new KinesisStreamOptions @@ -229,20 +250,25 @@ private sealed class BlockingKinesisAdapterFactory(Serializer Volatile.Read(ref _partitionDiscoveryCount); + public Task PartitionDiscoveryStarted => _partitionDiscoveryStarted.Task; + public void CompletePartitionDiscovery() => _partitionDiscovery.SetResult(); internal override async Task GetPartitionIdsAsync(CancellationToken cancellationToken) { Interlocked.Increment(ref _partitionDiscoveryCount); + _partitionDiscoveryStarted.TrySetResult(); await _partitionDiscovery.Task.WaitAsync(cancellationToken); return ["shard-1"]; } From f1e5a0fe82317f0f4ec0956349d609f15404999a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 19:28:03 -0700 Subject: [PATCH 48/90] style(streaming): use file-scoped namespaces Align the new checkpoint and recoverable-stream source files with the repository namespace convention. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dee65f83-7e7e-46d7-af2e-e14d0ff8cd6e --- .../IRecoverableStreamDataAdapter.cs | 51 +- .../IRecoverableStreamQueueCache.cs | 35 +- .../IRecoverableStreamSource.cs | 63 ++- .../QueueAdapterReceiverRegistry.cs | 107 ++-- .../RecoverableStreamQueueCache.cs | 495 +++++++++-------- .../RecoverableStreamReceiver.cs | 511 +++++++++--------- .../RecoverableStreamStartPosition.cs | 43 +- 7 files changed, 649 insertions(+), 656 deletions(-) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs index db6c5a351d8..790c067b95d 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs @@ -2,36 +2,35 @@ using Orleans.Runtime; using Orleans.Streams; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Adapts immutable source records to and from pooled cache storage. +/// +/// The source record type. +public interface IRecoverableStreamDataAdapter : ICacheDataAdapter { /// - /// Adapts immutable source records to and from pooled cache storage. + /// Gets the stream and provider position for a source record. /// - /// The source record type. - public interface IRecoverableStreamDataAdapter : ICacheDataAdapter - { - /// - /// Gets the stream and provider position for a source record. - /// - StreamPosition GetStreamPosition(TQueueMessage queueMessage); + StreamPosition GetStreamPosition(TQueueMessage queueMessage); - /// - /// Packs a source record into pooled cache storage. - /// - CachedMessage FromQueueMessage( - StreamPosition streamPosition, - TQueueMessage queueMessage, - DateTime dequeueTimeUtc, - Func> getSegment); + /// + /// Packs a source record into pooled cache storage. + /// + CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment); - /// - /// Gets the provider offset encoded in a cached message. - /// - string GetOffset(ref CachedMessage cachedMessage); + /// + /// Gets the provider offset encoded in a cached message. + /// + string GetOffset(ref CachedMessage cachedMessage); - /// - /// Tries to extract a provider offset from a delivery token. - /// - bool TryGetOffset(StreamSequenceToken token, out string offset); - } + /// + /// Tries to extract a provider offset from a delivery token. + /// + bool TryGetOffset(StreamSequenceToken token, out string offset); } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs index 15aa21eca78..70d8b408a2c 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs @@ -4,26 +4,25 @@ using Orleans.Runtime; using Orleans.Streams; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Cache contract used by . +/// +/// The immutable source record type. +public interface IRecoverableStreamQueueCache : IQueueCache, IDisposable { /// - /// Cache contract used by . + /// Packs and adds ordered source records to the cache. /// - /// The immutable source record type. - public interface IRecoverableStreamQueueCache : IQueueCache, IDisposable - { - /// - /// Packs and adds ordered source records to the cache. - /// - IReadOnlyList Add( - IReadOnlyList messages, - DateTime dequeueTimeUtc); + IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc); - /// - /// Tries to get the newest cached provider position. - /// - bool TryGetNewestPosition( - [NotNullWhen(true)] out StreamSequenceToken? token, - [NotNullWhen(true)] out string? offset); - } + /// + /// Tries to get the newest cached provider position. + /// + bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset); } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs index 97e4383fb8b..f740a801fd3 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs @@ -2,43 +2,42 @@ using System.Threading; using System.Threading.Tasks; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Reads ordered records from a recoverable stream partition. +/// +/// The source record type. +public interface IRecoverableStreamSource { /// - /// Reads ordered records from a recoverable stream partition. + /// Initializes the source at the requested position. /// - /// The source record type. - public interface IRecoverableStreamSource - { - /// - /// Initializes the source at the requested position. - /// - /// The durable checkpoint or configured start policy. - /// The cancellation token. - /// A durable checkpoint always takes precedence and reads must begin strictly after it. - Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken); + /// The durable checkpoint or configured start policy. + /// The cancellation token. + /// A durable checkpoint always takes precedence and reads must begin strictly after it. + Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken); - /// - /// Reads an ordered batch of immutable source records. - /// - Task> Read(int maxCount, CancellationToken cancellationToken); + /// + /// Reads an ordered batch of immutable source records. + /// + Task> Read(int maxCount, CancellationToken cancellationToken); - /// - /// Notifies the source that records were successfully admitted to the cache. - /// - /// The admitted records. - /// Sources should advance volatile read offsets only in this callback. - void MessagesAdded(IReadOnlyList messages) { } + /// + /// Notifies the source that records were successfully admitted to the cache. + /// + /// The admitted records. + /// Sources should advance volatile read offsets only in this callback. + void MessagesAdded(IReadOnlyList messages) { } - /// - /// Notifies the source that records could not be admitted to the cache. - /// - /// The records which were not admitted. - void MessagesAddFailed(IReadOnlyList messages) { } + /// + /// Notifies the source that records could not be admitted to the cache. + /// + /// The records which were not admitted. + void MessagesAddFailed(IReadOnlyList messages) { } - /// - /// Shuts the partition source down. - /// - Task Shutdown(CancellationToken cancellationToken); - } + /// + /// Shuts the partition source down. + /// + Task Shutdown(CancellationToken cancellationToken); } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs index d3e098d450e..3490c3d3c28 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -5,71 +5,70 @@ using System.Threading; using Orleans.Streams; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Ensures receiver and cache factories share one coordinator instance per queue. +/// +/// The combined receiver and cache type. +public sealed class QueueAdapterReceiverRegistry + where TReceiver : class, IQueueAdapterReceiver, IQueueCache { + private readonly ConcurrentDictionary> _receivers = new(); + private readonly Func _factory; + /// - /// Ensures receiver and cache factories share one coordinator instance per queue. + /// Initializes a new instance of the class. /// - /// The combined receiver and cache type. - public sealed class QueueAdapterReceiverRegistry - where TReceiver : class, IQueueAdapterReceiver, IQueueCache + public QueueAdapterReceiverRegistry(Func factory) { - private readonly ConcurrentDictionary> _receivers = new(); - private readonly Func _factory; + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + } - /// - /// Initializes a new instance of the class. - /// - public QueueAdapterReceiverRegistry(Func factory) + /// + /// Gets the registered receiver instances. + /// + public IReadOnlyDictionary Receivers + => _receivers + .Where(pair => pair.Value.IsValueCreated) + .ToDictionary(pair => pair.Key, pair => pair.Value.Value); + + /// + /// Gets or creates the coordinator for a queue. + /// + public TReceiver GetOrCreate(QueueId queueId) + { + var receiver = _receivers.GetOrAdd( + queueId, + static (id, factory) => new( + () => factory(id), + LazyThreadSafetyMode.ExecutionAndPublication), + _factory); + try { - _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + return receiver.Value; } - - /// - /// Gets the registered receiver instances. - /// - public IReadOnlyDictionary Receivers - => _receivers - .Where(pair => pair.Value.IsValueCreated) - .ToDictionary(pair => pair.Key, pair => pair.Value.Value); - - /// - /// Gets or creates the coordinator for a queue. - /// - public TReceiver GetOrCreate(QueueId queueId) + catch { - var receiver = _receivers.GetOrAdd( - queueId, - static (id, factory) => new( - () => factory(id), - LazyThreadSafetyMode.ExecutionAndPublication), - _factory); - try - { - return receiver.Value; - } - catch - { - ((ICollection>>)_receivers) - .Remove(new(queueId, receiver)); - throw; - } + ((ICollection>>)_receivers) + .Remove(new(queueId, receiver)); + throw; } + } - /// - /// Removes a receiver if it is still the registered instance for the queue. - /// - public bool Remove(QueueId queueId, TReceiver receiver) + /// + /// Removes a receiver if it is still the registered instance for the queue. + /// + public bool Remove(QueueId queueId, TReceiver receiver) + { + if (!_receivers.TryGetValue(queueId, out var registered) + || !registered.IsValueCreated + || !ReferenceEquals(registered.Value, receiver)) { - if (!_receivers.TryGetValue(queueId, out var registered) - || !registered.IsValueCreated - || !ReferenceEquals(registered.Value, receiver)) - { - return false; - } - - return ((ICollection>>)_receivers) - .Remove(new(queueId, registered)); + return false; } + + return ((ICollection>>)_receivers) + .Remove(new(queueId, registered)); } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 19ef94ea2d5..18e6848759a 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -5,322 +5,321 @@ using Orleans.Runtime; using Orleans.Streams; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Provider-neutral pooled cache for immutable recoverable stream records. +/// +/// The source record type. +public sealed class RecoverableStreamQueueCache : IRecoverableStreamQueueCache { + private readonly int _defaultMaxAddCount; + private readonly IObjectPool _bufferPool; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly IEvictionStrategy _evictionStrategy; + private readonly IQueueFlowController? _flowController; + private readonly int? _maxCacheSize; + private readonly PooledQueueCache _cache; + private FixedSizeBuffer? _currentBuffer; + /// - /// Provider-neutral pooled cache for immutable recoverable stream records. + /// Initializes a new instance of the class. /// - /// The source record type. - public sealed class RecoverableStreamQueueCache : IRecoverableStreamQueueCache + public RecoverableStreamQueueCache( + int defaultMaxAddCount, + IObjectPool bufferPool, + IRecoverableStreamDataAdapter dataAdapter, + IEvictionStrategy evictionStrategy, + ILogger logger, + IQueueFlowController? flowController = null, + ICacheMonitor? cacheMonitor = null, + TimeSpan? cacheMonitorWriteInterval = null, + TimeSpan? metadataMinTimeInCache = null, + int? maxCacheSize = null) { - private readonly int _defaultMaxAddCount; - private readonly IObjectPool _bufferPool; - private readonly IRecoverableStreamDataAdapter _dataAdapter; - private readonly IEvictionStrategy _evictionStrategy; - private readonly IQueueFlowController? _flowController; - private readonly int? _maxCacheSize; - private readonly PooledQueueCache _cache; - private FixedSizeBuffer? _currentBuffer; - - /// - /// Initializes a new instance of the class. - /// - public RecoverableStreamQueueCache( - int defaultMaxAddCount, - IObjectPool bufferPool, - IRecoverableStreamDataAdapter dataAdapter, - IEvictionStrategy evictionStrategy, - ILogger logger, - IQueueFlowController? flowController = null, - ICacheMonitor? cacheMonitor = null, - TimeSpan? cacheMonitorWriteInterval = null, - TimeSpan? metadataMinTimeInCache = null, - int? maxCacheSize = null) + if (defaultMaxAddCount <= 0) { - if (defaultMaxAddCount <= 0) - { - throw new ArgumentOutOfRangeException(nameof(defaultMaxAddCount)); - } - - _defaultMaxAddCount = defaultMaxAddCount; - _bufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool)); - _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); - _evictionStrategy = evictionStrategy ?? throw new ArgumentNullException(nameof(evictionStrategy)); - _flowController = flowController; - if (maxCacheSize <= 0) - { - throw new ArgumentOutOfRangeException(nameof(maxCacheSize)); - } - - _maxCacheSize = maxCacheSize; - _cache = new PooledQueueCache( - dataAdapter, - logger ?? throw new ArgumentNullException(nameof(logger)), - cacheMonitor, - cacheMonitorWriteInterval, - metadataMinTimeInCache); - _evictionStrategy.PurgeObservable = _cache; - _evictionStrategy.OnPurged = OnPurged; + throw new ArgumentOutOfRangeException(nameof(defaultMaxAddCount)); } - /// - /// Gets the most recently purged provider offset. - /// - public string? LastPurgedOffset { get; private set; } - - /// - /// Gets the number of records currently held in the cache. - /// - public int ItemCount => _cache.ItemCount; - - /// - /// Tries to get the newest cached provider position. - /// - public bool TryGetNewestPosition( - [NotNullWhen(true)] out StreamSequenceToken? token, - [NotNullWhen(true)] out string? offset) + _defaultMaxAddCount = defaultMaxAddCount; + _bufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _evictionStrategy = evictionStrategy ?? throw new ArgumentNullException(nameof(evictionStrategy)); + _flowController = flowController; + if (maxCacheSize <= 0) { - if (_cache.Newest is not { } newest) - { - token = null; - offset = null; - return false; - } + throw new ArgumentOutOfRangeException(nameof(maxCacheSize)); + } - token = _dataAdapter.GetSequenceToken(ref newest); - offset = _dataAdapter.GetOffset(ref newest); - return true; + _maxCacheSize = maxCacheSize; + _cache = new PooledQueueCache( + dataAdapter, + logger ?? throw new ArgumentNullException(nameof(logger)), + cacheMonitor, + cacheMonitorWriteInterval, + metadataMinTimeInCache); + _evictionStrategy.PurgeObservable = _cache; + _evictionStrategy.OnPurged = OnPurged; + } + + /// + /// Gets the most recently purged provider offset. + /// + public string? LastPurgedOffset { get; private set; } + + /// + /// Gets the number of records currently held in the cache. + /// + public int ItemCount => _cache.ItemCount; + + /// + /// Tries to get the newest cached provider position. + /// + public bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset) + { + if (_cache.Newest is not { } newest) + { + token = null; + offset = null; + return false; } - /// - /// Packs and adds ordered source records to the cache. - /// - public IReadOnlyList Add( - IReadOnlyList messages, - DateTime dequeueTimeUtc) + token = _dataAdapter.GetSequenceToken(ref newest); + offset = _dataAdapter.GetOffset(ref newest); + return true; + } + + /// + /// Packs and adds ordered source records to the cache. + /// + public IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc) + { + ArgumentNullException.ThrowIfNull(messages); + var positions = new List(messages.Count); + var cachedMessages = new List(messages.Count); + var allocatedBuffers = new List(); + var initialBuffer = _currentBuffer; + var initialBufferPosition = initialBuffer?.Position ?? 0; + var batchBuffer = initialBuffer; + try { - ArgumentNullException.ThrowIfNull(messages); - var positions = new List(messages.Count); - var cachedMessages = new List(messages.Count); - var allocatedBuffers = new List(); - var initialBuffer = _currentBuffer; - var initialBufferPosition = initialBuffer?.Position ?? 0; - var batchBuffer = initialBuffer; - try - { - foreach (var message in messages) - { - var position = _dataAdapter.GetStreamPosition(message); - cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); - positions.Add(position); - } - } - catch + foreach (var message in messages) { - initialBuffer?.ResetTo(initialBufferPosition); - foreach (var buffer in allocatedBuffers) - { - buffer.Dispose(); - } - - throw; + var position = _dataAdapter.GetStreamPosition(message); + cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); + positions.Add(position); } - - _cache.Add(cachedMessages, dequeueTimeUtc); + } + catch + { + initialBuffer?.ResetTo(initialBufferPosition); foreach (var buffer in allocatedBuffers) { - _evictionStrategy.OnBlockAllocated(buffer); + buffer.Dispose(); } - _currentBuffer = batchBuffer; - return positions; + throw; + } - ArraySegment GetBatchSegment(int size) - { - if (batchBuffer is not null && batchBuffer.TryGetSegment(size, out var segment)) - { - return segment; - } - - var buffer = _bufferPool.Allocate(); - if (!buffer.TryGetSegment(size, out segment)) - { - buffer.Dispose(); - buffer = new FixedSizeBuffer(size); - _ = buffer.TryGetSegment(size, out segment); - } - - allocatedBuffers.Add(buffer); - batchBuffer = buffer; - return segment; - } + _cache.Add(cachedMessages, dequeueTimeUtc); + foreach (var buffer in allocatedBuffers) + { + _evictionStrategy.OnBlockAllocated(buffer); } - /// - public int GetMaxAddCount() + _currentBuffer = batchBuffer; + return positions; + + ArraySegment GetBatchSegment(int size) { - var result = _defaultMaxAddCount; - if (_maxCacheSize is { } maxCacheSize) + if (batchBuffer is not null && batchBuffer.TryGetSegment(size, out var segment)) { - result = Math.Min(result, Math.Max(0, maxCacheSize - _cache.ItemCount)); + return segment; } - if (_flowController is not null) + var buffer = _bufferPool.Allocate(); + if (!buffer.TryGetSegment(size, out segment)) { - var flowControlLimit = _flowController.GetMaxAddCount(); - if (flowControlLimit >= 0) - { - result = Math.Min(result, flowControlLimit); - } + buffer.Dispose(); + buffer = new FixedSizeBuffer(size); + _ = buffer.TryGetSegment(size, out segment); } - return result; + allocatedBuffers.Add(buffer); + batchBuffer = buffer; + return segment; } + } - /// - public void AddToCache(IList messages) + /// + public int GetMaxAddCount() + { + var result = _defaultMaxAddCount; + if (_maxCacheSize is { } maxCacheSize) { - // Source records are admitted directly by the receiver coordinator. + result = Math.Min(result, Math.Max(0, maxCacheSize - _cache.ItemCount)); } - /// - public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + if (_flowController is not null) { - purgedItems = null; - // Pressure indicates that a lagging cursor can still need the oldest records. Time-based - // eviction in that state would turn backpressure into data loss. UpdateDeliveryProgress - // removes records through the safe delivery watermark and releases pressure as consumers advance. - if (!IsUnderPressure()) + var flowControlLimit = _flowController.GetMaxAddCount(); + if (flowControlLimit >= 0) { - _evictionStrategy.PerformPurge(DateTime.UtcNow); - if (_cache.IsEmpty) - { - _currentBuffer = null; - } + result = Math.Min(result, flowControlLimit); } - - return false; } - /// - public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) - => new Cursor(_cache, streamId, token); - - /// - public IQueueCacheCursor GetCacheCursorAtPosition( - StreamId streamId, - StreamSubscriptionStartPosition startPosition) - => new Cursor(_cache, streamId, startPosition); + return result; + } - /// - public bool IsUnderPressure() => GetMaxAddCount() <= 0; + /// + public void AddToCache(IList messages) + { + // Source records are admitted directly by the receiver coordinator. + } - /// - public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + { + purgedItems = null; + // Pressure indicates that a lagging cursor can still need the oldest records. Time-based + // eviction in that state would turn backpressure into data loss. UpdateDeliveryProgress + // removes records through the safe delivery watermark and releases pressure as consumers advance. + if (!IsUnderPressure()) { - if (earliestSubscriptionToken is null) - { - return; - } - - CachedMessage? lastPurged = null; - var itemsPurged = 0; - while (_cache.Oldest is { } oldest - && _dataAdapter.Compare(ref oldest, earliestSubscriptionToken) <= 0) - { - LastPurgedOffset = _dataAdapter.GetOffset(ref oldest); - lastPurged = oldest; - itemsPurged++; - _cache.RemoveOldestMessage(); - } - - _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + _evictionStrategy.PerformPurge(DateTime.UtcNow); if (_cache.IsEmpty) { _currentBuffer = null; } } - /// - public void Dispose() + return false; + } + + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => new Cursor(_cache, streamId, token); + + /// + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => new Cursor(_cache, streamId, startPosition); + + /// + public bool IsUnderPressure() => GetMaxAddCount() <= 0; + + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + if (earliestSubscriptionToken is null) { - CachedMessage? lastPurged = null; - var itemsPurged = 0; - while (_cache.Oldest is { } oldest) - { - lastPurged = oldest; - itemsPurged++; - _cache.RemoveOldestMessage(); - } + return; + } - _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); - _currentBuffer = null; - _evictionStrategy.OnPurged = null; + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest + && _dataAdapter.Compare(ref oldest, earliestSubscriptionToken) <= 0) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref oldest); + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); } - private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + if (_cache.IsEmpty) { - if (lastPurged is { } message) - { - LastPurgedOffset = _dataAdapter.GetOffset(ref message); - } + _currentBuffer = null; } + } - private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress + /// + public void Dispose() + { + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest) { - private readonly PooledQueueCache _cache; - private readonly object _cursor; - private IBatchContainer? _current; + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); + } - public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? token) - { - _cache = cache; - _cursor = cache.GetCursor(streamId, token); - } + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + _currentBuffer = null; + _evictionStrategy.OnPurged = null; + } - public Cursor( - PooledQueueCache cache, - StreamId streamId, - StreamSubscriptionStartPosition startPosition) - { - _cache = cache; - _cursor = cache.GetCursorAtPosition(streamId, startPosition); - } + private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) + { + if (lastPurged is { } message) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref message); + } + } - public void Dispose() - { - } + private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress + { + private readonly PooledQueueCache _cache; + private readonly object _cursor; + private IBatchContainer? _current; - public StreamSequenceToken? SafeSequenceToken => _cache.GetSafeSequenceToken(_cursor); + public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? token) + { + _cache = cache; + _cursor = cache.GetCursor(streamId, token); + } - public void SetDeliveredThrough(StreamSequenceToken token) - => _cache.SetCursorDeliveredThrough(_cursor, token); + public Cursor( + PooledQueueCache cache, + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + { + _cache = cache; + _cursor = cache.GetCursorAtPosition(streamId, startPosition); + } - public IBatchContainer? GetCurrent(out Exception? exception) - { - exception = null; - return _current; - } + public void Dispose() + { + } - public bool MoveNext() - { - if (!_cache.TryGetNextMessage(_cursor, out var next)) - { - return false; - } + public StreamSequenceToken? SafeSequenceToken => _cache.GetSafeSequenceToken(_cursor); - _current = next; - return true; - } + public void SetDeliveredThrough(StreamSequenceToken token) + => _cache.SetCursorDeliveredThrough(_cursor, token); - public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); + public IBatchContainer? GetCurrent(out Exception? exception) + { + exception = null; + return _current; + } - public void RecordDeliveryFailure() + public bool MoveNext() + { + if (!_cache.TryGetNextMessage(_cursor, out var next)) { - _cache.RecordDeliveryFailure(_cursor); + return false; } - public void RecordDeliverySuccess() => _cache.RecordDeliverySuccess(_cursor); + _current = next; + return true; } + + public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); + + public void RecordDeliveryFailure() + { + _cache.RecordDeliveryFailure(_cursor); + } + + public void RecordDeliverySuccess() => _cache.RecordDeliverySuccess(_cursor); } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 7b664a3ad95..19967372d53 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -6,327 +6,326 @@ using System.Threading.Tasks; using Orleans.Streams; -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Coordinates a recoverable stream partition pipeline comprising a partition source, pooled cache, and durable checkpoint. +/// +/// The source record type. +public sealed class RecoverableStreamReceiver : IQueueAdapterReceiver, IQueueCache { + private readonly IRecoverableStreamSource _source; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly IRecoverableStreamQueueCache _cache; + private readonly IStreamQueueCheckpointer _checkpointer; + private readonly bool _startFromNow; + private readonly object _lifecycleLock = new(); + private readonly CancellationTokenSource _lifecycleCancellation = new(); + private Task? _initializeTask; + private CancellationToken _initializeTaskOwnerToken; + private int _running; + private int _shutdown; + /// - /// Coordinates a recoverable stream partition pipeline comprising a partition source, pooled cache, and durable checkpoint. + /// Initializes a new instance of the class. /// - /// The source record type. - public sealed class RecoverableStreamReceiver : IQueueAdapterReceiver, IQueueCache + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + RecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) + : this(source, dataAdapter, (IRecoverableStreamQueueCache)cache, checkpointer, startFromNow) { - private readonly IRecoverableStreamSource _source; - private readonly IRecoverableStreamDataAdapter _dataAdapter; - private readonly IRecoverableStreamQueueCache _cache; - private readonly IStreamQueueCheckpointer _checkpointer; - private readonly bool _startFromNow; - private readonly object _lifecycleLock = new(); - private readonly CancellationTokenSource _lifecycleCancellation = new(); - private Task? _initializeTask; - private CancellationToken _initializeTaskOwnerToken; - private int _running; - private int _shutdown; - - /// - /// Initializes a new instance of the class. - /// - public RecoverableStreamReceiver( - IRecoverableStreamSource source, - IRecoverableStreamDataAdapter dataAdapter, - RecoverableStreamQueueCache cache, - IStreamQueueCheckpointer checkpointer, - bool startFromNow) - : this(source, dataAdapter, (IRecoverableStreamQueueCache)cache, checkpointer, startFromNow) - { - } + } - /// - /// Initializes a new instance of the class. - /// - public RecoverableStreamReceiver( - IRecoverableStreamSource source, - IRecoverableStreamDataAdapter dataAdapter, - IRecoverableStreamQueueCache cache, - IStreamQueueCheckpointer checkpointer, - bool startFromNow) - { - _source = source ?? throw new ArgumentNullException(nameof(source)); - _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); - _cache = cache ?? throw new ArgumentNullException(nameof(cache)); - _checkpointer = checkpointer ?? throw new ArgumentNullException(nameof(checkpointer)); - _startFromNow = startFromNow; - } + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + IRecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) + { + _source = source ?? throw new ArgumentNullException(nameof(source)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _checkpointer = checkpointer ?? throw new ArgumentNullException(nameof(checkpointer)); + _startFromNow = startFromNow; + } - /// - public async Task Initialize(TimeSpan timeout) - { - using var cancellation = timeout == Timeout.InfiniteTimeSpan - ? null - : new CancellationTokenSource(timeout); - var cancellationToken = cancellation?.Token ?? CancellationToken.None; - await EnsureInitialized(cancellationToken); - } + /// + public async Task Initialize(TimeSpan timeout) + { + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + await EnsureInitialized(cancellationToken); + } - /// - /// Initializes the receiver. - /// - /// The cancellation token. - public Task Initialize(CancellationToken cancellationToken) - => EnsureInitialized(cancellationToken); + /// + /// Initializes the receiver. + /// + /// The cancellation token. + public Task Initialize(CancellationToken cancellationToken) + => EnsureInitialized(cancellationToken); - private async Task EnsureInitialized(CancellationToken cancellationToken) + private async Task EnsureInitialized(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + while (true) { - cancellationToken.ThrowIfCancellationRequested(); - while (true) + Task initializeTask; + CancellationToken initializeTaskOwnerToken; + lock (_lifecycleLock) { - Task initializeTask; - CancellationToken initializeTaskOwnerToken; - lock (_lifecycleLock) - { - if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) - { - return; - } - - if (_initializeTask is null || _initializeTask.IsCompleted) - { - _initializeTaskOwnerToken = cancellationToken; - _initializeTask = InitializeCore(cancellationToken); - } - - initializeTask = _initializeTask; - initializeTaskOwnerToken = _initializeTaskOwnerToken; - } - - try + if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) { - await initializeTask.WaitAsync(cancellationToken); return; } - catch (OperationCanceledException) - when (!cancellationToken.IsCancellationRequested - && Volatile.Read(ref _shutdown) == 0 - && initializeTaskOwnerToken.IsCancellationRequested - && initializeTask.IsCanceled) + + if (_initializeTask is null || _initializeTask.IsCompleted) { - // The caller which started this shared initialization canceled it. Once that - // task has settled, loop and create a fresh attempt for this still-active caller. + _initializeTaskOwnerToken = cancellationToken; + _initializeTask = InitializeCore(cancellationToken); } + + initializeTask = _initializeTask; + initializeTaskOwnerToken = _initializeTaskOwnerToken; } - } - private async Task InitializeCore(CancellationToken initializationToken) - { - using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( - _lifecycleCancellation.Token, - initializationToken); - var lifecycleToken = cancellation.Token; - var checkpoint = await _checkpointer.Load(lifecycleToken); - await _source.Initialize( - new RecoverableStreamStartPosition( - _checkpointer.CheckpointExists ? checkpoint : null, - _startFromNow), - lifecycleToken); - if (Volatile.Read(ref _shutdown) != 0) + try { + await initializeTask.WaitAsync(cancellationToken); return; } + catch (OperationCanceledException) + when (!cancellationToken.IsCancellationRequested + && Volatile.Read(ref _shutdown) == 0 + && initializeTaskOwnerToken.IsCancellationRequested + && initializeTask.IsCanceled) + { + // The caller which started this shared initialization canceled it. Once that + // task has settled, loop and create a fresh attempt for this still-active caller. + } + } + } - Volatile.Write(ref _running, 1); + private async Task InitializeCore(CancellationToken initializationToken) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + _lifecycleCancellation.Token, + initializationToken); + var lifecycleToken = cancellation.Token; + var checkpoint = await _checkpointer.Load(lifecycleToken); + await _source.Initialize( + new RecoverableStreamStartPosition( + _checkpointer.CheckpointExists ? checkpoint : null, + _startFromNow), + lifecycleToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return; } - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task> GetQueueMessagesAsync(int maxCount) - => GetQueueMessagesAsync(maxCount, CancellationToken.None); + Volatile.Write(ref _running, 1); + } - /// - public async Task> GetQueueMessagesAsync( - int maxCount, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) - { - return []; - } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> GetQueueMessagesAsync(int maxCount) + => GetQueueMessagesAsync(maxCount, CancellationToken.None); - await EnsureInitialized(cancellationToken); - if (Volatile.Read(ref _shutdown) != 0) - { - return []; - } + /// + public async Task> GetQueueMessagesAsync( + int maxCount, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) + { + return []; + } - var messages = await _source.Read(maxCount, cancellationToken); - if (messages.Count == 0) - { - return []; - } + await EnsureInitialized(cancellationToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return []; + } - IReadOnlyList positions; - try - { - positions = _cache.Add(messages, DateTime.UtcNow); - _source.MessagesAdded(messages); - } - catch - { - _source.MessagesAddFailed(messages); - throw; - } + var messages = await _source.Read(maxCount, cancellationToken); + if (messages.Count == 0) + { + return []; + } - var result = new List(positions.Count); - foreach (var position in positions) - { - result.Add(new StreamActivityNotificationBatch(position)); - } + IReadOnlyList positions; + try + { + positions = _cache.Add(messages, DateTime.UtcNow); + _source.MessagesAdded(messages); + } + catch + { + _source.MessagesAddFailed(messages); + throw; + } - return result; + var result = new List(positions.Count); + foreach (var position in positions) + { + result.Add(new StreamActivityNotificationBatch(position)); } - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task MessagesDeliveredAsync(IList messages) - => Task.CompletedTask; + return result; + } - /// - public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) - => cancellationToken.IsCancellationRequested - ? Task.FromCanceled(cancellationToken) - : Task.CompletedTask; + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task MessagesDeliveredAsync(IList messages) + => Task.CompletedTask; - /// - public async Task Shutdown(TimeSpan timeout) - { - if (Interlocked.Exchange(ref _shutdown, 1) != 0) - { - return; - } + /// + public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; - Volatile.Write(ref _running, 0); - _lifecycleCancellation.Cancel(); - using var cancellation = timeout == Timeout.InfiniteTimeSpan - ? null - : new CancellationTokenSource(timeout); - var cancellationToken = cancellation?.Token ?? CancellationToken.None; - List? exceptions = null; - Task? initializeTask; - lock (_lifecycleLock) - { - initializeTask = _initializeTask; - } + /// + public async Task Shutdown(TimeSpan timeout) + { + if (Interlocked.Exchange(ref _shutdown, 1) != 0) + { + return; + } - if (initializeTask is not null) - { - try - { - await initializeTask.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) - when (_lifecycleCancellation.IsCancellationRequested - && !cancellationToken.IsCancellationRequested) - { - } - catch (Exception exception) - { - (exceptions ??= []).Add(exception); - } - } + Volatile.Write(ref _running, 0); + _lifecycleCancellation.Cancel(); + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + List? exceptions = null; + Task? initializeTask; + lock (_lifecycleLock) + { + initializeTask = _initializeTask; + } + if (initializeTask is not null) + { try { - await _checkpointer.FlushAsync(cancellationToken); - } - catch (Exception exception) - { - (exceptions ??= []).Add(exception); + await initializeTask.WaitAsync(cancellationToken); } - - try + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) { - await _source.Shutdown(cancellationToken); } catch (Exception exception) { (exceptions ??= []).Add(exception); } + } - try - { - _cache.Dispose(); - } - catch (Exception exception) - { - (exceptions ??= []).Add(exception); - } + try + { + await _checkpointer.FlushAsync(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } - if (exceptions is [var singleException]) - { - ExceptionDispatchInfo.Capture(singleException).Throw(); - } + try + { + await _source.Shutdown(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } - if (exceptions is { Count: > 1 }) - { - throw new AggregateException(exceptions); - } + try + { + _cache.Dispose(); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); } - /// - public int GetMaxAddCount() => _cache.GetMaxAddCount(); + if (exceptions is [var singleException]) + { + ExceptionDispatchInfo.Capture(singleException).Throw(); + } - /// - public void AddToCache(IList messages) + if (exceptions is { Count: > 1 }) { + throw new AggregateException(exceptions); } + } - /// - public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) - => _cache.TryPurgeFromCache(out purgedItems); + /// + public int GetMaxAddCount() => _cache.GetMaxAddCount(); - /// - public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) - => _cache.GetCacheCursor(streamId, token); + /// + public void AddToCache(IList messages) + { + } - /// - public IQueueCacheCursor GetCacheCursorAtPosition( - StreamId streamId, - StreamSubscriptionStartPosition startPosition) - => _cache.GetCacheCursorAtPosition(streamId, startPosition); + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _cache.TryPurgeFromCache(out purgedItems); - /// - public bool IsUnderPressure() => _cache.IsUnderPressure(); + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.GetCacheCursor(streamId, token); - /// - public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) - { - if (Volatile.Read(ref _shutdown) != 0) - { - return; - } + /// + public IQueueCacheCursor GetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => _cache.GetCacheCursorAtPosition(streamId, startPosition); - var progressToken = earliestSubscriptionToken; - string? offset = null; - if (progressToken is null) - { - _ = _cache.TryGetNewestPosition(out progressToken, out offset); - } + /// + public bool IsUnderPressure() => _cache.IsUnderPressure(); - _cache.UpdateDeliveryProgress(progressToken, utcNow); - if (progressToken is not null - && (offset is not null || _dataAdapter.TryGetOffset(progressToken, out offset))) - { - _checkpointer.Update(offset, utcNow, CancellationToken.None); - } + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + if (Volatile.Read(ref _shutdown) != 0) + { + return; } - private sealed class StreamActivityNotificationBatch(StreamPosition position) : IBatchContainer + var progressToken = earliestSubscriptionToken; + string? offset = null; + if (progressToken is null) { - public StreamId StreamId => position.StreamId; + _ = _cache.TryGetNewestPosition(out progressToken, out offset); + } + + _cache.UpdateDeliveryProgress(progressToken, utcNow); + if (progressToken is not null + && (offset is not null || _dataAdapter.TryGetOffset(progressToken, out offset))) + { + _checkpointer.Update(offset, utcNow, CancellationToken.None); + } + } + + private sealed class StreamActivityNotificationBatch(StreamPosition position) : IBatchContainer + { + public StreamId StreamId => position.StreamId; - public StreamSequenceToken SequenceToken => position.SequenceToken; + public StreamSequenceToken SequenceToken => position.SequenceToken; - public IEnumerable> GetEvents() => throw new NotSupportedException(); + public IEnumerable> GetEvents() => throw new NotSupportedException(); - public bool ImportRequestContext() => throw new NotSupportedException(); - } + public bool ImportRequestContext() => throw new NotSupportedException(); } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs index 0b9af8d59a7..322810d7a6a 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs @@ -1,29 +1,28 @@ -namespace Orleans.Providers.Streams.Common +namespace Orleans.Providers.Streams.Common; + +/// +/// Describes where a recoverable stream source begins reading. +/// +public readonly struct RecoverableStreamStartPosition { /// - /// Describes where a recoverable stream source begins reading. + /// Initializes a new instance of the struct. /// - public readonly struct RecoverableStreamStartPosition + /// The durable checkpoint, or if one does not exist. + /// Whether a source without a checkpoint starts at its current tail. + public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) { - /// - /// Initializes a new instance of the struct. - /// - /// The durable checkpoint, or if one does not exist. - /// Whether a source without a checkpoint starts at its current tail. - public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) - { - Checkpoint = checkpoint; - StartFromNow = startFromNow; - } + Checkpoint = checkpoint; + StartFromNow = startFromNow; + } - /// - /// Gets the durable checkpoint. Sources must begin strictly after this value. - /// - public string? Checkpoint { get; } + /// + /// Gets the durable checkpoint. Sources must begin strictly after this value. + /// + public string? Checkpoint { get; } - /// - /// Gets a value indicating whether a source without a checkpoint starts at its current tail. - /// - public bool StartFromNow { get; } - } + /// + /// Gets a value indicating whether a source without a checkpoint starts at its current tail. + /// + public bool StartFromNow { get; } } From 9dbb1cf813252bb9b07691a2c346511eeaeb9074 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:31:08 -0700 Subject: [PATCH 49/90] fix(streaming): route unavailable clients and trace cleanup --- src/Orleans.Runtime/Catalog/Catalog.cs | 2 +- .../Diagnostics/StreamingEvents.cs | 115 ++++++++++++++++ .../PersistentStreamPullingAgent.cs | 28 +++- .../ClientObserverRoutingTests.cs | 123 ++++++++++++++++++ .../StreamingTests/ClientStreamTestRunner.cs | 13 +- ...stentStreamPullingAgentDiagnosticsTests.cs | 95 ++++++++++++++ .../PersistentStreamPullingAgentTests.cs | 2 +- .../StreamingDiagnosticObserver.cs | 46 +++++++ 8 files changed, 420 insertions(+), 4 deletions(-) create mode 100644 test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index b35aa50dbd0..a50f3f1d0a1 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -140,7 +140,7 @@ internal int UnregisterGrainForTesting(GrainId grain) rehydrationContext?.Dispose(); return result; } - else if (grainId.IsSystemTarget()) + else if (grainId.IsSystemTarget() || grainId.IsClient()) { rehydrationContext?.Dispose(); return null; diff --git a/src/Orleans.Streaming/Diagnostics/StreamingEvents.cs b/src/Orleans.Streaming/Diagnostics/StreamingEvents.cs index 8034ecc2402..dbe735f6cc5 100644 --- a/src/Orleans.Streaming/Diagnostics/StreamingEvents.cs +++ b/src/Orleans.Streaming/Diagnostics/StreamingEvents.cs @@ -87,6 +87,81 @@ public sealed class MessageDelivered( public readonly IBatchContainer Batch = batch; } + /// + /// Event payload for a failed delivery attempt to a stream consumer. + /// + /// The stream provider. + /// The stream identifier. + /// The subscription identifier. + /// The delivering silo. + /// The consumer endpoint. + /// The attempted batch position. + /// The delivery failure. + public sealed class MessageDeliveryFailed( + string streamProvider, + StreamId streamId, + Guid subscriptionId, + SiloAddress? siloAddress, + IAddressable consumer, + StreamSequenceToken sequenceToken, + Exception exception) : StreamingEvent(streamProvider, siloAddress) + { + /// The stream identifier. + public readonly StreamId StreamId = streamId; + /// The subscription identifier. + public readonly Guid SubscriptionId = subscriptionId; + /// The consumer endpoint. + public readonly IAddressable Consumer = consumer; + /// The attempted batch position. + public readonly StreamSequenceToken SequenceToken = sequenceToken; + /// The delivery failure. + public readonly Exception Exception = exception; + } + + /// + /// Identifies the stage of an unavailable consumer's unregistration request. + /// + public enum SubscriptionUnregistrationStage + { + /// The pulling agent is requesting unregistration. + Requested, + /// The pubsub unregistration call completed successfully. + Completed, + /// The pubsub unregistration call failed. + Failed, + } + + /// + /// Reports the outcome of an unavailable consumer's unregistration request. + /// + /// The stream provider. + /// The stream identifier. + /// The subscription identifier. + /// The requesting silo. + /// The consumer endpoint. + /// The request stage. + /// The failure when the request failed. + public sealed class SubscriptionUnregistration( + string streamProvider, + StreamId streamId, + Guid subscriptionId, + SiloAddress? siloAddress, + IAddressable consumer, + SubscriptionUnregistrationStage stage, + Exception? exception) : StreamingEvent(streamProvider, siloAddress) + { + /// The stream identifier. + public readonly StreamId StreamId = streamId; + /// The subscription identifier. + public readonly Guid SubscriptionId = subscriptionId; + /// The consumer endpoint. + public readonly IAddressable Consumer = consumer; + /// The request stage. + public readonly SubscriptionUnregistrationStage Stage = stage; + /// The failure when the request failed. + public readonly Exception? Exception = exception; + } + /// /// Event payload for when a stream becomes inactive due to no activity. /// @@ -799,6 +874,46 @@ static void Emit(string streamProviderName, StreamConsumerData consumerData, IBa } } + internal static void EmitMessageDeliveryFailed( + string streamProviderName, + StreamConsumerData consumerData, + StreamSequenceToken sequenceToken, + SiloAddress? siloAddress, + Exception exception) + { + if (Listener.IsEnabled(nameof(MessageDeliveryFailed))) + { + Listener.Write(nameof(MessageDeliveryFailed), new MessageDeliveryFailed( + streamProviderName, + consumerData.StreamId.StreamId, + consumerData.SubscriptionId.Guid, + siloAddress, + consumerData.StreamConsumer, + sequenceToken, + exception)); + } + } + + internal static void EmitSubscriptionUnregistration( + string streamProviderName, + StreamConsumerData consumerData, + SiloAddress? siloAddress, + SubscriptionUnregistrationStage stage, + Exception? exception = null) + { + if (Listener.IsEnabled(nameof(SubscriptionUnregistration))) + { + Listener.Write(nameof(SubscriptionUnregistration), new SubscriptionUnregistration( + streamProviderName, + consumerData.StreamId.StreamId, + consumerData.SubscriptionId.Guid, + siloAddress, + consumerData.StreamConsumer, + stage, + exception)); + } + } + internal static void EmitStreamInactive(string streamProviderName, StreamId streamId, TimeSpan inactivityPeriod, SiloAddress? siloAddress) { if (!Listener.IsEnabled(nameof(StreamInactive))) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 17763fb0715..d92b827d4ff 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -1781,6 +1781,7 @@ private sealed class QueueCacheCursorContractException(string message) : Invalid } catch (Exception ex) { + StreamingEvents.EmitMessageDeliveryFailed(streamProviderName, consumerData, batch.SequenceToken, Silo, ex); LogWarningFailedToDeliverMessage(consumerData.SubscriptionId, consumerData.StreamId, ex); throw; } @@ -1854,7 +1855,7 @@ private async Task ErrorProtocol( if (exceptionOccured is ClientNotAvailableException) { LogWarningConsumerIsDead(consumerData.StreamConsumer, consumerData.StreamId); - await pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId, CancellationToken.None); + UnregisterUnavailableConsumer(consumerData, cancellationToken).Ignore(); return true; } @@ -1910,6 +1911,25 @@ bool IsCurrent() => operationId == (isDeliveryError ? consumerData.HandshakeGeneration : consumerData.HandshakeRequestId); } + private async Task UnregisterUnavailableConsumer(StreamConsumerData consumerData, CancellationToken cancellationToken) + { + StreamingEvents.EmitSubscriptionUnregistration( + streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Requested); + try + { + await pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId, cancellationToken); + StreamingEvents.EmitSubscriptionUnregistration( + streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Completed); + } + catch (Exception exception) + { + StreamingEvents.EmitSubscriptionUnregistration( + streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Failed, exception); + LogWarningUnregisterUnavailableConsumer(consumerData.SubscriptionId, consumerData.StreamId, exception); + throw; + } + } + private static async Task> PubsubRegisterProducer( IStreamPubSub pubSub, QualifiedStreamId streamId, @@ -2130,6 +2150,12 @@ private readonly struct QueueIdLogRecord(QueueId queueId) )] private partial void LogWarningConsumerIsDead(IStreamConsumerExtension consumer, QualifiedStreamId streamId); + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Failed to unregister unavailable consumer subscription {SubscriptionId} for stream {StreamId}." + )] + private partial void LogWarningUnregisterUnavailableConsumer(GuidId subscriptionId, QualifiedStreamId streamId, Exception exception); + [LoggerMessage( Level = LogLevel.Error, EventId = (int)ErrorCode.PersistentStreamPullingAgent_17, diff --git a/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs new file mode 100644 index 00000000000..50673592ca4 --- /dev/null +++ b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs @@ -0,0 +1,123 @@ +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Orleans; +using Orleans.Runtime; +using Orleans.Runtime.GrainDirectory; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace Tester.ClientConnectionTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +[TestCategory("BVT")] +public class ClientObserverRoutingTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StaleClientRouteIsResolvedThroughClientDirectory(bool unavailable) + { + var builder = new InProcessTestClusterBuilder(2); + builder.ConfigureHost(hostBuilder => TestDefaultConfiguration.ConfigureHostConfiguration(hostBuilder.Configuration)); + builder.ConfigureSilo((_, siloBuilder) => siloBuilder.ConfigureServices(services => + { + services.AddSingleton(); + services.AddSingleton(services => services.GetRequiredService()); + })); + await using var cluster = builder.Build(); + await cluster.DeployAsync(TestContext.Current.CancellationToken); + + var sender = cluster.Silos[0].ServiceProvider; + var receiver = cluster.Silos[1].ServiceProvider; + var observer = new EchoObserver(); + var target = unavailable + ? ObserverGrainId.Create(ClientGrainId.Create()).GrainId + : receiver.GetRequiredService().CreateObjectReference(observer).GetGrainId(); + Assert.True(ClientGrainId.TryParse(target, out var clientId)); + var directory = sender.GetRequiredService(); + directory.Arm(clientId.GrainId, cluster.Silos[0].SiloAddress); + var reference = sender.GetRequiredService().GetGrain(target); + + if (unavailable) + { + await Assert.ThrowsAsync( + () => reference.Echo(42).WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken)); + Assert.Equal(0, observer.CallCount); + } + else + { + Assert.Equal(42, await reference.Echo(42).WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken)); + Assert.Equal(1, observer.CallCount); + } + + Assert.Equal(1, directory.StaleRoutesReturned); + Assert.Null(sender.GetRequiredService().FindTarget(target)); + Assert.Null(receiver.GetRequiredService().FindTarget(target)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UnavailableClientReturnsNoLocalActivation(bool observer) + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureHost(hostBuilder => TestDefaultConfiguration.ConfigureHostConfiguration(hostBuilder.Configuration)); + await using var cluster = builder.Build(); + await cluster.DeployAsync(TestContext.Current.CancellationToken); + + var clientId = ClientGrainId.Create(); + var grainId = observer ? ObserverGrainId.Create(clientId).GrainId : clientId.GrainId; + var catalog = cluster.Silos[0].ServiceProvider.GetRequiredService(); + + // An absent local client route is handled by client placement, which can + // find another gateway or report ClientNotAvailableException to the caller. + Assert.Null(catalog.GetOrCreateActivation(grainId, requestContextData: null, rehydrationContext: null)); + Assert.Null(cluster.Silos[0].ServiceProvider.GetRequiredService().FindTarget(grainId)); + } + + private sealed class EchoObserver : IClientRoutingObserver + { + public int CallCount { get; private set; } + + public Task Echo(int value) + { + CallCount++; + return Task.FromResult(value); + } + } + + private sealed class RedirectOnceClientDirectory(ClientDirectory inner) : ILocalClientDirectory + { + private GrainId _clientId; + private GrainAddress? _staleAddress; + public int StaleRoutesReturned { get; private set; } + + public void Arm(GrainId clientId, SiloAddress siloAddress) + { + _clientId = clientId; + _staleAddress = GrainAddress.GetAddress(siloAddress, clientId, ActivationId.NewId()); + } + + public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List? addresses) + { + if (grainId == _clientId && Interlocked.Exchange(ref _staleAddress, null) is { } stale) + { + StaleRoutesReturned++; + addresses = [stale]; + return true; + } + + return inner.TryLocalLookup(grainId, out addresses); + } + + public ValueTask> Lookup(GrainId grainId) => inner.Lookup(grainId); + } +} + +public interface IClientRoutingObserver : IGrainObserver +{ + Task Echo(int value); +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs index 0238bb81039..d38ea7e60c1 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs @@ -76,7 +76,18 @@ public async Task StreamConsumerOnDroppedClientTest( { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_timeout); - await streamingObserver.WaitForSubscriptionUnregisteredAsync(streamId, droppedSubscriptionId, streamProviderName, cts.Token); + try + { + await streamingObserver.WaitForSubscriptionUnregisteredAsync(streamId, droppedSubscriptionId, streamProviderName, cts.Token); + } + catch (OperationCanceledException exception) when (!cancellationToken.IsCancellationRequested && cts.IsCancellationRequested) + { + var diagnostics = streamingObserver.GetSubscriptionDiagnostics(streamId, droppedSubscriptionId, streamProviderName); + var message = $"Timed out unregistering dropped subscription {droppedSubscriptionId} on {streamProviderName}/{streamId}. " + + $"Dropped clients: {string.Join(", ", droppedClients)}.{Environment.NewLine}{diagnostics}"; + output?.WriteLine(message); + throw new TimeoutException(message, exception); + } } if (getDeliveryFailureCount is not null) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs new file mode 100644 index 00000000000..7a79a6adb95 --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -0,0 +1,95 @@ +using System.Collections.Concurrent; +using System.Reactive.Linq; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streaming.Diagnostics; +using Orleans.Streams; +using Xunit; + +namespace UnitTests.StreamingTests; + +public partial class PersistentStreamPullingAgentTests +{ + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlockingDelivery(bool failUnregistration) + { + var streamId = new QualifiedStreamId("provider", StreamId.Create("unregister", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var token = new EventSequenceTokenV2(1); + var cache = new ScriptedQueueCache(); + cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); + var (accessor, pubSub, streamData) = await CreateInitializedAgentWithStream( + streamId, token, cache, new StreamPullingAgentOptions()); + var unavailable = new ClientNotAvailableException(ClientGrainId.Create().GrainId); + var consumer = Substitute.For(); + consumer.DeliverBatch(default!, default, default!, default, default) + .ReturnsForAnyArgs(Task.FromException(unavailable)); + var data = streamData.AddConsumer(subscriptionId, streamId, consumer, filterData: null, DateTime.UtcNow); + data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); + data.IsRegistered = true; + var unregister = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(unregister.Task); + var events = new ConcurrentQueue(); + var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var observer = StreamingEvents.AllEvents.Subscribe(value => + { + if (value is StreamingEvents.MessageDeliveryFailed failed && failed.SubscriptionId == subscriptionId.Guid) + { + events.Enqueue(value); + } + else if (value is StreamingEvents.SubscriptionUnregistration registration && registration.SubscriptionId == subscriptionId.Guid) + { + events.Enqueue(value); + if (registration.Stage != StreamingEvents.SubscriptionUnregistrationStage.Requested) + { + outcome.TrySetResult(registration); + } + } + }); + + try + { + await accessor.RunConsumerCursor(data).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + var failed = Assert.Single(events.OfType()); + Assert.Same(unavailable, failed.Exception); + Assert.Same(consumer, failed.Consumer); + Assert.Equal(token, failed.SequenceToken); + Assert.Equal(streamId.StreamId, failed.StreamId); + Assert.Equal(StreamingEvents.SubscriptionUnregistrationStage.Requested, + Assert.Single(events.OfType()).Stage); + Assert.False(outcome.Task.IsCompleted); + + var exception = new InvalidOperationException("unregistration storage failure"); + if (failUnregistration) + { + unregister.SetException(exception); + } + else + { + unregister.SetResult(); + } + + var completed = await outcome.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal( + failUnregistration ? StreamingEvents.SubscriptionUnregistrationStage.Failed : StreamingEvents.SubscriptionUnregistrationStage.Completed, + completed.Stage); + Assert.Equal(streamId.StreamId, completed.StreamId); + Assert.Same(consumer, completed.Consumer); + Assert.Same(failUnregistration ? exception : null, completed.Exception); + _ = pubSub.Received(1).UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + } + finally + { + unregister.TrySetResult(); + await accessor.Shutdown(); + } + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 09f44a68988..3cd93337be9 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -21,7 +21,7 @@ namespace UnitTests.StreamingTests { - public class PersistentStreamPullingAgentTests + public partial class PersistentStreamPullingAgentTests { [TestSuite("BVT")] [TestProvider("None")] diff --git a/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs b/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs index b701811d7d0..3384e0610e7 100644 --- a/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs +++ b/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs @@ -408,6 +408,52 @@ await _events .ConfigureAwait(false); } + /// + /// Summarizes the most recent lifecycle events for one subscription. + /// + public string GetSubscriptionDiagnostics(StreamId streamId, Guid subscriptionId, string streamProvider) + { + var recent = new Queue(); + using var subscription = _events.Subscribe(value => + { + var entry = value switch + { + StreamingEvents.MessageDeliveryFailed failed + when MatchesSubscription(failed.StreamId, failed.SubscriptionId, failed.StreamProvider, streamId, subscriptionId, streamProvider) + => $"Delivery failed on {failed.SiloAddress} to {failed.Consumer} at {failed.SequenceToken}: {failed.Exception}", + StreamingEvents.SubscriptionUnregistration unregister + when MatchesSubscription(unregister.StreamId, unregister.SubscriptionId, unregister.StreamProvider, streamId, subscriptionId, streamProvider) + => $"Unregistration {unregister.Stage} on {unregister.SiloAddress} for {unregister.Consumer}: {unregister.Exception}", + StreamingEvents.SubscriptionUnregistered unregistered + when MatchesSubscription(unregistered.StreamId, unregistered.SubscriptionId, unregistered.StreamProvider, streamId, subscriptionId, streamProvider) + => $"Durably unregistered on {unregistered.SiloAddress}", + StreamingEvents.SubscriptionAttached attached + when MatchesSubscription(attached.StreamId, attached.SubscriptionId, attached.StreamProvider, streamId, subscriptionId, streamProvider) + => $"Attached on {attached.SiloAddress} to {attached.ConsumerGrainId}", + StreamingEvents.ConsumerCursorDrained drained + when MatchesSubscription(drained.StreamId, drained.SubscriptionId, drained.StreamProvider, streamId, subscriptionId, streamProvider) + => $"Cursor drained on {drained.SiloAddress}", + _ => null, + }; + if (entry is not null) + { + lock (recent) + { + if (recent.Count == 16) + { + recent.Dequeue(); + } + + recent.Enqueue(entry); + } + } + }); + lock (recent) + { + return recent.Count == 0 ? "No matching lifecycle events were observed." : string.Join(Environment.NewLine, recent); + } + } + /// /// Waits for a specific number of subscriptions to be durably removed from a stream. /// From 90b8ab9bb86790d008efdd766f03d90cdc970bc7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:28:30 -0700 Subject: [PATCH 50/90] fix(streaming): advance bounded recovery cursors safely --- .../Common/PooledCache/PooledQueueCache.cs | 19 +- .../Streams/PooledQueueCacheTests.cs | 52 ++++ ...rsistentStreamPullingAgentRecoveryTests.cs | 282 ++++++++++++++++++ .../PersistentStreamPullingAgentTests.cs | 7 +- .../RecoverableStreamReceiverTests.cs | 129 +++++++- 5 files changed, 475 insertions(+), 14 deletions(-) create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index bcaf86bd35a..7c06bda23e0 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -301,10 +301,12 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) return null; } - // If sequenceToken is too new to be in cache, unset token, and wait for more data. + // The retained partition prefix precedes the requested start, so it is safe to scan + // while keeping the inclusive target pending for a later read. CachedMessage newestMessage = newestBlock.Value.NewestMessage; if (cacheDataAdapter.Compare(ref newestMessage, sequenceToken) < 0) { + cursor.RecordScanned(cacheDataAdapter.GetSequenceToken(ref newestMessage)); cursor.State = CursorStates.Unset; cursor.SequenceToken = sequenceToken; return null; @@ -405,17 +407,18 @@ private bool TryGetCacheMiss( private void SetCursorAtEarliestAvailable(Cursor cursor) { - if (messageBlocks.Last is { } oldestBlock) + if (IsEmpty) { - cursor.State = CursorStates.EarliestAvailableSet; - cursor.CurrentBlock = oldestBlock; - cursor.Index = oldestBlock.Value.OldestMessageIndex; - cursor.SequenceToken = oldestBlock.Value.GetOldestSequenceToken(cacheDataAdapter); - cursor.BlockGeneration = oldestBlock.Value.Generation; + SetWaitingAtCurrentEnd(cursor); return; } - SetWaitingAtCurrentEnd(cursor); + var oldestBlock = messageBlocks.Last!; + cursor.State = CursorStates.EarliestAvailableSet; + cursor.CurrentBlock = oldestBlock; + cursor.Index = oldestBlock.Value.OldestMessageIndex; + cursor.SequenceToken = oldestBlock.Value.GetOldestSequenceToken(cacheDataAdapter); + cursor.BlockGeneration = oldestBlock.Value.Generation; } private void SetWaitingAtCurrentEnd(Cursor cursor) diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs index 7c96353c4ff..4915bf77822 100644 --- a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs @@ -344,6 +344,58 @@ public void EarliestAvailableCursorRecoversWhenInitialMessageIsPurged() Assert.Equal(2, message.SequenceToken.SequenceNumber); } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(1)] + [InlineData(16 * 1024)] + public void EarliestAvailableCursorCreatedAfterDrainingBlock_WaitsForFirstMatchingMessage(int messageCount) + { + var (cache, converter) = CreateCache(); + var targetStream = StreamId.Create(TestStreamNamespace, Guid.NewGuid()); + var otherStream = StreamId.Create(TestStreamNamespace, Guid.NewGuid()); + var now = DateTime.UnixEpoch; + var messages = Enumerable.Range(1, messageCount) + .Select(sequence => converter.ToCachedMessage( + new TestQueueMessage { StreamId = targetStream, SequenceNumber = sequence }, now)) + .ToList(); + cache.Add(messages, now); + long sequenceNumber = messageCount; + byte[] payload = [1, 2, 3]; + + for (var cycle = 0; cycle < 2; cycle++) + { + while (!cache.IsEmpty) + { + cache.RemoveOldestMessage(); + } + + Assert.Equal(0, cache.ItemCount); + Assert.Null(cache.Oldest); + Assert.Null(cache.Newest); + var cursor = cache.GetCursorAtPosition(targetStream, StreamSubscriptionStartPosition.EarliestAvailable); + Assert.False(cache.TryGetNextMessage(cursor, out _)); + + cache.Add( + [ + converter.ToCachedMessage( + new TestQueueMessage { StreamId = otherStream, SequenceNumber = ++sequenceNumber }, now), + ], now); + Assert.False(cache.TryGetNextMessage(cursor, out _)); + + cache.Add( + [ + converter.ToCachedMessage( + new TestQueueMessage { StreamId = targetStream, SequenceNumber = ++sequenceNumber, Data = payload }, now), + ], now); + + Assert.True(cache.TryGetNextMessage(cursor, out var message)); + var batch = Assert.IsType(message); + Assert.Equal(targetStream, batch.StreamId); + Assert.Equal(sequenceNumber, batch.SequenceToken.SequenceNumber); + Assert.Equal(payload, batch.Data); + Assert.False(cache.TryGetNextMessage(cursor, out _)); + } + } + [Fact, TestCategory("BVT"), TestCategory("Streaming")] public void EarliestAvailableWaitingCursorSurvivesAnchorPurge() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs new file mode 100644 index 00000000000..04fd4332669 --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs @@ -0,0 +1,282 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streaming.Diagnostics; +using Orleans.Streams; +using Orleans.TestingHost.Diagnostics; +using TestExtensions; +using Xunit; +using RecoveryBatch = UnitTests.StreamingTests.RecoverableStreamReceiverTests.TestBatchContainer; +using RecoveryMessage = UnitTests.StreamingTests.RecoverableStreamReceiverTests.TestQueueMessage; + +namespace UnitTests.StreamingTests; + +public partial class PersistentStreamPullingAgentTests +{ + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task FutureStartToken_BeyondFiniteCapacity_PreservesDeliveryBarriers(bool implicitSubscription) + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeout = TimeSpan.FromSeconds(10); + var timeProvider = new FakeTimeProvider(DateTimeOffset.UnixEpoch); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var stream = StreamId.Create("namespace", Guid.NewGuid()); + var otherStream = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStream = new QualifiedStreamId("provider", stream); + var qualifiedOtherStream = new QualifiedStreamId("provider", otherStream); + var targetSubscription = GuidId.GetGuidId(implicitSubscription + ? SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid()) + : SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var slowSubscription = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var source = new BoundedRecoverySource( + Enumerable.Range(1, 9) + .Select(sequence => new RecoveryMessage( + sequence is 2 or 4 or 6 or 9 ? otherStream : stream, + sequence, + $"payload-{sequence}")) + .ToList()); + var adapter = new RecoverableStreamReceiverTests.TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 3, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new RecoverableStreamReceiverTests.NoOpEvictionStrategy(), + NullLogger.Instance, + maxCacheSize: 3); + var checkpoints = new List(); + var checkpointer = Substitute.For>(); + checkpointer.CheckpointExists.Returns(true); + checkpointer.Load(Arg.Any()).Returns(Task.FromResult("0")); + checkpointer.When(value => value.Update( + Arg.Any(), Arg.Any(), Arg.Any())) + .Do(call => checkpoints.Add(long.Parse(call.Arg(), CultureInfo.InvariantCulture))); + var receiver = new RecoverableStreamReceiver( + source, adapter, cache, checkpointer, startFromNow: false); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(queueId).Returns(receiver); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, cancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + var firstDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retryDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retryResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var deliveredTokens = new List(); + var attempts = 0; + var firstExpectedToken = implicitSubscription ? 8 : 7; + var targetConsumer = Substitute.For(); + targetConsumer.GetSequenceToken(targetSubscription, Arg.Any()) + .Returns(Task.FromResult(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(7)))); + targetConsumer.DeliverBatch( + targetSubscription, qualifiedStream, Arg.Any(), + Arg.Any(), Arg.Any()) + .Returns(call => + { + var batch = call.Arg(); + deliveredTokens.Add(batch.SequenceToken.SequenceNumber); + if (batch.SequenceToken.SequenceNumber != firstExpectedToken) + { + return Task.FromResult(null); + } + + if (++attempts == 1) + { + firstDelivery.SetResult(batch); + return firstResult.Task; + } + + retryDelivery.SetResult(batch); + return retryResult.Task; + }); + var slowConsumer = new RecordingConsumer(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(5))); + using var diagnostics = new DiagnosticEventCollector(StreamingEvents.ListenerName); + + await InitializeAgent(agent); + try + { + Assert.Equal("0", source.StartPosition.Checkpoint); + await accessor.RegisterStream(qualifiedStream, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + await accessor.RegisterStream(qualifiedOtherStream, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await accessor.GetPubSubCache())[qualifiedStream]; + var targetData = streamData.AddConsumer( + targetSubscription, qualifiedStream, targetConsumer, filterData: null, timeProvider.GetUtcNow().UtcDateTime); + var slowData = streamData.AddConsumer( + slowSubscription, qualifiedStream, slowConsumer, filterData: null, timeProvider.GetUtcNow().UtcDateTime); + Assert.True(await accessor.DoHandshakeWithConsumer(targetData, new EventSequenceTokenV2(1))); + Assert.True(await accessor.DoHandshakeWithConsumer(slowData, new EventSequenceTokenV2(1))); + targetData.IsRegistered = true; + slowData.IsRegistered = true; + + await UpdateProgress(); + Assert.Empty(checkpoints); + await accessor.RunQueuePump(queueId, cancellationToken); + await accessor.GetPubSubCache(); + Assert.Equal(1, source.ReadCount); + Assert.Equal(3, source.AdmittedThrough); + Assert.Equal(3, cache.ItemCount); + Assert.Equal(0, receiver.GetMaxAddCount()); + Assert.Equal(3, targetData.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(3, slowData.LastSafePartitionToken?.SequenceNumber); + Assert.Empty(deliveredTokens); + Assert.Empty(slowConsumer.DeliveredTokens); + + await UpdateProgress(); + Assert.Equal(3, Assert.Single(checkpoints)); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(3, receiver.GetMaxAddCount()); + + await accessor.RunQueuePump(queueId, cancellationToken); + await WaitForPhase(slowConsumer.Delivered.Task, "slow consumer receives record 5"); + await accessor.GetPubSubCache(); + Assert.Equal(2, source.ReadCount); + Assert.Equal(6, source.AdmittedThrough); + Assert.Equal(5, Assert.Single(slowConsumer.DeliveredTokens).SequenceNumber); + Assert.Equal(6, targetData.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(3, slowData.LastSafePartitionToken?.SequenceNumber); + Assert.Empty(deliveredTokens); + + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3 }, checkpoints); + Assert.Equal(3, cache.ItemCount); + Assert.True(receiver.IsUnderPressure()); + await accessor.RunQueuePump(queueId, cancellationToken); + Assert.Equal(2, source.ReadCount); + + var slowDrained = WaitForDrain(slowSubscription); + slowConsumer.ReleaseDelivery(); + await slowDrained; + await accessor.GetPubSubCache(); + Assert.Equal(6, slowData.LastSafePartitionToken?.SequenceNumber); + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 6 }, checkpoints); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(3, receiver.GetMaxAddCount()); + + var targetDrained = WaitForDrain(targetSubscription); + await accessor.RunQueuePump(queueId, cancellationToken); + var pendingBatch = Assert.IsType( + await WaitForPhase(firstDelivery.Task, $"recovery consumer receives record {firstExpectedToken}")); + await accessor.GetPubSubCache(); + Assert.Equal(firstExpectedToken, pendingBatch.SequenceToken.SequenceNumber); + Assert.Equal($"payload-{firstExpectedToken}", pendingBatch.Payload); + Assert.Equal(3, source.ReadCount); + Assert.Equal(9, source.AdmittedThrough); + Assert.Equal(9, slowData.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(new long[] { 5, 7, 8 }, slowConsumer.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.Null(targetData.LastProcessedToken); + var safeBeforeDelivery = implicitSubscription ? 7 : 6; + Assert.Equal(safeBeforeDelivery, targetData.LastSafePartitionToken?.SequenceNumber); + + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 6, safeBeforeDelivery }, checkpoints); + Assert.Equal(9 - safeBeforeDelivery, cache.ItemCount); + Assert.Equal(safeBeforeDelivery.ToString(CultureInfo.InvariantCulture), cache.LastPurgedOffset); + + firstResult.SetException(new InvalidOperationException("Retry the pending recovery delivery.")); + var retriedBatch = await WaitForPhase(retryDelivery.Task, $"recovery consumer retries record {firstExpectedToken}"); + Assert.Same(pendingBatch, retriedBatch); + await accessor.GetPubSubCache(); + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 6, safeBeforeDelivery, safeBeforeDelivery }, checkpoints); + Assert.Equal(9 - safeBeforeDelivery, cache.ItemCount); + Assert.Null(targetData.LastProcessedToken); + + retryResult.SetResult(null); + await targetDrained; + await accessor.GetPubSubCache(); + Assert.Equal(8, targetData.LastProcessedToken?.SequenceNumber); + Assert.Equal(9, targetData.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(implicitSubscription ? new long[] { 8, 8 } : new long[] { 7, 7, 8 }, deliveredTokens); + Assert.Empty(slowConsumer.Errors); + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 6, safeBeforeDelivery, safeBeforeDelivery, 9 }, checkpoints); + Assert.Equal("9", cache.LastPurgedOffset); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(3, receiver.GetMaxAddCount()); + } + finally + { + firstResult.TrySetResult(null); + retryResult.TrySetResult(null); + slowConsumer.ReleaseDelivery(); + await accessor.Shutdown(); + } + + Task WaitForDrain(GuidId subscription) + => WaitForPhase( + diagnostics.WaitForEventAsync( + nameof(StreamingEvents.ConsumerCursorDrained), + entry => entry.Payload is StreamingEvents.ConsumerCursorDrained drained + && drained.StreamId == stream && drained.SubscriptionId == subscription.Guid, + timeout, + cancellationToken), + $"subscription {subscription} drains"); + + async Task WaitForPhase(Task task, string phase) + { + try + { + return await task.WaitAsync(timeout, cancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Timed out waiting for {phase} on {qualifiedStream}; implicit={implicitSubscription}, " + + $"reads={source.ReadCount}, admitted={source.AdmittedThrough}, cached={cache.ItemCount}, " + + $"checkpoints=[{string.Join(", ", checkpoints)}].", + exception); + } + } + + async Task UpdateProgress() + { + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await accessor.GetPubSubCache(); + } + } + + private sealed class BoundedRecoverySource(IReadOnlyList messages) + : IRecoverableStreamSource + { + public RecoverableStreamStartPosition StartPosition { get; private set; } + public long AdmittedThrough { get; private set; } + public int ReadCount { get; private set; } + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + StartPosition = position; + AdmittedThrough = long.Parse(position.Checkpoint!, CultureInfo.InvariantCulture); + return Task.CompletedTask; + } + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ReadCount++; + return Task.FromResult>( + messages.Where(message => message.SequenceNumber > AdmittedThrough).Take(maxCount).ToList()); + } + + public void MessagesAdded(IReadOnlyList admitted) + => AdmittedThrough = admitted[^1].SequenceNumber; + + public Task Shutdown(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 3cd93337be9..a3122fd3bc7 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -528,8 +528,11 @@ private static PersistentStreamPullingAgent CreateAgent( messagingInstruments: CreateMessagingInstruments(), messagingProcessingInstruments: CreateMessagingProcessingInstruments()); - receiver ??= Substitute.For(); - receiver.Initialize(Arg.Any()).Returns(Task.CompletedTask); + if (receiver is null) + { + receiver = Substitute.For(); + receiver.Initialize(Arg.Any()).Returns(Task.CompletedTask); + } var queueAdapter = Substitute.For(); queueAdapter.Name.Returns("provider"); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 2147c856235..ef5f162e167 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -609,6 +609,127 @@ public void Cache_BatchedMatchesRemainPendingUntilWholeDeliverySucceeds() Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Cache_FutureStartAcrossCapacity_PreservesInclusiveTargetDuringPurge(bool createCursorBeforeFirstRead) + { + var stream = StreamId.Create("namespace", Guid.NewGuid()); + var otherStream = StreamId.Create("namespace", Guid.NewGuid()); + using var cache = new RecoverableStreamQueueCache( + 3, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance, + maxCacheSize: 3); + var target = new EventSequenceTokenV2(7); + var cursor = createCursorBeforeFirstRead ? cache.GetCacheCursor(stream, target) : null; + if (cursor is not null) + { + Assert.False(cursor.MoveNext()); + Assert.Null(Assert.IsAssignableFrom(cursor).SafeSequenceToken); + } + + for (var first = 1; first <= 4; first += 3) + { + cache.Add( + Enumerable.Range(first, 3) + .Select(sequence => new TestQueueMessage( + sequence % 2 == 0 ? otherStream : stream, sequence, $"payload-{sequence}")) + .ToList(), + DateTime.UnixEpoch); + cursor ??= cache.GetCacheCursor(stream, target); + Assert.True(cache.IsUnderPressure()); + Assert.Equal(0, cache.GetMaxAddCount()); + Assert.False(cursor.MoveNext()); + var safeToken = Assert.IsAssignableFrom(cursor).SafeSequenceToken; + Assert.NotNull(safeToken); + Assert.Equal(first + 2, safeToken.SequenceNumber); + + cache.UpdateDeliveryProgress(safeToken, DateTime.UnixEpoch); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(3, cache.GetMaxAddCount()); + Assert.Equal((first + 2).ToString(CultureInfo.InvariantCulture), cache.LastPurgedOffset); + } + + using var targetCursor = cursor!; + var progress = Assert.IsAssignableFrom(targetCursor); + cache.Add( + [ + new TestQueueMessage(stream, 7, "target"), + new TestQueueMessage(otherStream, 8, "other"), + new TestQueueMessage(stream, 9, "next"), + ], DateTime.UnixEpoch); + Assert.True(targetCursor.MoveNext()); + Assert.Equal("target", Assert.IsType(targetCursor.GetCurrent(out _)).Payload); + Assert.True(targetCursor.MoveNext()); + Assert.Equal("next", Assert.IsType(targetCursor.GetCurrent(out _)).Payload); + Assert.Equal(6, progress.SafeSequenceToken?.SequenceNumber); + cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); + Assert.Equal(3, cache.ItemCount); + Assert.True(cache.IsUnderPressure()); + Assert.Equal("6", cache.LastPurgedOffset); + + targetCursor.RecordDeliveryFailure(); + Assert.Equal(6, progress.SafeSequenceToken?.SequenceNumber); + Assert.True(targetCursor.MoveNext()); + Assert.Equal(target, targetCursor.GetCurrent(out _)!.SequenceToken); + progress.RecordDeliverySuccess(); + cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); + Assert.Equal("7", cache.LastPurgedOffset); + Assert.Equal(2, cache.ItemCount); + + Assert.True(targetCursor.MoveNext()); + Assert.Equal(9, targetCursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.Equal(8, progress.SafeSequenceToken?.SequenceNumber); + progress.RecordDeliverySuccess(); + Assert.False(targetCursor.MoveNext()); + cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); + Assert.Equal("9", cache.LastPurgedOffset); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(3, cache.GetMaxAddCount()); + } + + [Fact] + public void Cache_FutureStartScanWaitsForPendingDeliveryAcknowledgement() + { + var stream = StreamId.Create("namespace", Guid.NewGuid()); + using var cache = new RecoverableStreamQueueCache( + 3, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance, + maxCacheSize: 3); + cache.Add([new TestQueueMessage(stream, 1, "pending")], DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(stream, new EventSequenceTokenV2(1)); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + cache.Add( + [ + new TestQueueMessage(stream, 2, "before-start-2"), + new TestQueueMessage(stream, 3, "before-start-3"), + ], DateTime.UnixEpoch); + + cursor.Refresh(new EventSequenceTokenV2(7)); + + Assert.False(cursor.MoveNext()); + Assert.Null(progress.SafeSequenceToken); + cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); + Assert.Equal(3, cache.ItemCount); + Assert.True(cache.IsUnderPressure()); + progress.RecordDeliverySuccess(); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); + Assert.Equal(0, cache.ItemCount); + + cache.Add([new TestQueueMessage(stream, 7, "target")], DateTime.UnixEpoch); + Assert.True(cursor.MoveNext()); + Assert.Equal(7, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + } + [Fact] public void Cache_DeliveredThroughScansIntermediatePartitionRecordsWithoutRedelivery() { @@ -873,9 +994,9 @@ private static RecoverableStreamReceiver CreateReceiver( return new(source, adapter, cache, checkpointer, startFromNow: false); } - private sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); + internal sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); - private sealed class TestDataAdapter : IRecoverableStreamDataAdapter + internal sealed class TestDataAdapter : IRecoverableStreamDataAdapter { public int CompareCallCount { get; private set; } public int PositionCallCount { get; private set; } @@ -988,7 +1109,7 @@ public bool TryGetOffset(StreamSequenceToken token, out string offset) } } - private sealed class TestBatchContainer( + internal sealed class TestBatchContainer( StreamId streamId, StreamSequenceToken sequenceToken, string payload) : IBatchContainer @@ -1143,7 +1264,7 @@ public Task Shutdown(CancellationToken cancellationToken) } } - private sealed class NoOpEvictionStrategy : IEvictionStrategy + internal sealed class NoOpEvictionStrategy : IEvictionStrategy { public int PerformPurgeCount { get; private set; } From 1cd14cfc043aa2bbb5db878c5f601462ef45d68a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:38:20 -0700 Subject: [PATCH 51/90] test(streaming): use concrete unavailable consumer fixture --- .../ClientObserverRoutingTests.cs | 3 ++ ...stentStreamPullingAgentDiagnosticsTests.cs | 28 ++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs index 50673592ca4..380aea9ec7b 100644 --- a/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs +++ b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs @@ -15,6 +15,9 @@ namespace Tester.ClientConnectionTests; [TestCategory("BVT")] public class ClientObserverRoutingTests { + public static ClientNotAvailableException CreateUnavailableClientException() + => new(ClientGrainId.Create().GrainId); + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 7a79a6adb95..7a1267be840 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -27,10 +27,8 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); var (accessor, pubSub, streamData) = await CreateInitializedAgentWithStream( streamId, token, cache, new StreamPullingAgentOptions()); - var unavailable = new ClientNotAvailableException(ClientGrainId.Create().GrainId); - var consumer = Substitute.For(); - consumer.DeliverBatch(default!, default, default!, default, default) - .ReturnsForAnyArgs(Task.FromException(unavailable)); + var unavailable = Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException(); + var consumer = new UnavailableConsumer(unavailable); var data = streamData.AddConsumer(subscriptionId, streamId, consumer, filterData: null, DateTime.UtcNow); data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); data.IsRegistered = true; @@ -51,6 +49,28 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking { outcome.TrySetResult(registration); } + + private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension + { + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, + StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => Task.FromException(exception); + + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception error, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => Task.FromResult(null); + } } }); From 0aa22f006dcf23dfc8d0fe80bde73168f7e9eaad Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:32:01 -0700 Subject: [PATCH 52/90] fix(streaming): preserve ADO.NET retention continuity --- .../docs/streaming/stream-providers.md | 6 +- .../AdoNetRecoverableStream.cs | 72 ++++- .../MySQL-Streaming.sql | 27 +- .../PostgreSQL-Streaming.sql | 23 +- .../SQLServer-Streaming.sql | 51 ++-- .../Storage/RelationalOrleansQueries.cs | 6 +- .../Shared/Storage/RelationalStorage.cs | 2 + .../Streaming/AdoNetQueueAdapterTests.cs | 189 +++++++++++++ .../Streaming/AdoNetRecoverableStreamTests.cs | 265 +++++++++++++++++- .../Streaming/AdoNetStreamPartitionTests.cs | 227 ++++++++++++++- 10 files changed, 803 insertions(+), 65 deletions(-) diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 234324eb6b8..2b17eb4c930 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -70,7 +70,11 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S ## ADO.NET partitioned stream provider (alpha) -Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline resumes strictly after the earliest safe scan/delivery checkpoint, redelivering uncheckpointed records after recovery. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. retains checkpointed records for one day by default before cleanup. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Retention and cleanup intervals round fractional seconds upward. Cleanup is bounded by and deletes a contiguous eligible prefix. can fault one failing subscription while preserving the multicast partition record. +Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. Message-ID allocation and insertion commit together, producing gap-free IDs within each partition. The stream partition pipeline resumes strictly after the earliest safe scan/delivery checkpoint, redelivering uncheckpointed records after recovery. Initial acquisition can start immediately before the earliest retained message. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. + + retains checkpointed records for one day by default before cleanup. makes older records eligible for hard deletion, including unprocessed records. Retention and cleanup intervals round fractional seconds upward. Cleanup inspects at most records in message-ID order and deletes the eligible prefix ending before the first ineligible record. A younger leading record therefore retains later records even when database timestamps move backward. + +The receiver checks each read batch for consecutive IDs beginning immediately after its admitted offset. When a cleanup response fails, the next read reconciles durable retained bounds with that offset, using the next message ID to detect fully deleted history. An unread gap permanently faults that receiver's source with ; subsequent reads report the same failure. Size retention for the expected processing lag and recovery window, and use the reported partition and missing message IDs to investigate data loss. can fault one failing subscription while preserving the multicast partition record. The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index 4823a2793d6..93c48f28754 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -16,11 +16,17 @@ internal sealed partial class AdoNetRecoverableStream( private Task? _acquisitionTask; private DataNotAvailableException? _retentionFailure; private long? _pendingHardDeletedThrough; + private bool _cleanupNeedsReconciliation; internal Task AcquisitionCompletion => Volatile.Read(ref _acquisitionTask) ?? Task.CompletedTask; public async ValueTask Load(CancellationToken cancellationToken) { + if (Volatile.Read(ref _retentionFailure) is { } retentionFailure) + { + throw retentionFailure; + } + var acquisitionTask = queries.AcquireStreamPartitionAsync( serviceId, providerId, @@ -103,6 +109,12 @@ public async Task> Read( throw retentionFailure; } + if (_cleanupNeedsReconciliation) + { + await ReconcileRetainedBounds(cancellationToken); + _cleanupNeedsReconciliation = false; + } + var messages = await queries.ReadStreamMessagesAsync( serviceId, providerId, @@ -111,6 +123,26 @@ public async Task> Read( Math.Min(maxCount, options.MaxMessagesPerRead), cancellationToken); + var readThrough = _readOffset; + foreach (var message in messages) + { + if (message.MessageId != readThrough + 1) + { + throw RetainFailure( + $"has a retention gap: expected message {readThrough + 1}, received message {message.MessageId}."); + } + + readThrough = message.MessageId; + } + + if (messages.Count == 0) + { + await ReconcileRetainedBounds(cancellationToken); + } + + // A failed response can follow a committed deletion. Reconcile against the admitted + // offset on the next read, before allowing any later records into the cache. + _cleanupNeedsReconciliation = true; var cleanup = await queries.CleanupStreamMessagesAsync( serviceId, providerId, @@ -135,21 +167,19 @@ options.MaximumRetentionPeriod is { } maximum cleanup.Checkpoint); } - var readThrough = messages.Count > 0 ? messages[^1].MessageId : _readOffset; if (cleanup.HardDeletedThroughMessageId is { } hardDeletedThrough && hardDeletedThrough > readThrough) { - var failure = new DataNotAvailableException( - $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' lost unread retained records after " + throw RetainFailure( + "lost unread retained records after " + $"message {readThrough}: hard retention deleted through message {hardDeletedThrough}."); - var retainedFailure = Interlocked.CompareExchange(ref _retentionFailure, failure, comparand: null) ?? failure; - throw retainedFailure; } _pendingHardDeletedThrough = cleanup.HardDeletedThroughMessageId is { } deletedThrough && deletedThrough > _readOffset ? deletedThrough : null; + _cleanupNeedsReconciliation = false; return messages as IReadOnlyList ?? messages.ToList(); } @@ -174,10 +204,9 @@ public void MessagesAddFailed(IReadOnlyList messages) return; } - var failure = new DataNotAvailableException( - $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' could not retain hard-deleted records " + RetainFailure( + "could not retain hard-deleted records " + $"through message {hardDeletedThrough} because cache admission failed before the read offset advanced."); - Interlocked.CompareExchange(ref _retentionFailure, failure, comparand: null); _pendingHardDeletedThrough = null; } @@ -190,13 +219,36 @@ private void ThrowIfRetentionGap(AdoNetStreamPartitionState state) { if (HasRetentionGap(state)) { - throw new DataNotAvailableException( - $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' has a retention gap: " + throw RetainFailure( + "has a retention gap: " + $"checkpoint {state.Checkpoint}, earliest retained record {state.EarliestMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}, " + $"next message id {state.NextMessageId}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); } } + private async Task ReconcileRetainedBounds(CancellationToken cancellationToken) + { + var state = await queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId, cancellationToken) + ?? throw new InvalidOperationException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' is missing while reconciling retained history."); + var earliestAvailablePosition = state.EarliestMessageId ?? state.NextMessageId; + if (_readOffset < earliestAvailablePosition - 1 + || (state.NextMessageId > _readOffset + 1 && state.TailMessageId <= _readOffset)) + { + throw RetainFailure( + $"has a retention gap after admitted message {_readOffset}: " + + $"earliest retained record {state.EarliestMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}, " + + $"next message id {state.NextMessageId}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); + } + } + + private DataNotAvailableException RetainFailure(string detail) + { + var failure = new DataNotAvailableException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' {detail}"); + return Interlocked.CompareExchange(ref _retentionFailure, failure, comparand: null) ?? failure; + } + internal static bool HasRetentionGap(AdoNetStreamPartitionState state) { if (state.Checkpoint is not { } checkpoint) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 3adeea780a4..9535eacaaae 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -412,6 +412,7 @@ BEGIN DECLARE _EarliestMessageId BIGINT; DECLARE _TailMessageId BIGINT; DECLARE _PartitionExists BOOLEAN DEFAULT FALSE; + DECLARE _FirstIneligibleMessageId BIGINT; DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -468,17 +469,13 @@ BEGIN CREATE TEMPORARY TABLE OrleansStreamCleanupBatch ( MessageId BIGINT NOT NULL, + Eligible BOOLEAN NOT NULL, PRIMARY KEY (MessageId) ); - INSERT INTO OrleansStreamCleanupBatch (MessageId) - SELECT MessageId - FROM OrleansStreamMessage - WHERE ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND - ( + INSERT INTO OrleansStreamCleanupBatch (MessageId, Eligible) + SELECT MessageId, + CASE WHEN ( _Checkpoint IS NOT NULL AND MessageId <= _Checkpoint @@ -489,11 +486,23 @@ BEGIN _MaximumRetentionPeriodSeconds IS NOT NULL AND CreatedOn < DATE_SUB(_Now, INTERVAL _MaximumRetentionPeriodSeconds SECOND) ) - ) + THEN TRUE ELSE FALSE END + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId ORDER BY MessageId LIMIT _CleanupBatchSize FOR UPDATE; + SELECT MIN(MessageId) + INTO _FirstIneligibleMessageId + FROM OrleansStreamCleanupBatch + WHERE Eligible = FALSE; + + DELETE FROM OrleansStreamCleanupBatch + WHERE MessageId >= _FirstIneligibleMessageId; + SELECT COUNT(*), MAX(MessageId), diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 8b1e66e8d50..363af497346 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -433,13 +433,8 @@ BEGIN WITH Candidate AS ( - SELECT M.MessageId - FROM OrleansStreamMessage AS M - WHERE M.ServiceId = _ServiceId - AND M.ProviderId = _ProviderId - AND M.QueueId = _QueueId - AND - ( + SELECT M.MessageId, + CASE WHEN ( _Checkpoint IS NOT NULL AND M.MessageId <= _Checkpoint @@ -450,19 +445,29 @@ BEGIN _MaximumRetentionPeriodSeconds IS NOT NULL AND M.CreatedOn < _Now - make_interval(secs => _MaximumRetentionPeriodSeconds) ) - ) + THEN TRUE ELSE FALSE END AS Eligible + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId ORDER BY M.MessageId FOR UPDATE LIMIT _CleanupBatchSize ), + Boundary AS + ( + SELECT MIN(MessageId) FILTER (WHERE NOT Eligible) AS FirstIneligibleMessageId + FROM Candidate + ), Deleted AS ( DELETE FROM OrleansStreamMessage AS M - USING Candidate AS C + USING Candidate AS C, Boundary AS B WHERE M.ServiceId = _ServiceId AND M.ProviderId = _ProviderId AND M.QueueId = _QueueId AND M.MessageId = C.MessageId + AND (B.FirstIneligibleMessageId IS NULL OR C.MessageId < B.FirstIneligibleMessageId) RETURNING M.MessageId ) SELECT diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index 73ed997bb5f..8fe0e04858c 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -532,6 +532,8 @@ BEGIN DECLARE @StartedTransaction BIT = 0; DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); DECLARE @Checkpoint BIGINT; + DECLARE @FirstIneligibleMessageId BIGINT; + DECLARE @Candidate TABLE (MessageId BIGINT NOT NULL PRIMARY KEY, Eligible BIT NOT NULL); DECLARE @Deleted TABLE (MessageId BIGINT NOT NULL, HardDeleted BIT NOT NULL); BEGIN TRY @@ -582,29 +584,31 @@ BEGIN RETURN; END; - ;WITH Candidate AS - ( - SELECT TOP (@CleanupBatchSize) - MessageId - FROM OrleansStreamMessage WITH (UPDLOCK, READCOMMITTEDLOCK, ROWLOCK) - WHERE ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND + INSERT INTO @Candidate (MessageId, Eligible) + SELECT TOP (@CleanupBatchSize) + MessageId, + CASE WHEN ( - ( - @Checkpoint IS NOT NULL - AND MessageId <= @Checkpoint - AND CheckpointedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) - ) - OR - ( - @MaximumRetentionPeriodSeconds IS NOT NULL - AND CreatedOn < DATEADD(SECOND, -@MaximumRetentionPeriodSeconds, @Now) - ) + @Checkpoint IS NOT NULL + AND MessageId <= @Checkpoint + AND CheckpointedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) ) - ORDER BY MessageId - ) + OR + ( + @MaximumRetentionPeriodSeconds IS NOT NULL + AND CreatedOn < DATEADD(SECOND, -@MaximumRetentionPeriodSeconds, @Now) + ) + THEN 1 ELSE 0 END + FROM OrleansStreamMessage WITH (UPDLOCK, READCOMMITTEDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ORDER BY MessageId; + + SELECT @FirstIneligibleMessageId = MIN(MessageId) + FROM @Candidate + WHERE Eligible = 0; + DELETE Message OUTPUT Deleted.MessageId, @@ -614,11 +618,12 @@ BEGIN END INTO @Deleted (MessageId, HardDeleted) FROM OrleansStreamMessage AS Message - INNER JOIN Candidate + INNER JOIN @Candidate AS Candidate ON Candidate.MessageId = Message.MessageId WHERE Message.ServiceId = @ServiceId AND Message.ProviderId = @ProviderId - AND Message.QueueId = @QueueId; + AND Message.QueueId = @QueueId + AND (@FirstIneligibleMessageId IS NULL OR Message.MessageId < @FirstIneligibleMessageId); SELECT CAST(1 AS BIT) AS Ran, diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index 9408540925f..17ba19a51bc 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -618,7 +618,8 @@ internal Task> ReadStreamMessagesAsync( internal Task GetStreamPartitionBoundsAsync( string serviceId, string providerId, - string queueId) + string queueId, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); @@ -641,7 +642,8 @@ internal Task> ReadStreamMessagesAsync( ProviderId = providerId, QueueId = queueId }, - result => result.SingleOrDefault()); + result => result.SingleOrDefault(), + cancellationToken); } /// diff --git a/src/AdoNet/Shared/Storage/RelationalStorage.cs b/src/AdoNet/Shared/Storage/RelationalStorage.cs index 5ae76bae8c5..41bc8217b54 100644 --- a/src/AdoNet/Shared/Storage/RelationalStorage.cs +++ b/src/AdoNet/Shared/Storage/RelationalStorage.cs @@ -286,6 +286,8 @@ private async Task, int>> ExecuteReaderAsync { using (var reader = await command.ExecuteReaderAsync(commandBehavior, cancellationToken).ConfigureAwait(continueOnCapturedContext: false)) { + // MySql.Data can return a null reader after canceling a synchronous command. + cancellationToken.ThrowIfCancellationRequested(); CancellationTokenRegistration cancellationRegistration = default; try { diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 0859640ff2a..87d92d8f34e 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -1,8 +1,15 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging.Abstractions; using MySql.Data.MySqlClient; +using Npgsql; using Orleans.Configuration; +using Orleans.Providers.Streams.Common; using Orleans.Streaming.AdoNet; using Orleans.Streams; +using Orleans.TestingHost.Utils; using Orleans.Tests.SqlUtils; using TestExtensions; using UnitTests.General; @@ -181,6 +188,188 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() public ValueTask DisposeAsync() => ValueTask.CompletedTask; + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AcquisitionCancellation_BlockedCommandSettlesBeforeReplacement(bool cancelRead) + { + var cancellationToken = TestContext.Current.CancellationToken; + var serviceId = $"service-{Guid.NewGuid():N}"; + var providerId = $"provider-{Guid.NewGuid():N}"; + var queueId = QueueId.GetQueueId($"queue-{Guid.NewGuid():N}", 0, 0); + var partitionId = queueId.ToString(); + var initial = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, partitionId, false, cancellationToken); + var streamId = StreamId.Create("acquisition", Guid.NewGuid()); + var appended = await _queries.AppendStreamMessageAsync( + serviceId, providerId, partitionId, streamId.FullKey.ToArray(), streamId.Namespace.Length, [1]); + QueueAdapterReceiverRegistry registry = null!; + registry = new(_ => + { + var receiver = new AdoNetQueueAdapterReceiver( + providerId, + partitionId, + new AdoNetStreamOptions { StartFromNow = false }, + new ClusterOptions { ServiceId = serviceId }, + new SimpleQueueCacheOptions(), + _queries, + _fixture.Serializer.GetSerializer(), + NullLogger.Instance); + receiver.OnShutdown = stopped => registry.Remove(queueId, stopped); + return receiver; + }); + var first = registry.GetOrCreate(queueId); + + await using var lockConnection = CreateConnection(); + await lockConnection.OpenAsync(cancellationToken); + await using var transaction = await lockConnection.BeginTransactionAsync(cancellationToken); + await using (var command = lockConnection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = + """ + UPDATE OrleansStreamPartition SET ModifiedOn = @ModifiedOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """; + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", partitionId); + AddParameter(command, "ModifiedOn", DateTime.UtcNow.AddDays(-1)); + Assert.Equal(1, await command.ExecuteNonQueryAsync(cancellationToken)); + } + + await using var observer = lockConnection.CreateCommand(); + observer.Transaction = transaction; + observer.CommandText = invariant switch + { + AdoNetInvariants.InvariantNameSqlServer => "SELECT @@SPID", + AdoNetInvariants.InvariantNameMySql => "SELECT CONNECTION_ID()", + AdoNetInvariants.InvariantNamePostgreSql => "SELECT pg_backend_pid()", + _ => throw new NotSupportedException(invariant), + }; + var lockOwner = Convert.ToInt64(await observer.ExecuteScalarAsync(cancellationToken), CultureInfo.InvariantCulture); + observer.CommandText = GetBlockedAcquisitionQuery(lockConnection); + AddParameter(observer, "LockOwner", lockOwner); + using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var lockReleased = false; + Task> read = null!; + try + { + var blocked = WaitForBlockedAcquisitions(1); + read = first.GetQueueMessagesAsync(1, readCancellation.Token); + await blocked; + Assert.False(read.IsCompleted); + Assert.Same(first, registry.GetOrCreate(queueId)); + + if (cancelRead) + { + readCancellation.Cancel(); + } + + var shutdown = first.Shutdown(TimeSpan.FromSeconds(20)); + await Assert.ThrowsAnyAsync(() => read.WaitAsync(TimeSpan.FromSeconds(20), cancellationToken)); + await shutdown.WaitAsync(TimeSpan.FromSeconds(20), cancellationToken); + await WaitForBlockedAcquisitions(0); + + await transaction.RollbackAsync(cancellationToken); + lockReleased = true; + var afterCancellation = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, partitionId, cancellationToken); + Assert.Equal(initial.OwnerEpoch, afterCancellation!.OwnerEpoch); + Assert.Equal(initial.Checkpoint, afterCancellation.Checkpoint); + + var replacement = registry.GetOrCreate(queueId); + Assert.NotSame(first, replacement); + try + { + await replacement.Initialize(TimeSpan.FromSeconds(20)); + var acquired = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, partitionId, cancellationToken); + Assert.Equal(initial.OwnerEpoch + 1, acquired!.OwnerEpoch); + var advanced = await _queries.AdvanceStreamCheckpointAsync( + serviceId, providerId, partitionId, acquired.OwnerEpoch, appended.MessageId, cancellationToken); + Assert.True(advanced!.Updated); + Assert.Equal(appended.MessageId, advanced.Checkpoint); + Assert.Equal(acquired.OwnerEpoch, advanced.OwnerEpoch); + var final = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, partitionId, cancellationToken); + Assert.Equal(acquired.OwnerEpoch, final!.OwnerEpoch); + Assert.Equal(appended.MessageId, final.Checkpoint); + } + finally + { + await replacement.Shutdown(TimeSpan.FromSeconds(20)); + } + } + finally + { + readCancellation.Cancel(); + if (!lockReleased) + { + await transaction.RollbackAsync(CancellationToken.None); + } + + await first.Shutdown(TimeSpan.FromSeconds(20)); + if (read is not null) + { + await ((Task)read).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } + } + + Task WaitForBlockedAcquisitions(int expected) + => TestingUtils.WaitUntilAsync( + async (lastTry, token) => + { + var actual = Convert.ToInt32(await observer.ExecuteScalarAsync(token), CultureInfo.InvariantCulture); + if (lastTry) + { + Assert.True(actual == expected, + $"{invariant} partition {serviceId}/{providerId}/{partitionId}: expected {expected} acquisitions blocked by session {lockOwner}, observed {actual}."); + } + + return actual == expected; + }, + TimeSpan.FromSeconds(15), + TimeSpan.FromMilliseconds(50), + cancellationToken, + predicateExpression: $"{invariant} acquisition lock wait count for session {lockOwner} becomes {expected}"); + } + + private DbConnection CreateConnection() + => invariant switch + { + AdoNetInvariants.InvariantNameSqlServer => new SqlConnection(_storage.ConnectionString), + AdoNetInvariants.InvariantNameMySql => new MySqlConnection(_storage.ConnectionString), + AdoNetInvariants.InvariantNamePostgreSql => new NpgsqlConnection(_storage.ConnectionString), + _ => throw new NotSupportedException(invariant), + }; + + private string GetBlockedAcquisitionQuery(DbConnection connection) + => invariant switch + { + AdoNetInvariants.InvariantNameSqlServer => + "SELECT COUNT(*) FROM sys.dm_exec_requests WHERE blocking_session_id = @LockOwner", + AdoNetInvariants.InvariantNamePostgreSql => + "SELECT COUNT(*) FROM pg_locks WHERE NOT granted AND @LockOwner = ANY(pg_blocking_pids(pid))", + AdoNetInvariants.InvariantNameMySql when connection.ServerVersion.Contains("MariaDB", StringComparison.OrdinalIgnoreCase) => + """ + SELECT COUNT(*) FROM information_schema.INNODB_LOCK_WAITS AS W + INNER JOIN information_schema.INNODB_TRX AS B ON B.trx_id = W.blocking_trx_id + WHERE B.trx_mysql_thread_id = @LockOwner + """, + AdoNetInvariants.InvariantNameMySql => + """ + SELECT COUNT(*) FROM performance_schema.data_lock_waits AS W + INNER JOIN performance_schema.threads AS B ON B.THREAD_ID = W.BLOCKING_THREAD_ID + WHERE B.PROCESSLIST_ID = @LockOwner + """, + _ => throw new NotSupportedException(invariant), + }; + + private static void AddParameter(IDbCommand command, string name, object value) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.Value = value; + command.Parameters.Add(parameter); + } + [GenerateSerializer] [Alias("Tester.AdoNet.Streaming.AdoNetQueueAdapterTests.TestModel")] public record TestModel( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index c435e65a75e..138d31433f5 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -211,6 +211,7 @@ public async Task ShutdownNotification_WaitsForAnyAcquisitionCompletionBeforeAll [InlineData(StreamQueryKind.Read)] [InlineData(StreamQueryKind.Advance)] [InlineData(StreamQueryKind.Cleanup)] + [InlineData(StreamQueryKind.Bounds)] public async Task StreamingQueries_PropagateCancellationToken(StreamQueryKind queryKind) { var storage = new CapturingRelationalStorage(); @@ -231,6 +232,10 @@ public async Task StreamingQueries_PropagateCancellationToken(StreamQueryKind qu _ = await queries.CleanupStreamMessagesAsync( "service", "provider", "queue", 1, 2, 3, 4, cancellation.Token); break; + case StreamQueryKind.Bounds: + _ = await queries.GetStreamPartitionBoundsAsync( + "service", "provider", "queue", cancellation.Token); + break; default: throw new ArgumentOutOfRangeException(nameof(queryKind)); } @@ -277,7 +282,7 @@ public async Task Read_HardRetentionPastReturnedBatchPermanentlyFaultsSource() (nameof(AdoNetStreamMessage.ServiceId), "service"), (nameof(AdoNetStreamMessage.ProviderId), "provider"), (nameof(AdoNetStreamMessage.QueueId), "queue"), - (nameof(AdoNetStreamMessage.MessageId), 10L), + (nameof(AdoNetStreamMessage.MessageId), 1L), (nameof(AdoNetStreamMessage.StreamIdBytes), new byte[] { 1 }), (nameof(AdoNetStreamMessage.StreamNamespaceLength), 0), (nameof(AdoNetStreamMessage.CreatedOn), DateTime.UtcNow), @@ -327,7 +332,7 @@ public async Task MessagesAddFailed_AfterHardDeletionPermanentlyFaultsSource() (nameof(AdoNetStreamMessage.ServiceId), "service"), (nameof(AdoNetStreamMessage.ProviderId), "provider"), (nameof(AdoNetStreamMessage.QueueId), "queue"), - (nameof(AdoNetStreamMessage.MessageId), 10L), + (nameof(AdoNetStreamMessage.MessageId), 1L), (nameof(AdoNetStreamMessage.StreamIdBytes), new byte[] { 1 }), (nameof(AdoNetStreamMessage.StreamNamespaceLength), 0), (nameof(AdoNetStreamMessage.CreatedOn), DateTime.UtcNow), @@ -337,14 +342,14 @@ public async Task MessagesAddFailed_AfterHardDeletionPermanentlyFaultsSource() [ Record( (nameof(AdoNetStreamCleanupResult.Ran), true), - (nameof(AdoNetStreamCleanupResult.DeletedCount), 10), - (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), 10L), - (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 10), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 1), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), 1L), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 1), (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), 1L), - (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), 10L), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), 1L), (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), - (nameof(AdoNetStreamCleanupResult.EarliestMessageId), 11L), - (nameof(AdoNetStreamCleanupResult.TailMessageId), 11L)), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), 2L), + (nameof(AdoNetStreamCleanupResult.TailMessageId), 2L)), ], }; var source = new AdoNetRecoverableStream( @@ -365,6 +370,210 @@ public async Task MessagesAddFailed_AfterHardDeletionPermanentlyFaultsSource() Assert.Equal(readsAfterFailure, storage.ReadCallCount); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MessagesAdded_PartialAdmissionPreservesHardDeletedSuffix(bool admitWholeBatch) + { + var storage = new CapturingRelationalStorage + { + ReadRecords = [MessageRecord(1), MessageRecord(2), MessageRecord(3)], + CleanupRecords = + [ + Record( + (nameof(AdoNetStreamCleanupResult.Ran), true), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 3), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), 3L), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 3), + (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), 1L), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), 3L), + (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), null), + (nameof(AdoNetStreamCleanupResult.TailMessageId), null)), + ], + }; + var source = CreateSource(storage); + var messages = await source.Read(3, TestContext.Current.CancellationToken); + storage.CleanupRecords = null; + source.MessagesAdded([messages[0]]); + source.MessagesAdded([messages[1]]); + if (admitWholeBatch) + { + source.MessagesAdded([messages[2]]); + source.MessagesAddFailed([]); + storage.ReadRecords = [MessageRecord(4)]; + Assert.Equal(4, Assert.Single(await source.Read(1, TestContext.Current.CancellationToken)).MessageId); + } + else + { + source.MessagesAddFailed([messages[2]]); + var failure = await Assert.ThrowsAsync( + () => source.Read(1, TestContext.Current.CancellationToken)); + Assert.Contains("through message 3", failure.Message); + Assert.Contains("cache admission failed", failure.Message); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Read_MissingFirstOrInteriorMessagePermanentlyFaultsSource(bool interiorGap) + { + var storage = new CapturingRelationalStorage + { + ReadRecords = interiorGap ? [MessageRecord(1), MessageRecord(3)] : [MessageRecord(2)], + }; + var source = CreateSource(storage); + + var failure = await Assert.ThrowsAsync( + () => source.Read(10, TestContext.Current.CancellationToken)); + var calls = storage.ReadCallCount; + var repeated = await Assert.ThrowsAsync( + () => source.Read(10, TestContext.Current.CancellationToken)); + + Assert.Contains(interiorGap ? "expected message 2" : "expected message 1", failure.Message); + Assert.Same(failure, repeated); + Assert.Equal(calls, storage.ReadCallCount); + Assert.DoesNotContain(nameof(DbStoredQueries.CleanupStreamMessagesKey), storage.Parameters.Keys); + } + + [Theory] + [InlineData(1, false)] + [InlineData(3, false)] + [InlineData(3, true)] + public async Task Read_CommittedCleanupWithLostResponsePermanentlyFaultsSource(int deletedCount, bool appendAfterCleanup) + { + var storage = new CapturingRelationalStorage { ReadRecords = [MessageRecord(1), MessageRecord(2), MessageRecord(3)] }; + var lostResponse = new IOException("Cleanup committed before the connection failed."); + storage.OnCleanup = () => + { + storage.OnCleanup = null; + storage.ReadRecords = deletedCount == 1 + ? [MessageRecord(2), MessageRecord(3)] + : appendAfterCleanup ? [MessageRecord(4)] : []; + storage.PartitionRecord = PartitionRecord( + checkpoint: 0, + nextMessageId: appendAfterCleanup ? 5 : 4, + earliest: deletedCount == 1 ? 2 : appendAfterCleanup ? 4 : null, + tail: appendAfterCleanup ? 4 : deletedCount == 1 ? 3 : null); + throw lostResponse; + }; + var source = CreateSource(storage); + + Assert.Same(lostResponse, await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken))); + var failure = await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken)); + var calls = storage.ReadCallCount; + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken))); + + Assert.Contains("after admitted message 0", failure.Message); + Assert.Equal(calls, storage.ReadCallCount); + Assert.Contains(nameof(DbStoredQueries.GetStreamPartitionBoundsKey), storage.Parameters.Keys); + Assert.DoesNotContain(nameof(DbStoredQueries.AcquireStreamPartitionKey), storage.Parameters.Keys); + } + + [Fact] + public async Task Read_FailedCleanupWithIntactHistoryRetriesFromAdmittedOffset() + { + var storage = new CapturingRelationalStorage + { + ReadRecords = [MessageRecord(1), MessageRecord(2)], + PartitionRecord = PartitionRecord(0, 3, 1, 2), + }; + storage.OnCleanup = () => + { + storage.OnCleanup = null; + throw new IOException("Cleanup rolled back."); + }; + var source = CreateSource(storage); + + await Assert.ThrowsAsync(() => source.Read(2, TestContext.Current.CancellationToken)); + var messages = await source.Read(2, TestContext.Current.CancellationToken); + Assert.Equal([1L, 2L], messages.Select(message => message.MessageId)); + source.MessagesAdded(messages); + storage.ReadRecords = [MessageRecord(3)]; + Assert.Equal(3, Assert.Single(await source.Read(1, TestContext.Current.CancellationToken)).MessageId); + Assert.DoesNotContain(nameof(DbStoredQueries.AcquireStreamPartitionKey), storage.Parameters.Keys); + } + + [Fact] + public async Task Read_InitialRetentionBaselineResumesBeforeEarliestRetainedMessage() + { + var storage = new CapturingRelationalStorage + { + PartitionRecord = PartitionRecord(9, 12, 10, 11), + ReadRecords = [MessageRecord(10), MessageRecord(11)], + }; + var source = CreateSource(storage); + var loaded = await source.Load(TestContext.Current.CancellationToken); + await source.Initialize(new(loaded.Checkpoint, startFromNow: false), TestContext.Current.CancellationToken); + + Assert.Equal("9", loaded.Checkpoint); + var messages = await source.Read(2, TestContext.Current.CancellationToken); + Assert.Equal([10L, 11L], messages.Select(message => message.MessageId)); + } + + [Fact] + public async Task Load_RetentionGapPermanentlyFaultsWithoutReacquiring() + { + var storage = new CapturingRelationalStorage { PartitionRecord = PartitionRecord(0, 4, null, null) }; + var source = CreateSource(storage); + var failure = await Assert.ThrowsAsync( + () => source.Load(TestContext.Current.CancellationToken).AsTask()); + var calls = storage.ReadCallCount; + + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Load(TestContext.Current.CancellationToken).AsTask())); + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Read(1, TestContext.Current.CancellationToken))); + Assert.Equal(calls, storage.ReadCallCount); + } + + [Fact] + public async Task Read_CheckpointedHistoryHolesPreserveUnreadContinuity() + { + var storage = new CapturingRelationalStorage + { + PartitionRecord = PartitionRecord(3, 5, 1, 4), + ReadRecords = [MessageRecord(4)], + }; + var source = CreateSource(storage); + await source.Load(TestContext.Current.CancellationToken); + + Assert.Equal(4, Assert.Single(await source.Read(1, TestContext.Current.CancellationToken)).MessageId); + } + + [Theory] + [InlineData(0, 4, null, null, true)] + [InlineData(2, 4, 1, 2, true)] + [InlineData(3, 4, null, null, false)] + [InlineData(3, 5, 4, 4, false)] + public async Task Read_EmptyBatchReconcilesDeletedHistoryAndConcurrentArrivals( + long checkpoint, long nextMessageId, long? earliest, long? tail, bool gap) + { + var storage = new CapturingRelationalStorage(); + var source = CreateSource(storage); + await source.Load(TestContext.Current.CancellationToken); + await source.Initialize(new(checkpoint.ToString(System.Globalization.CultureInfo.InvariantCulture), false), TestContext.Current.CancellationToken); + storage.PartitionRecord = PartitionRecord(checkpoint, nextMessageId, earliest, tail); + + if (gap) + { + var failure = await Assert.ThrowsAsync( + () => source.Read(1, TestContext.Current.CancellationToken)); + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Read(1, TestContext.Current.CancellationToken))); + } + else + { + Assert.Empty(await source.Read(1, TestContext.Current.CancellationToken)); + storage.ReadRecords = [MessageRecord(checkpoint + 1)]; + Assert.Equal(checkpoint + 1, Assert.Single(await source.Read(1, TestContext.Current.CancellationToken)).MessageId); + } + } + [Theory] [InlineData(null, 1L, null, false)] [InlineData(0L, 1L, null, false)] @@ -402,6 +611,31 @@ private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage private static IDataRecord Record(params (string Name, object? Value)[] values) => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); + private static AdoNetRecoverableStream CreateSource(IRelationalStorage storage) + => new("service", "provider", "queue", new AdoNetStreamOptions { MaxMessagesPerRead = 10 }, CreateQueries(storage), NullLogger.Instance); + + private static IDataRecord MessageRecord(long messageId) + => Record( + (nameof(AdoNetStreamMessage.ServiceId), "service"), + (nameof(AdoNetStreamMessage.ProviderId), "provider"), + (nameof(AdoNetStreamMessage.QueueId), "queue"), + (nameof(AdoNetStreamMessage.MessageId), messageId), + (nameof(AdoNetStreamMessage.StreamIdBytes), new byte[] { 1 }), + (nameof(AdoNetStreamMessage.StreamNamespaceLength), 0), + (nameof(AdoNetStreamMessage.CreatedOn), DateTime.UtcNow), + (nameof(AdoNetStreamMessage.Payload), new byte[] { 2 })); + + private static IDataRecord PartitionRecord(long checkpoint, long nextMessageId, long? earliest, long? tail) + => Record( + (nameof(AdoNetStreamPartitionState.ServiceId), "service"), + (nameof(AdoNetStreamPartitionState.ProviderId), "provider"), + (nameof(AdoNetStreamPartitionState.QueueId), "queue"), + (nameof(AdoNetStreamPartitionState.OwnerEpoch), 1L), + (nameof(AdoNetStreamPartitionState.NextMessageId), nextMessageId), + (nameof(AdoNetStreamPartitionState.Checkpoint), checkpoint), + (nameof(AdoNetStreamPartitionState.EarliestMessageId), earliest), + (nameof(AdoNetStreamPartitionState.TailMessageId), tail)); + private sealed class BlockingAdoNetQueueAdapterFactory( RelationalOrleansQueries queries, IQueueAdapter adapter, @@ -545,9 +779,13 @@ private sealed class CapturingRelationalStorage : IRelationalStorage public Dictionary> Parameters { get; } = []; - public IReadOnlyList ReadRecords { get; init; } = []; + public IReadOnlyList ReadRecords { get; set; } = []; - public IReadOnlyList? CleanupRecords { get; init; } + public IDataRecord PartitionRecord { get; set; } = AdoNetRecoverableStreamTests.PartitionRecord(0, 1, null, null); + + public Action? OnCleanup { get; set; } + + public IReadOnlyList? CleanupRecords { get; set; } public int ReadCallCount { get; private set; } @@ -569,8 +807,14 @@ public async Task> ReadAsync( Parameters[query] = command.Parameters.Cast() .ToDictionary(parameter => parameter.ParameterName, parameter => parameter.Value is DBNull ? null : parameter.Value); + if (query == nameof(DbStoredQueries.CleanupStreamMessagesKey)) + { + OnCleanup?.Invoke(); + } + var records = query switch { + nameof(DbStoredQueries.AcquireStreamPartitionKey) or nameof(DbStoredQueries.GetStreamPartitionBoundsKey) => [PartitionRecord], nameof(DbStoredQueries.ReadStreamMessagesKey) => ReadRecords, nameof(DbStoredQueries.AdvanceStreamCheckpointKey) => [ @@ -644,5 +888,6 @@ public enum StreamQueryKind Read, Advance, Cleanup, + Bounds, } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs index 0589d25d496..575ec87f6a1 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -660,6 +660,198 @@ public async Task GetStreamPartitionBounds_ReflectsCurrentState() #region Cleanup: retention, hard ceiling diagnostics, batching, and throttling + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoverableStream_ContinuityChecksOnlyUnreadHistory(bool checkpointBeforeGap) + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + var initial = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, false); + if (checkpointBeforeGap) + { + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, initial.OwnerEpoch, 2); + } + + await _storage.ExecuteAsync( + """ + DELETE FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId = 2 + """, + command => + { + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }, + cancellationToken: TestContext.Current.CancellationToken); + var source = new AdoNetRecoverableStream( + serviceId, providerId, queueId, new AdoNetStreamOptions { StartFromNow = false }, _queries.Inner, NullLogger.Instance); + var loaded = await source.Load(TestContext.Current.CancellationToken); + Assert.Equal(checkpointBeforeGap ? "2" : "0", loaded.Checkpoint); + if (checkpointBeforeGap) + { + Assert.Equal(3, Assert.Single(await source.Read(3, TestContext.Current.CancellationToken)).MessageId); + } + else + { + var failure = await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken)); + Assert.Contains("expected message 2, received message 3", failure.Message); + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken))); + Assert.Equal(0, (await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId))!.Checkpoint); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task CleanupStreamMessages_StopsAtFirstIneligibleMessage(bool hardRetention, bool eligibleFirstMessage) + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: !hardRetention); + var timestampColumn = hardRetention ? "CreatedOn" : "CheckpointedOn"; + await _storage.ExecuteAsync( + $""" + UPDATE OrleansStreamMessage + SET {timestampColumn} = CASE + WHEN MessageId = @FirstAgedId OR MessageId = @SecondAgedId THEN @AgedOn + ELSE @YoungOn END + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "AgedOn", DateTime.UtcNow.AddDays(-2)); + AddParameter(command, "YoungOn", DateTime.UtcNow.AddDays(1)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + AddParameter(command, "FirstAgedId", eligibleFirstMessage ? 1L : 2L); + AddParameter(command, "SecondAgedId", eligibleFirstMessage ? 3L : 2L); + }, + cancellationToken: TestContext.Current.CancellationToken); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + var cleanup = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, 60, hardRetention ? 120 : null, 60, 3); + Assert.True(cleanup.Ran); + Assert.Equal(eligibleFirstMessage ? 1 : 0, cleanup.DeletedCount); + Assert.Equal(eligibleFirstMessage ? 1L : (long?)null, cleanup.DeletedThroughMessageId); + Assert.Equal(hardRetention && eligibleFirstMessage ? 1 : 0, cleanup.HardDeletedCount); + var remaining = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, 0, 10); + long[] expectedIds = eligibleFirstMessage ? [2, 3] : [1, 2, 3]; + Assert.Equal(expectedIds, remaining.Select(message => message.MessageId)); + + await MakeCleanupDueAsync(serviceId, providerId, queueId); + var repeated = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, 60, hardRetention ? 120 : null, 60, 3); + Assert.Equal(0, repeated.DeletedCount); + } + + [Theory] + [InlineData(1, false)] + [InlineData(3, false)] + [InlineData(3, true)] + public async Task RecoverableStream_CommittedCleanupWithLostResponsePermanentlyFaults( + int cleanupBatchSize, bool appendAfterCleanup) + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + var storedQueries = new DbStoredQueries((await _storage.ReadAsync( + DbStoredQueries.GetQueriesKey, + null, + (record, _, _) => Task.FromResult(new KeyValuePair( + (string)record["QueryKey"], (string)record["QueryText"])), + cancellationToken: TestContext.Current.CancellationToken)).ToDictionary(pair => pair.Key, pair => pair.Value)); + var lostResponseStorage = new LostCleanupResponseStorage(_storage, storedQueries.CleanupStreamMessagesKey); + var source = new AdoNetRecoverableStream( + serviceId, + providerId, + queueId, + new AdoNetStreamOptions + { + StartFromNow = false, + MaxMessagesPerRead = 3, + RetentionPeriod = TimeSpan.FromMinutes(1), + MaximumRetentionPeriod = TimeSpan.FromMinutes(2), + CleanupBatchSize = cleanupBatchSize, + }, + new RelationalOrleansQueries(lostResponseStorage, storedQueries), + NullLogger.Instance); + var loaded = await source.Load(TestContext.Current.CancellationToken); + Assert.Equal("0", loaded.Checkpoint); + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + Assert.Same(lostResponseStorage.Failure, await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken))); + var afterCleanup = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, 0, 10); + Assert.Equal(Enumerable.Range(cleanupBatchSize + 1, 3 - cleanupBatchSize).Select(id => (long)id), + afterCleanup.Select(message => message.MessageId)); + if (appendAfterCleanup) + { + Assert.Equal(4, (await AppendAsync(serviceId, providerId, queueId)).MessageId); + } + + var failure = await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken)); + Assert.Contains("after admitted message 0", failure.Message); + Assert.Same(failure, await Assert.ThrowsAsync( + () => source.Read(3, TestContext.Current.CancellationToken))); + var bounds = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + Assert.Equal(0, bounds!.Checkpoint); + Assert.Equal(long.Parse(loaded.Version, CultureInfo.InvariantCulture), bounds.OwnerEpoch); + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + public async Task RecoverableStream_InitialAcquisitionUsesRetainedBaseline(int deletedCount) + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + var cleanup = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 60, 120, 60, deletedCount); + Assert.Equal(deletedCount, cleanup.DeletedCount); + var source = new AdoNetRecoverableStream( + serviceId, providerId, queueId, new AdoNetStreamOptions { StartFromNow = false }, _queries.Inner, NullLogger.Instance); + var loaded = await source.Load(TestContext.Current.CancellationToken); + Assert.Equal(deletedCount.ToString(CultureInfo.InvariantCulture), loaded.Checkpoint); + await AppendAsync(serviceId, providerId, queueId); + + var messages = await source.Read(10, TestContext.Current.CancellationToken); + Assert.Equal(Enumerable.Range(deletedCount + 1, 4 - deletedCount).Select(id => (long)id), + messages.Select(message => message.MessageId)); + } + [Fact] public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentionElapses() { @@ -1195,7 +1387,7 @@ public Task> ReadStreamMessagesAsync( string serviceId, string providerId, string queueId) - => Inner.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + => Inner.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId, TestContext.Current.CancellationToken); public Task CleanupStreamMessagesAsync( string serviceId, @@ -1216,5 +1408,38 @@ public Task CleanupStreamMessagesAsync( TestContext.Current.CancellationToken); } + private sealed class LostCleanupResponseStorage(IRelationalStorage inner, string cleanupQuery) : IRelationalStorage + { + private bool _failed; + + public IOException Failure { get; } = new("Cleanup committed before its response reached the source."); + public string InvariantName => inner.InvariantName; + public string ConnectionString => inner.ConnectionString; + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + var result = await inner.ReadAsync(query, parameterProvider, selector, commandBehavior, cancellationToken); + if (!_failed && query == cleanupQuery) + { + _failed = true; + throw Failure; + } + + return result; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + => inner.ExecuteAsync(query, parameterProvider, commandBehavior, cancellationToken); + } + #endregion SQL Server } From bb3e829acbe99e64c4d8ce376d8f44211c259b94 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:35:26 -0700 Subject: [PATCH 53/90] feat(streaming): bound recovery cache encoded bytes --- .../docs/streaming/streaming-operations.md | 16 +- .../Streams/KinesisPooledAdapterReceiver.cs | 4 +- .../Streams/KinesisStreamOptions.cs | 18 + .../AdoNetQueueAdapterReceiver.cs | 1 + .../AdoNetStreamOptions.cs | 11 + .../AdoNetStreamOptionsValidator.cs | 5 + .../IRecoverableStreamQueueCache.cs | 7 +- .../IRecoverableStreamSource.cs | 11 +- .../RecoverableStreamQueueCache.cs | 166 ++++- .../RecoverableStreamReceiver.cs | 65 +- .../RecoveryCacheMemoryOptionsTests.cs | 270 +++++++ .../RecoveryCacheMemoryOptionsTests.cs | 48 ++ .../RecoveryCacheMemoryTests.cs | 665 ++++++++++++++++++ 13 files changed, 1262 insertions(+), 25 deletions(-) create mode 100644 test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs create mode 100644 test/Extensions/Orleans.Streaming.Kinesis.Tests/RecoveryCacheMemoryOptionsTests.cs create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs diff --git a/docs/site/src/content/docs/streaming/streaming-operations.md b/docs/site/src/content/docs/streaming/streaming-operations.md index 2999b27fb59..4eeb6e6d083 100644 --- a/docs/site/src/content/docs/streaming/streaming-operations.md +++ b/docs/site/src/content/docs/streaming/streaming-operations.md @@ -45,7 +45,7 @@ Persistent providers expose common configuration through their stream configurat - bounds delivery attempts before the configured failure handler is involved. - controls item capacity for providers using the simple queue cache. -Provider-specific controls matter as much as common controls: , Event Hubs partitions and cache-pressure settings, Redis `ReadCount` and retention, NATS `BatchSize` and `PartitionCount`, and ADO.NET visibility, expiry, and dead-letter settings. +Provider-specific controls matter as much as common controls: , Event Hubs partitions and cache-pressure settings, Redis `ReadCount` and retention, NATS `BatchSize` and `PartitionCount`, and ADO.NET read size, checkpoints, and retention. Change one bottleneck at a time. More queues can increase parallelism but also broker cost, polling load, cache memory, and rebalance work. Reducing polling delay can lower latency while increasing empty reads. @@ -57,6 +57,20 @@ Each record contains one published batch of events. A value such as `25` reduces The bound is measured in records. The aggregate byte size depends on the serialized payloads in those records, including all events in each published batch. Size producer batches and payloads so that a complete dequeue response fits the configured Orleans message-body limit. The queue grain removes records before its response is serialized; an oversized response can therefore lose those records when serialization fails. Tune the count alongside measured response sizes and queue lag. +### Budget ADO.NET and Kinesis recovery caches + + and set an independent encoded-data budget for each partition or shard. Each defaults to 64 MiB. The budget counts retained cache segments: serialized payloads and their length framing, plus encoded Kinesis shard sequence numbers. also caps retained records, with a default of 4,096. + +The receiver admits records in source order until either capacity is reached. It keeps the remainder of one fetched batch for subsequent admission and advances the source read offset only through admitted records. If the next record exceeds the available bytes, the cache applies backpressure until safe subscription delivery progress releases enough capacity. An empty cache admits one record larger than the configured budget; that record is delivered and released before further admission. Set producer record-size limits to bound this progress exception. + +Plan silo memory from the number of locally owned partitions, including the highest partition count expected during reassignment. For example, 16 partitions with 64 MiB budgets permit 1 GiB of retained encoded segments before the oversized-record exception. Add headroom for: + +- One fetched source batch per partition. ADO.NET reads at most records, defaulting to 1,000. Kinesis reads at most 1,000 records, also subject to the service response limit. These records can retain provider response buffers and decoded objects while waiting for admission. +- Buffer capacity and free-pool retention. Each partition uses 1 MiB pooled buffers; records larger than a pooled buffer use an exact-sized standalone buffer. Sequential packing leaves unused tails, and the oldest buffer can retain a delivered prefix while later records in that buffer remain cached. Pooled buffers are retained for reuse up to the pool's observed high-water mark. +- Cache metadata, source record wrappers, in-flight delivery copies, deserialized events, and the rest of the silo workload. Delivery batching and event object shape affect this headroom. + +Use encoded record sizes together with producer record-size limits to choose the budget and read count. Lower read counts reduce staged-batch memory at the cost of more source requests. Monitor sustained cache pressure and provider backlog; provision retention to cover the slowest required consumer's recovery window. Safe delivery progress releases both record and byte capacity, and shutdown releases staged records and active cache-buffer ownership. + ## Observe health Export Orleans meters and correlate them with broker metrics and application event IDs. Useful Orleans instruments include: diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index 2302b0b5f2a..fb5bbc020bc 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -41,7 +41,8 @@ public KinesisPooledAdapterReceiver( KinesisShardTopologyMonitor topologyMonitor, TimeSpan getRecordsInterval, TimeProvider timeProvider, - Action? onShutdown = null) + Action? onShutdown = null, + long maxCacheSizeBytes = 64 * 1024 * 1024) { _checkpointerFactory = checkpointerFactory; _partition = partition; @@ -66,6 +67,7 @@ public KinesisPooledAdapterReceiver( _dataAdapter, evictionStrategy, logger, + maxCacheSizeBytes: maxCacheSizeBytes, maxCacheSize: cacheOptions.CacheSize); } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisStreamOptions.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisStreamOptions.cs index 9d1177da8a8..30925933efe 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisStreamOptions.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisStreamOptions.cs @@ -79,6 +79,18 @@ public string? ConnectionString /// public string StreamName { get; set; } = "OrleansTestStream"; + /// + /// Gets or sets the per-shard budget for retained encoded cache segments, in bytes. + /// The default is 64 MiB. + /// + /// + /// The budget includes payload framing and encoded shard sequence numbers. Records are admitted + /// in order, and an empty cache admits one oversized record so delivery can make progress. + /// Additional memory includes one fetched batch of up to 1,000 records, buffer slack and pool + /// retention, metadata, and deserialized delivery objects. + /// + public long MaxCacheSizeBytes { get; set; } = 64 * 1024 * 1024; + /// /// Gets or sets the minimum interval between calls to Kinesis GetRecords for a shard. /// @@ -114,6 +126,12 @@ public void ValidateConfiguration() $"The {nameof(KinesisStreamOptions.GetRecordsInterval)} must be at least {MinimumGetRecordsInterval} for the Kinesis stream provider '{name}'."); } + if (options.MaxCacheSizeBytes <= 0) + { + throw new OrleansConfigurationException( + $"The {nameof(KinesisStreamOptions.MaxCacheSizeBytes)} must be greater than zero for the Kinesis stream provider '{name}'."); + } + if (options.TopologyCheckInterval <= TimeSpan.Zero) { throw new OrleansConfigurationException( diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index a63a29deffb..79638afc1fa 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -51,6 +51,7 @@ public AdoNetQueueAdapterReceiver( dataAdapter, evictionStrategy, logger, + maxCacheSizeBytes: streamOptions.MaxCacheSizeBytes, maxCacheSize: cacheOptions.CacheSize); _inner = new RecoverableStreamReceiver( _source, diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index aa9074ff0a7..e8356e04e46 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -46,6 +46,17 @@ public class AdoNetStreamOptions /// public int MaxMessagesPerRead { get; set; } = 1_000; + /// + /// Gets or sets the per-partition budget for retained encoded cache segments, in bytes. + /// The default is 64 MiB. + /// + /// + /// The budget includes payload framing. Records are admitted in order, and an empty cache admits + /// one oversized record so delivery can make progress. Additional memory includes one fetched batch + /// bounded by , buffer slack and pool retention, metadata, and delivery objects. + /// + public long MaxCacheSizeBytes { get; set; } = 64 * 1024 * 1024; + /// /// Gets or sets the interval between checkpoint persistence attempts. /// diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs index 620b98fdfbb..9ec06e71d97 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs @@ -26,6 +26,11 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxMessagesPerRead)} must be greater than zero."); } + if (options.MaxCacheSizeBytes <= 0) + { + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxCacheSizeBytes)} must be greater than zero."); + } + if (options.CheckpointPersistInterval <= TimeSpan.Zero) { throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CheckpointPersistInterval)} must be greater than zero."); diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs index 70d8b408a2c..b0f0b0f406b 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs @@ -13,8 +13,13 @@ namespace Orleans.Providers.Streams.Common; public interface IRecoverableStreamQueueCache : IQueueCache, IDisposable { /// - /// Packs and adds ordered source records to the cache. + /// Packs and adds an ordered prefix of source records to the cache. /// + /// The positions of the admitted prefix, in source order. + /// + /// The receiver retains the remaining records for a subsequent admission attempt. + /// A packing exception leaves the cache's previously admitted contents intact. + /// IReadOnlyList Add( IReadOnlyList messages, DateTime dequeueTimeUtc); diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs index f740a801fd3..72c797c3cf4 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs @@ -21,19 +21,28 @@ public interface IRecoverableStreamSource /// /// Reads an ordered batch of immutable source records. /// + /// The maximum number of records to return. + /// The cancellation token. Task> Read(int maxCount, CancellationToken cancellationToken); /// /// Notifies the source that records were successfully admitted to the cache. /// /// The admitted records. - /// Sources should advance volatile read offsets only in this callback. + /// + /// Each callback covers an ordered prefix of the remaining records from one read. + /// Sources advance volatile read offsets through that prefix. The list is valid for the duration of the callback. + /// void MessagesAdded(IReadOnlyList messages) { } /// /// Notifies the source that records could not be admitted to the cache. /// /// The records which were not admitted. + /// + /// The list contains the remaining records from the read and is valid for the duration of the callback. + /// The next read resumes after the last successfully admitted prefix. + /// void MessagesAddFailed(IReadOnlyList messages) { } /// diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index 18e6848759a..b4909eaa9f0 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -19,8 +19,11 @@ public sealed class RecoverableStreamQueueCache : IRecoverableStr private readonly IEvictionStrategy _evictionStrategy; private readonly IQueueFlowController? _flowController; private readonly int? _maxCacheSize; + private readonly long _maxCacheSizeBytes; + private readonly Queue _messageSizes = new(); private readonly PooledQueueCache _cache; private FixedSizeBuffer? _currentBuffer; + private long? _pendingMessageSize; /// /// Initializes a new instance of the class. @@ -36,6 +39,52 @@ public RecoverableStreamQueueCache( TimeSpan? cacheMonitorWriteInterval = null, TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) + : this( + defaultMaxAddCount, + bufferPool, + dataAdapter, + evictionStrategy, + logger, + long.MaxValue, + flowController, + cacheMonitor, + cacheMonitorWriteInterval, + metadataMinTimeInCache, + maxCacheSize) + { + } + + /// + /// Initializes a cache with a per-partition encoded-data budget. + /// + /// The maximum number of records requested per read. + /// The pool which provides encoded-data buffers. + /// The source record adapter. + /// The cache eviction strategy. + /// The logger. + /// The positive budget for encoded segments belonging to retained records. + /// An optional additional flow controller. + /// An optional cache monitor. + /// The cache monitoring interval. + /// The minimum metadata retention interval. + /// An optional maximum retained record count. + /// + /// Records are admitted in order. An empty cache admits one record larger than the byte budget + /// so that delivery can make progress. Capacity planning also accounts for buffer slack and pool + /// retention, a fetched source batch, metadata, and deserialized delivery objects. + /// + public RecoverableStreamQueueCache( + int defaultMaxAddCount, + IObjectPool bufferPool, + IRecoverableStreamDataAdapter dataAdapter, + IEvictionStrategy evictionStrategy, + ILogger logger, + long maxCacheSizeBytes, + IQueueFlowController? flowController = null, + ICacheMonitor? cacheMonitor = null, + TimeSpan? cacheMonitorWriteInterval = null, + TimeSpan? metadataMinTimeInCache = null, + int? maxCacheSize = null) { if (defaultMaxAddCount <= 0) { @@ -52,6 +101,8 @@ public RecoverableStreamQueueCache( throw new ArgumentOutOfRangeException(nameof(maxCacheSize)); } + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxCacheSizeBytes); + _maxCacheSizeBytes = maxCacheSizeBytes; _maxCacheSize = maxCacheSize; _cache = new PooledQueueCache( dataAdapter, @@ -73,6 +124,11 @@ public RecoverableStreamQueueCache( /// public int ItemCount => _cache.ItemCount; + /// + /// Gets the encoded segment bytes belonging to records currently held in the cache. + /// + public long SizeInBytes { get; private set; } + /// /// Tries to get the newest cached provider position. /// @@ -93,26 +149,65 @@ public bool TryGetNewestPosition( } /// - /// Packs and adds ordered source records to the cache. + /// Packs and adds the ordered prefix of source records which fits the cache's capacity. /// + /// + /// The returned positions identify the admitted prefix. A packing exception rolls back the entire + /// attempted prefix. An empty cache admits one oversized record to permit delivery progress. + /// public IReadOnlyList Add( IReadOnlyList messages, DateTime dequeueTimeUtc) { ArgumentNullException.ThrowIfNull(messages); - var positions = new List(messages.Count); - var cachedMessages = new List(messages.Count); + if (IsUnderBytePressure()) + { + return []; + } + + var count = _maxCacheSize is { } maxCacheSize + ? Math.Min(messages.Count, maxCacheSize - _cache.ItemCount) + : messages.Count; + var positions = new List(count); + var cachedMessages = new List(count); + var messageSizes = new List(count); var allocatedBuffers = new List(); var initialBuffer = _currentBuffer; var initialBufferPosition = initialBuffer?.Position ?? 0; var batchBuffer = initialBuffer; + var batchSize = 0L; + var recordSize = 0L; + long? pendingMessageSize = null; try { - foreach (var message in messages) + for (var i = 0; i < count; i++) { - var position = _dataAdapter.GetStreamPosition(message); - cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); - positions.Add(position); + var recordBuffer = batchBuffer; + var recordBufferPosition = recordBuffer?.Position ?? 0; + var allocatedBufferCount = allocatedBuffers.Count; + recordSize = 0; + try + { + var message = messages[i]; + var position = _dataAdapter.GetStreamPosition(message); + cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); + positions.Add(position); + messageSizes.Add(recordSize); + batchSize += recordSize; + } + catch (ByteBudgetExceededException exception) + { + recordBuffer?.ResetTo(recordBufferPosition); + for (var j = allocatedBuffers.Count - 1; j >= allocatedBufferCount; j--) + { + allocatedBuffers[j].Dispose(); + allocatedBuffers.RemoveAt(j); + } + + batchBuffer = recordBuffer; + pendingMessageSize = exception.RequiredSize; + break; + } } } catch @@ -127,6 +222,13 @@ public IReadOnlyList Add( } _cache.Add(cachedMessages, dequeueTimeUtc); + foreach (var size in messageSizes) + { + _messageSizes.Enqueue(size); + } + + SizeInBytes += batchSize; + _pendingMessageSize = pendingMessageSize; foreach (var buffer in allocatedBuffers) { _evictionStrategy.OnBlockAllocated(buffer); @@ -137,6 +239,14 @@ public IReadOnlyList Add( ArraySegment GetBatchSegment(int size) { + var requiredSize = checked(recordSize + size); + if ((_cache.ItemCount > 0 || cachedMessages.Count > 0) + && requiredSize > _maxCacheSizeBytes - SizeInBytes - batchSize) + { + throw new ByteBudgetExceededException(requiredSize); + } + + recordSize = requiredSize; if (batchBuffer is not null && batchBuffer.TryGetSegment(size, out var segment)) { return segment; @@ -159,6 +269,11 @@ ArraySegment GetBatchSegment(int size) /// public int GetMaxAddCount() { + if (IsUnderBytePressure()) + { + return 0; + } + var result = _defaultMaxAddCount; if (_maxCacheSize is { } maxCacheSize) { @@ -177,6 +292,11 @@ public int GetMaxAddCount() return result; } + private bool IsUnderBytePressure() + => _cache.ItemCount > 0 + && (SizeInBytes >= _maxCacheSizeBytes + || _pendingMessageSize > _maxCacheSizeBytes - SizeInBytes); + /// public void AddToCache(IList messages) { @@ -193,10 +313,7 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList if (!IsUnderPressure()) { _evictionStrategy.PerformPurge(DateTime.UtcNow); - if (_cache.IsEmpty) - { - _currentBuffer = null; - } + UpdateByteAccounting(); } return false; @@ -234,11 +351,8 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke _cache.RemoveOldestMessage(); } + UpdateByteAccounting(); _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); - if (_cache.IsEmpty) - { - _currentBuffer = null; - } } /// @@ -253,19 +367,39 @@ public void Dispose() _cache.RemoveOldestMessage(); } + UpdateByteAccounting(); + _pendingMessageSize = null; _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); - _currentBuffer = null; _evictionStrategy.OnPurged = null; } private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) { + UpdateByteAccounting(); if (lastPurged is { } message) { LastPurgedOffset = _dataAdapter.GetOffset(ref message); } } + private void UpdateByteAccounting() + { + while (_messageSizes.Count > _cache.ItemCount) + { + SizeInBytes -= _messageSizes.Dequeue(); + } + + if (_cache.IsEmpty) + { + _currentBuffer = null; + } + } + + private sealed class ByteBudgetExceededException(long requiredSize) : Exception + { + public long RequiredSize { get; } = requiredSize; + } + private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress { private readonly PooledQueueCache _cache; diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 19967372d53..85441e14584 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -23,6 +23,9 @@ public sealed class RecoverableStreamReceiver : IQueueAdapterRece private readonly CancellationTokenSource _lifecycleCancellation = new(); private Task? _initializeTask; private CancellationToken _initializeTaskOwnerToken; + private TQueueMessage[]? _pendingMessages; + private int _pendingStart; + private DateTime _pendingDequeueTime; private int _running; private int _shutdown; @@ -156,24 +159,64 @@ public async Task> GetQueueMessagesAsync( return []; } - var messages = await _source.Read(maxCount, cancellationToken); - if (messages.Count == 0) + var readCount = Math.Min(maxCount, _cache.GetMaxAddCount()); + if (readCount <= 0) { return []; } + if (_pendingMessages is null) + { + using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, _lifecycleCancellation.Token); + var messages = await _source.Read(readCount, readCancellation.Token); + if (Volatile.Read(ref _shutdown) != 0 || messages.Count == 0) + { + return []; + } + + _pendingMessages = new TQueueMessage[messages.Count]; + for (var i = 0; i < messages.Count; i++) + { + _pendingMessages[i] = messages[i]; + } + + _pendingDequeueTime = DateTime.UtcNow; + } + + var pending = _pendingMessages; + var remaining = new ArraySegment(pending, _pendingStart, pending.Length - _pendingStart); + var attempted = remaining.Slice(0, Math.Min(readCount, remaining.Count)); IReadOnlyList positions; try { - positions = _cache.Add(messages, DateTime.UtcNow); - _source.MessagesAdded(messages); + positions = _cache.Add(attempted, _pendingDequeueTime); + if (positions.Count > 0) + { + _source.MessagesAdded(attempted.Slice(0, positions.Count)); + } } catch { - _source.MessagesAddFailed(messages); + try + { + _source.MessagesAddFailed(remaining); + } + finally + { + ClearPendingMessages(); + } + throw; } + Array.Clear(pending, _pendingStart, positions.Count); + _pendingStart += positions.Count; + if (_pendingStart == pending.Length) + { + ClearPendingMessages(); + } + var result = new List(positions.Count); foreach (var position in positions) { @@ -204,6 +247,7 @@ public async Task Shutdown(TimeSpan timeout) Volatile.Write(ref _running, 0); _lifecycleCancellation.Cancel(); + ClearPendingMessages(); using var cancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); @@ -318,6 +362,17 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke } } + private void ClearPendingMessages() + { + if (_pendingMessages is { } pending) + { + Array.Clear(pending, _pendingStart, pending.Length - _pendingStart); + } + + _pendingMessages = null; + _pendingStart = 0; + } + private sealed class StreamActivityNotificationBatch(StreamPosition position) : IBatchContainer { public StreamId StreamId => position.StreamId; diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs new file mode 100644 index 00000000000..a9fc42bbe85 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs @@ -0,0 +1,270 @@ +using System.Data; +using System.Reflection; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streaming.AdoNet; +using Orleans.Streaming.AdoNet.Storage; +using Orleans.Streams; +using TestExtensions; + +namespace Tester.AdoNet.Streaming; + +[TestSuite("BVT")] +[TestArea("Streaming")] +[TestProvider("None")] +[TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] +public sealed class RecoveryCacheMemoryOptionsTests +{ + [Fact] + public void AdoNet_EncodedCacheBudgetDefaultsTo64MiB() + { + var options = Options(); + + Assert.Equal(64L * 1024 * 1024, options.MaxCacheSizeBytes); + new AdoNetStreamOptionsValidator(options, "memory").ValidateConfiguration(); + } + + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + public void AdoNet_RejectsNonPositiveEncodedCacheBudget(long bytes) + { + var options = Options(); + options.MaxCacheSizeBytes = bytes; + + var exception = Assert.Throws( + () => new AdoNetStreamOptionsValidator(options, "memory").ValidateConfiguration()); + + Assert.Contains(nameof(AdoNetStreamOptions.MaxCacheSizeBytes), exception.Message); + Assert.Contains("memory", exception.Message); + } + + [Theory] + [InlineData(1L)] + [InlineData(4L * 1024 * 1024 * 1024)] + [InlineData(long.MaxValue)] + public void AdoNet_AcceptsPositiveInt64EncodedCacheBudget(long bytes) + { + var options = Options(); + options.MaxCacheSizeBytes = bytes; + + new AdoNetStreamOptionsValidator(options, "memory").ValidateConfiguration(); + + Assert.Equal(bytes, options.MaxCacheSizeBytes); + } + + [Fact] + public async Task AdoNet_PartitionsApplyConfiguredByteBudgetAndStageOneRead() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var options = Options(); + options.MaxCacheSizeBytes = 128 * 1024; + options.MaxMessagesPerRead = 8; + var firstEvent = new string('a', 192 * 1024); + var secondEvent = new string('b', 256 * 1024); + var storage = new MemoryStorage(); + var receivers = new List(); + foreach (var queue in new[] { "first", "second" }) + { + var stream = StreamId.Create("memory", queue); + var payloads = new[] + { + AdoNetBatchContainer.ToMessagePayload(serializer, stream, [firstEvent], null), + AdoNetBatchContainer.ToMessagePayload(serializer, stream, [secondEvent], null), + }; + Assert.All(payloads, payload => Assert.True(payload.LongLength > options.MaxCacheSizeBytes)); + storage.Partitions.Add(queue, (stream, payloads)); + receivers.Add(new AdoNetQueueAdapterReceiver( + "provider", + queue, + options, + new ClusterOptions { ServiceId = "service" }, + new SimpleQueueCacheOptions { CacheSize = 8 }, + CreateQueries(storage), + serializer, + NullLogger.Instance)); + } + + try + { + foreach (var receiver in receivers) + { + var first = Assert.Single(await receiver.GetQueueMessagesAsync(8, CancellationToken.None)); + Assert.Equal(1, first.SequenceToken.SequenceNumber); + AssertEvent(receiver, first, firstEvent); + Assert.Equal(0, receiver.GetMaxAddCount()); + Assert.True(receiver.IsUnderPressure()); + Assert.Empty(await receiver.GetQueueMessagesAsync(8, CancellationToken.None)); + } + + Assert.Equal(2, storage.ReadRequests.Count); + receivers[1].UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + Assert.Equal(8, receivers[1].GetMaxAddCount()); + var second = Assert.Single(await receivers[1].GetQueueMessagesAsync(8, CancellationToken.None)); + Assert.Equal(2, second.SequenceToken.SequenceNumber); + Assert.Equal(StreamId.Create("memory", "second"), second.StreamId); + AssertEvent(receivers[1], second, secondEvent); + Assert.Equal(2, storage.ReadRequests.Count); + Assert.True(receivers[0].IsUnderPressure()); + Assert.Empty(await receivers[0].GetQueueMessagesAsync(8, CancellationToken.None)); + + receivers[1].UpdateDeliveryProgress(new EventSequenceTokenV2(2), DateTime.UnixEpoch); + Assert.Empty(await receivers[1].GetQueueMessagesAsync(8, CancellationToken.None)); + Assert.Equal([0L, 2L], storage.ReadRequests + .Where(request => request.Queue == "second") + .Select(request => request.After)); + Assert.Equal([0L], storage.ReadRequests + .Where(request => request.Queue == "first") + .Select(request => request.After)); + Assert.All(storage.ReadRequests, request => Assert.Equal(8, request.Count)); + } + finally + { + foreach (var receiver in receivers) + { + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + } + } + + Assert.Equal(2, storage.Checkpoints["second"]); + Assert.False(storage.Checkpoints.ContainsKey("first")); + } + + private static AdoNetStreamOptions Options() + => new() + { + Invariant = AdoNetInvariants.InvariantNameSqlServer, + ConnectionString = "configured", + }; + + private static void AssertEvent(IQueueCache receiver, IBatchContainer notification, string expected) + { + using var cursor = receiver.GetCacheCursor(notification.StreamId, notification.SequenceToken); + Assert.True(cursor.MoveNext()); + var batch = cursor.GetCurrent(out var exception); + Assert.Null(exception); + var item = Assert.Single(Assert.IsType(batch).GetEvents()); + Assert.Equal(expected, item.Item1); + Assert.Equal(notification.SequenceToken.SequenceNumber, item.Item2.SequenceNumber); + } + + private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage) + { + var queries = typeof(DbStoredQueries) + .GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) + .ToDictionary(property => property.Name, property => + property.Name == nameof(DbStoredQueries.StreamSchemaVersionKey) ? "2" : property.Name); + return new(storage, new DbStoredQueries(queries)); + } + + private sealed class MemoryStorage : IRelationalStorage + { + public Dictionary Partitions { get; } = []; + public Dictionary Checkpoints { get; } = []; + public List<(string Queue, long After, int Count)> ReadRequests { get; } = []; + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; + public string ConnectionString => "configured"; + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + using var command = new SqlCommand(); + parameterProvider?.Invoke(command); + var queue = (string)command.Parameters["QueueId"].Value; + var partition = Partitions[queue]; + var rows = new List>(); + switch (query) + { + case nameof(DbStoredQueries.AcquireStreamPartitionKey): + case nameof(DbStoredQueries.GetStreamPartitionBoundsKey): + rows.Add(new() + { + ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, + ["OwnerEpoch"] = 1L, ["NextMessageId"] = (long)partition.Payloads.Length + 1, + ["Checkpoint"] = Checkpoints.GetValueOrDefault(queue), + ["EarliestMessageId"] = 1L, ["TailMessageId"] = (long)partition.Payloads.Length, + }); + break; + case nameof(DbStoredQueries.ReadStreamMessagesKey): + var after = (long)command.Parameters["AfterMessageId"].Value; + var count = (int)command.Parameters["MaxCount"].Value; + ReadRequests.Add((queue, after, count)); + for (var i = (int)after; i < Math.Min(partition.Payloads.Length, after + count); i++) + { + rows.Add(new() + { + ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, + ["MessageId"] = (long)i + 1, + ["StreamIdBytes"] = partition.Stream.FullKey.ToArray(), + ["StreamNamespaceLength"] = partition.Stream.Namespace.Length, + ["CreatedOn"] = DateTime.UnixEpoch, ["Payload"] = partition.Payloads[i], + }); + } + + break; + case nameof(DbStoredQueries.CleanupStreamMessagesKey): + rows.Add(new() + { + ["Ran"] = false, ["DeletedCount"] = 0, ["DeletedThroughMessageId"] = null, + ["HardDeletedCount"] = 0, ["HardDeletedFromMessageId"] = null, + ["HardDeletedThroughMessageId"] = null, ["Checkpoint"] = Checkpoints.GetValueOrDefault(queue), + ["EarliestMessageId"] = 1L, ["TailMessageId"] = (long)partition.Payloads.Length, + }); + break; + case nameof(DbStoredQueries.AdvanceStreamCheckpointKey): + var checkpoint = (long)command.Parameters["Checkpoint"].Value; + Checkpoints[queue] = checkpoint; + rows.Add(new() + { + ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, + ["OwnerEpoch"] = 1L, ["Checkpoint"] = checkpoint, ["Updated"] = true, + }); + break; + default: + throw new InvalidOperationException($"Unexpected query: {query}"); + } + + var result = new List(); + if (rows.Count > 0) + { + using var table = new DataTable(); + foreach (var name in rows[0].Keys) + { + table.Columns.Add(name, typeof(object)); + } + + foreach (var row in rows) + { + table.Rows.Add(table.Columns.Cast() + .Select(column => row[column.ColumnName] ?? DBNull.Value).ToArray()); + } + + using var reader = table.CreateDataReader(); + while (reader.Read()) + { + result.Add(await selector(reader, 0, cancellationToken)); + } + } + + return result; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + => throw new InvalidOperationException($"Unexpected command: {query}"); + } +} diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/RecoveryCacheMemoryOptionsTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/RecoveryCacheMemoryOptionsTests.cs new file mode 100644 index 00000000000..c3b68ad12f5 --- /dev/null +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/RecoveryCacheMemoryOptionsTests.cs @@ -0,0 +1,48 @@ +using Orleans.Runtime; +using TestExtensions; +using Xunit; + +namespace Orleans.Streaming.Kinesis.Tests; + +[TestSuite("BVT")] +[TestArea("Streaming")] +[TestProvider("Kinesis")] +[TestCategory("BVT"), TestCategory("Kinesis")] +public sealed class RecoveryCacheMemoryOptionsTests +{ + [Fact] + public void Kinesis_EncodedCacheBudgetDefaultsTo64MiB() + { + var options = new KinesisStreamOptions(); + + Assert.Equal(64L * 1024 * 1024, options.MaxCacheSizeBytes); + new KinesisStreamOptionsValidator(options, "memory").ValidateConfiguration(); + } + + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + public void Kinesis_RejectsNonPositiveEncodedCacheBudget(long bytes) + { + var options = new KinesisStreamOptions { MaxCacheSizeBytes = bytes }; + + var exception = Assert.Throws( + () => new KinesisStreamOptionsValidator(options, "memory").ValidateConfiguration()); + + Assert.Contains(nameof(KinesisStreamOptions.MaxCacheSizeBytes), exception.Message); + Assert.Contains("memory", exception.Message); + } + + [Theory] + [InlineData(1L)] + [InlineData(4L * 1024 * 1024 * 1024)] + [InlineData(long.MaxValue)] + public void Kinesis_AcceptsPositiveInt64EncodedCacheBudget(long bytes) + { + var options = new KinesisStreamOptions { MaxCacheSizeBytes = bytes }; + + new KinesisStreamOptionsValidator(options, "memory").ValidateConfiguration(); + + Assert.Equal(bytes, options.MaxCacheSizeBytes); + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs new file mode 100644 index 00000000000..f2da41d49cc --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs @@ -0,0 +1,665 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streams; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +[TestCategory("BVT")] +public sealed class RecoveryCacheMemoryTests +{ + private static readonly StreamId _stream = StreamId.Create("memory", "partition"); + + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + public void Cache_RequiresPositiveByteBudget(long budget) + { + var exception = Assert.Throws(() => CreateCache(budget)); + Assert.Equal("maxCacheSizeBytes", exception.ParamName); + } + + [Fact] + public void Cache_ByteBudgetBlocksOrderedSuffixWithoutRepeatedPacking() + { + var adapter = new DataAdapter(); + var pool = new BufferPool(128); + using var cache = CreateCache(80, adapter, pool); + var messages = new[] { Record(1, 24), Record(2, 40), Record(3, 24) }; + + var admitted = cache.Add(messages, DateTime.UnixEpoch); + + Assert.Equal([1L, 2L], Sequences(admitted)); + Assert.Equal(64, cache.SizeInBytes); + Assert.Equal(2, cache.ItemCount); + Assert.Equal(0, cache.GetMaxAddCount()); + Assert.True(cache.IsUnderPressure()); + Assert.Equal(3, adapter.PackCount); + Assert.Equal(1, pool.AllocationCount); + for (var i = 0; i < 3; i++) + { + Assert.Empty(cache.Add([messages[2]], DateTime.UnixEpoch)); + Assert.False(cache.TryPurgeFromCache(out _)); + } + + Assert.Equal(3, adapter.PackCount); + Assert.Equal(1, pool.AllocationCount); + Assert.Equal(64, cache.SizeInBytes); + AssertPayload(cache, messages[0]); + AssertPayload(cache, messages[1]); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + + Assert.Equal(40, cache.SizeInBytes); + Assert.Equal(1000, cache.GetMaxAddCount()); + Assert.Equal([3L], Sequences(cache.Add([messages[2]], DateTime.UnixEpoch))); + Assert.Equal(64, cache.SizeInBytes); + AssertPayload(cache, messages[1]); + AssertPayload(cache, messages[2]); + } + + [Fact] + public void Cache_PartialWatermarkKeepsPressureUntilNextRecordFits() + { + using var cache = CreateCache(80); + var messages = new[] { Record(1, 24), Record(2, 40), Record(3, 64) }; + Assert.Equal([1L, 2L], Sequences(cache.Add(messages, DateTime.UnixEpoch))); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + + Assert.Equal(40, cache.SizeInBytes); + Assert.Equal(0, cache.GetMaxAddCount()); + Assert.Empty(cache.Add([messages[2]], DateTime.UnixEpoch)); + AssertPayload(cache, messages[1]); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(2), DateTime.UnixEpoch); + + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(1000, cache.GetMaxAddCount()); + Assert.Equal([3L], Sequences(cache.Add([messages[2]], DateTime.UnixEpoch))); + Assert.Equal(64, cache.SizeInBytes); + AssertPayload(cache, messages[2]); + } + + [Fact] + public void Cache_RecordCapacityAlsoBoundsAdmission() + { + using var cache = CreateCache(128, maxCount: 1); + var messages = new[] { Record(1, 16), Record(2, 16) }; + + Assert.Equal([1L], Sequences(cache.Add(messages, DateTime.UnixEpoch))); + Assert.Equal(16, cache.SizeInBytes); + Assert.Equal(0, cache.GetMaxAddCount()); + Assert.Empty(cache.Add([messages[1]], DateTime.UnixEpoch)); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + + Assert.Equal(1, cache.GetMaxAddCount()); + Assert.Equal([2L], Sequences(cache.Add([messages[1]], DateTime.UnixEpoch))); + AssertPayload(cache, messages[1]); + } + + [Fact] + public void Cache_EmptyCacheAdmitsOneOversizedRecordThenWaitsForDelivery() + { + var pool = new BufferPool(32); + var eviction = new EvictionStrategy(); + using var cache = CreateCache(16, pool: pool, eviction: eviction); + var large = Record(1, 96); + var small = Record(2, 16); + + Assert.Equal([1L], Sequences(cache.Add([large, small], DateTime.UnixEpoch))); + Assert.Equal(96, cache.SizeInBytes); + Assert.Equal(96, eviction.OwnedBytes); + Assert.Equal(1, eviction.OwnedCount); + Assert.Equal(0, pool.OwnedCount); + Assert.Equal(0, cache.GetMaxAddCount()); + AssertPayload(cache, large); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(0, eviction.OwnedCount); + Assert.Equal([2L], Sequences(cache.Add([small], DateTime.UnixEpoch))); + Assert.Equal(16, cache.SizeInBytes); + Assert.Equal(32, eviction.OwnedBytes); + AssertPayload(cache, small); + } + + [Fact] + public void Cache_FailedPackingRestoresBytesAndPartiallyUsedBuffer() + { + var pool = new BufferPool(64); + var adapter = new DataAdapter(); + var eviction = new EvictionStrategy(); + using var cache = CreateCache(200, adapter, pool, eviction); + var first = Record(1, 16); + _ = cache.Add([first], DateTime.UnixEpoch); + adapter.FailSequence = 3; + + Assert.Throws(() => cache.Add( + [Record(2, 32), Record(3, 64)], DateTime.UnixEpoch)); + + Assert.Equal(16, cache.SizeInBytes); + Assert.Equal(1, cache.ItemCount); + Assert.Equal(2, pool.AllocationCount); + Assert.Equal(1, pool.FreeCount); + Assert.Equal(1, pool.OwnedCount); + Assert.Equal(1, eviction.OwnedCount); + Assert.Equal(1000, cache.GetMaxAddCount()); + AssertPayload(cache, first); + + adapter.FailSequence = null; + var retry = Record(2, 48); + Assert.Equal([2L], Sequences(cache.Add([retry], DateTime.UnixEpoch))); + Assert.Equal(64, cache.SizeInBytes); + Assert.Equal(2, pool.AllocationCount); + AssertPayload(cache, first); + AssertPayload(cache, retry); + } + + [Fact] + public void Cache_ChronologicalPurgeReleasesByteAccounting() + { + var pool = new BufferPool(128); + using var cache = CreateCache(256, pool: pool, eviction: new EvictionStrategy(TimeSpan.Zero)); + _ = cache.Add([Record(1, 32)], DateTime.UnixEpoch); + var retained = Record(2, 64); + _ = cache.Add([retained], DateTime.UnixEpoch.AddSeconds(1)); + + Assert.False(cache.TryPurgeFromCache(out _)); + + Assert.Equal(1, cache.ItemCount); + Assert.Equal(64, cache.SizeInBytes); + Assert.Equal("1", cache.LastPurgedOffset); + Assert.Equal(1, pool.OwnedCount); + AssertPayload(cache, retained); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(2), DateTime.UnixEpoch.AddSeconds(2)); + + Assert.Equal(0, cache.ItemCount); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal("2", cache.LastPurgedOffset); + Assert.Equal(0, pool.OwnedCount); + Assert.Equal(1, pool.FreeCount); + } + + [Fact] + public void Cache_MultiplePartitionsRespectIndependentByteBudgets() + { + const int mebibyte = 1024 * 1024; + const int budget = 2 * mebibyte; + var caches = Enumerable.Range(0, 3) + .Select(_ => CreateCache(budget, pool: new BufferPool(mebibyte))) + .ToArray(); + var records = Enumerable.Range(0, caches.Length) + .Select(partition => new[] + { + Record(1, mebibyte / 4, partition), + Record(2, mebibyte + mebibyte / 4, partition), + Record(3, mebibyte / 4, partition), + Record(4, mebibyte / 2, partition), + }) + .ToArray(); + try + { + for (var i = 0; i < caches.Length; i++) + { + Assert.Equal([1L, 2L, 3L], Sequences(caches[i].Add(records[i], DateTime.UnixEpoch))); + Assert.Equal(7L * mebibyte / 4, caches[i].SizeInBytes); + Assert.True(caches[i].IsUnderPressure()); + AssertPayload(caches[i], records[i][1]); + } + + Assert.Equal(21L * mebibyte / 4, caches.Sum(cache => cache.SizeInBytes)); + for (var i = 1; i < caches.Length; i++) + { + caches[i].UpdateDeliveryProgress(new EventSequenceTokenV2(3), DateTime.UnixEpoch); + Assert.Equal([4L], Sequences(caches[i].Add([records[i][3]], DateTime.UnixEpoch))); + Assert.Equal(mebibyte / 2, caches[i].SizeInBytes); + Assert.False(caches[i].IsUnderPressure()); + AssertPayload(caches[i], records[i][3]); + } + + Assert.Equal(11L * mebibyte / 4, caches.Sum(cache => cache.SizeInBytes)); + Assert.True(caches[0].IsUnderPressure()); + Assert.Equal(3, caches[0].ItemCount); + AssertPayload(caches[0], records[0][0]); + AssertPayload(caches[0], records[0][2]); + } + finally + { + foreach (var cache in caches) + { + cache.Dispose(); + } + } + + Assert.All(caches, cache => Assert.Equal(0, cache.SizeInBytes)); + } + + [Fact] + public void Cache_RepeatedFillDrainReusesBuffersAndPreservesDeliveredCopies() + { + var pool = new BufferPool(64); + var eviction = new EvictionStrategy(); + using var cache = CreateCache(64, pool: pool, eviction: eviction); + var snapshots = new List<(TestRecord Record, byte[] Payload)>(); + for (var cycle = 0; cycle < 5; cycle++) + { + var first = Record(cycle * 2 + 1, 16); + var second = Record(cycle * 2 + 2, 24); + _ = cache.Add([first, second], DateTime.UnixEpoch); + Assert.Equal(40, cache.SizeInBytes); + using var cursor = cache.GetCacheCursor(first.StreamId, new EventSequenceTokenV2(first.Sequence)); + Assert.True(cursor.MoveNext()); + snapshots.Add((first, Assert.IsType(cursor.GetCurrent(out _)).Payload)); + + cache.UpdateDeliveryProgress(new EventSequenceTokenV2(second.Sequence), DateTime.UnixEpoch); + + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(0, pool.OwnedCount); + Assert.Equal(0, eviction.OwnedCount); + } + + Assert.Equal(1, pool.CreatedCount); + Assert.Equal(5, pool.AllocationCount); + Assert.Equal(5, pool.FreeCount); + foreach (var (record, payload) in snapshots) + { + Assert.Equal(record.Payload, payload); + } + } + + [Fact] + public async Task Receiver_StagesOneBatchAndAcknowledgesOnlyAdmittedPrefixes() + { + var source = new Source([40, 40, 24, 32]); + var adapter = new DataAdapter(); + using var cache = CreateCache(64, adapter); + var checkpoint = new Checkpointer(); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, checkpoint, false); + + Assert.Equal([1L], Sequences(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None))); + var staging = Assert.Single(source.AdmissionArrays); + Assert.Null(staging[0]); + Assert.Equal([2L, 3L, 4L], staging.Skip(1).Select(record => record.Sequence)); + Assert.Equal(1, source.ReadCount); + Assert.Equal(1, source.ReadOffset); + Assert.Empty(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None)); + Assert.Equal(1, source.ReadCount); + + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + Assert.Equal([2L, 3L], Sequences(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None))); + Assert.Equal(64, cache.SizeInBytes); + Assert.Equal(3, source.ReadOffset); + Assert.Equal("1", checkpoint.Offset); + Assert.All(staging.Take(3), Assert.Null); + Assert.Equal(4, staging[3].Sequence); + + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(2), DateTime.UnixEpoch); + Assert.Equal([4L], Sequences(await receiver.GetQueueMessagesAsync(1, CancellationToken.None))); + + Assert.Equal(56, cache.SizeInBytes); + Assert.Equal(1, source.ReadCount); + Assert.Equal([1000], source.RequestedCounts); + Assert.Equal([1L, 2L, 3L, 4L], source.Admitted.SelectMany(batch => batch)); + Assert.Equal([1, 2, 1], source.Admitted.Select(batch => batch.Length)); + Assert.Equal(4, source.ReadOffset); + Assert.Equal("2", checkpoint.Offset); + Assert.All(staging, Assert.Null); + Assert.Empty(source.Failed); + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + } + + [Fact] + public async Task Receiver_StagedPackingFailureClearsSuffixAndReplaysAfterAdmittedOffset() + { + var source = new Source([40, 40, 16, 32]); + var adapter = new DataAdapter(); + var pool = new BufferPool(64); + using var cache = CreateCache(64, adapter, pool); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, new Checkpointer(), false); + Assert.Equal([1L], Sequences(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None))); + var firstStaging = Assert.Single(source.AdmissionArrays); + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UnixEpoch); + adapter.FailSequence = 3; + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(1000, CancellationToken.None)); + + Assert.Equal([2L, 3L, 4L], Assert.Single(source.Failed)); + Assert.All(firstStaging, Assert.Null); + Assert.Equal(1, source.ReadOffset); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(0, pool.OwnedCount); + Assert.Equal(1000, receiver.GetMaxAddCount()); + + adapter.FailSequence = null; + Assert.Equal([2L, 3L], Sequences(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None))); + Assert.Equal(2, source.ReadCount); + Assert.Equal(56, cache.SizeInBytes); + AssertPayload(cache, Record(2, 40)); + AssertPayload(cache, Record(3, 16)); + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(3), DateTime.UnixEpoch); + Assert.Equal([4L], Sequences(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None))); + Assert.Equal([1L, 2L, 3L, 4L], source.Admitted.SelectMany(batch => batch)); + Assert.Equal(2, source.ReadCount); + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Receiver_ShutdownClearsPendingPayloadRootsAndCacheOwnership(bool failShutdown) + { + var source = new Source([40, 40]) { FailShutdown = failShutdown }; + var adapter = new DataAdapter(); + var pool = new BufferPool(64); + var eviction = new EvictionStrategy(); + using var cache = CreateCache(64, adapter, pool, eviction); + var checkpoint = new Checkpointer(); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, checkpoint, false); + Assert.Single(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None)); + var staging = Assert.Single(source.AdmissionArrays); + Assert.Null(staging[0]); + Assert.Equal(2, staging[1].Sequence); + + if (failShutdown) + { + await Assert.ThrowsAsync(() => receiver.Shutdown(Timeout.InfiniteTimeSpan)); + } + else + { + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + } + + Assert.All(staging, Assert.Null); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(0, pool.OwnedCount); + Assert.Equal(0, eviction.OwnedCount); + Assert.Equal(1, checkpoint.FlushCount); + Assert.True(source.Stopped); + Assert.Empty(await receiver.GetQueueMessagesAsync(1000, CancellationToken.None)); + Assert.Equal(1, source.ReadCount); + } + + [Fact] + public async Task Receiver_ShutdownPreventsLateSourceBatchFromRepopulatingCache() + { + var source = new DelayedSource(); + var adapter = new DataAdapter(); + var pool = new BufferPool(64); + using var cache = CreateCache(64, adapter, pool); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, new Checkpointer(), false); + var read = receiver.GetQueueMessagesAsync(1000, CancellationToken.None); + await source.Started.Task; + + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + Assert.True(source.ReadCancellation.IsCancellationRequested); + source.Completion.SetResult([Record(1, 40), Record(2, 40)]); + + Assert.Empty(await read); + Assert.Equal(0, adapter.PackCount); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(0, pool.AllocationCount); + Assert.Equal(0, source.AdmissionCount); + } + + [Fact] + public async Task Receiver_AdmitsFullReadWhenByteBudgetFits() + { + var source = new Source(Enumerable.Repeat(16, 1000).ToArray()); + var adapter = new DataAdapter(); + var pool = new BufferPool(16_000); + using var cache = CreateCache(16_000, adapter, pool); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, new Checkpointer(), false); + + var notifications = await receiver.GetQueueMessagesAsync(1000, CancellationToken.None); + + Assert.Equal(Enumerable.Range(1, 1000).Select(sequence => (long)sequence), Sequences(notifications)); + Assert.Equal(16_000, cache.SizeInBytes); + Assert.Equal(1000, adapter.PackCount); + Assert.Equal(1, source.ReadCount); + Assert.Equal(1000, Assert.Single(source.Admitted).Length); + Assert.Equal(1, pool.AllocationCount); + Assert.All(Assert.Single(source.AdmissionArrays), Assert.Null); + await receiver.Shutdown(Timeout.InfiniteTimeSpan); + } + + private static RecoverableStreamQueueCache CreateCache( + long budget, + DataAdapter? adapter = null, + BufferPool? pool = null, + EvictionStrategy? eviction = null, + int maxCount = 4096) + => new( + 1000, + pool ?? new BufferPool(128), + adapter ?? new DataAdapter(), + eviction ?? new EvictionStrategy(), + NullLogger.Instance, + maxCacheSizeBytes: budget, + maxCacheSize: maxCount); + + private static TestRecord Record(long sequence, int encodedSize, int partition = 0) + { + var payload = new byte[encodedSize - sizeof(int)]; + payload.AsSpan().Fill((byte)(sequence % 251)); + var stream = partition == 0 ? _stream : StreamId.Create("memory", $"partition-{partition}"); + return new(stream, sequence, payload); + } + + private static IEnumerable Sequences(IReadOnlyList positions) + => positions.Select(position => position.SequenceToken.SequenceNumber); + + private static IEnumerable Sequences(IList batches) + => batches.Select(batch => batch.SequenceToken.SequenceNumber); + + private static void AssertPayload(RecoverableStreamQueueCache cache, TestRecord record) + { + using var cursor = cache.GetCacheCursor(record.StreamId, new EventSequenceTokenV2(record.Sequence)); + Assert.True(cursor.MoveNext()); + var batch = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); + Assert.Equal(record.StreamId, batch.StreamId); + Assert.Equal(record.Sequence, batch.SequenceToken.SequenceNumber); + Assert.Equal(record.Payload, batch.Payload); + } + + private sealed record TestRecord(StreamId StreamId, long Sequence, byte[] Payload); + + private sealed class DataAdapter : IRecoverableStreamDataAdapter + { + public int PackCount { get; private set; } + public long? FailSequence { get; set; } + + public StreamPosition GetStreamPosition(TestRecord message) + => new(message.StreamId, new EventSequenceTokenV2(message.Sequence)); + + public CachedMessage FromQueueMessage( + StreamPosition position, + TestRecord message, + DateTime dequeueTimeUtc, + Func> getSegment) + { + PackCount++; + var segment = getSegment(SegmentBuilder.CalculateAppendSize(message.Payload)); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, message.Payload); + if (message.Sequence == FailSequence) + { + throw new InvalidOperationException("Packing failed after writing the record."); + } + + return new() + { + StreamId = position.StreamId, + SequenceNumber = message.Sequence, + Segment = segment, + EnqueueTimeUtc = dequeueTimeUtc, + DequeueTimeUtc = dequeueTimeUtc, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage message) + { + var offset = 0; + return new Batch( + message.StreamId, + GetSequenceToken(ref message), + SegmentBuilder.ReadNextBytes(message.Segment, ref offset).ToArray()); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage message) + => new EventSequenceTokenV2(message.SequenceNumber, message.EventIndex); + + public string GetOffset(ref CachedMessage message) + => message.SequenceNumber.ToString(CultureInfo.InvariantCulture); + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return true; + } + } + + private sealed class Batch(StreamId streamId, StreamSequenceToken token, byte[] payload) : IBatchContainer + { + public StreamId StreamId => streamId; + public StreamSequenceToken SequenceToken => token; + public byte[] Payload => payload; + public IEnumerable> GetEvents() => []; + public bool ImportRequestContext() => false; + } + + private sealed class Source(int[] encodedSizes) : IRecoverableStreamSource + { + public List RequestedCounts { get; } = []; + public List Admitted { get; } = []; + public List Failed { get; } = []; + public List AdmissionArrays { get; } = []; + public long ReadOffset { get; private set; } + public int ReadCount { get; private set; } + public bool Stopped { get; private set; } + public bool FailShutdown { get; init; } + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + ReadCount++; + RequestedCounts.Add(maxCount); + return Task.FromResult>(encodedSizes + .Select((size, index) => (size, sequence: index + 1)) + .Where(item => item.sequence > ReadOffset) + .Take(maxCount) + .Select(item => Record(item.sequence, item.size)) + .ToArray()); + } + + public void MessagesAdded(IReadOnlyList messages) + { + Admitted.Add(messages.Select(message => message.Sequence).ToArray()); + AdmissionArrays.Add(Assert.IsType>(messages).Array!); + ReadOffset = messages[^1].Sequence; + } + + public void MessagesAddFailed(IReadOnlyList messages) + => Failed.Add(messages.Select(message => message.Sequence).ToArray()); + + public Task Shutdown(CancellationToken cancellationToken) + { + Stopped = true; + return FailShutdown ? Task.FromException(new InvalidOperationException("Shutdown failed.")) : Task.CompletedTask; + } + } + + private sealed class DelayedSource : IRecoverableStreamSource + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken ReadCancellation { get; private set; } + public int AdmissionCount { get; private set; } + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + ReadCancellation = cancellationToken; + Started.SetResult(); + return Completion.Task; + } + + public void MessagesAdded(IReadOnlyList messages) => AdmissionCount++; + public Task Shutdown(CancellationToken cancellationToken) => Task.CompletedTask; + } + + private sealed class Checkpointer : IStreamQueueCheckpointer + { + public bool CheckpointExists => false; + public string? Offset { get; private set; } + public int FlushCount { get; private set; } + public Task Load() => Task.FromResult(string.Empty); + public Task Load(CancellationToken cancellationToken) => Load(); + public void Update(string offset, DateTime utcNow) => Offset = offset; + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) => Offset = offset; + public Task FlushAsync(CancellationToken cancellationToken) + { + FlushCount++; + return Task.CompletedTask; + } + } + + private sealed class BufferPool(int bufferSize) : IObjectPool + { + private readonly Stack _available = new(); + private readonly HashSet _owned = []; + public int CreatedCount { get; private set; } + public int AllocationCount { get; private set; } + public int FreeCount { get; private set; } + public int OwnedCount => _owned.Count; + + public FixedSizeBuffer Allocate() + { + AllocationCount++; + if (!_available.TryPop(out var buffer)) + { + buffer = new FixedSizeBuffer(bufferSize); + CreatedCount++; + } + + Assert.True(_owned.Add(buffer)); + buffer.Pool = this; + return buffer; + } + + public void Free(FixedSizeBuffer resource) + { + Assert.True(_owned.Remove(resource)); + FreeCount++; + _available.Push(resource); + } + } + + private sealed class EvictionStrategy(TimeSpan? retention = null) + : ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(retention ?? TimeSpan.MaxValue, retention ?? TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null) + { + public int OwnedCount => inUseBuffers.Count; + public long OwnedBytes => inUseBuffers.Sum(buffer => (long)buffer.SizeInByte); + } +} From 917b95c0ef8616f1d6ea318ee73865e95a54ee07 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:30:23 -0700 Subject: [PATCH 54/90] test(streaming): cover production provider recovery paths --- .../Streams/KinesisAdapterFactory.cs | 10 +- .../Streams/KinesisBatchContainer.cs | 15 +- .../Orleans.Streaming.csproj | 1 + ...ableStreamQueueCheckpointerAzuriteTests.cs | 296 +++++++ .../KinesisPooledRuntimeTests.cs | 773 ++++++++++++++++++ 5 files changed, 1089 insertions(+), 6 deletions(-) create mode 100644 test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs create mode 100644 test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 54766720a12..badc87e707d 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -31,6 +31,7 @@ internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQue private readonly ILogger _logger; private readonly Func _queueMapperFactory; private readonly IAmazonKinesis _client; + private readonly Func _clientFactory; private readonly TimeProvider _timeProvider; private readonly SemaphoreSlim _initializationSemaphore = new(1); @@ -44,7 +45,8 @@ public KinesisAdapterFactory( Serializer serializer, IStreamQueueCheckpointerFactory? checkpointerFactory, ILoggerFactory loggerFactory, - TimeProvider? timeProvider = null + TimeProvider? timeProvider = null, + Func? clientFactory = null ) { _options = options ?? throw new ArgumentNullException(nameof(options)); @@ -56,6 +58,7 @@ public KinesisAdapterFactory( _cacheOptions = cacheOptions; _logger = loggerFactory.CreateLogger(); _timeProvider = timeProvider ?? TimeProvider.System; + _clientFactory = clientFactory ?? (() => CreateClient(_options)); _queueMapperFactory = partitions => new HashRingBasedPartitionedStreamQueueMapper(partitions, Name); _client = CreateClient(); @@ -183,7 +186,8 @@ private KinesisPooledAdapterReceiver MakeReceiver( topologyMonitor, _options.GetRecordsInterval, _timeProvider, - receiver => receivers.Remove(queueId, receiver) + receiver => receivers.Remove(queueId, receiver), + maxCacheSizeBytes: _options.MaxCacheSizeBytes ); } @@ -192,7 +196,7 @@ private InitializationState GetInitializationState() ?? throw new InvalidOperationException( $"The Kinesis stream provider '{Name}' must complete {nameof(CreateAdapter)} before accessing initialized adapter state."); - internal IAmazonKinesis CreateClient() => CreateClient(_options); + internal IAmazonKinesis CreateClient() => _clientFactory(); internal static IAmazonKinesis CreateClient(KinesisStreamOptions options) { diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 99be23cfb43..0a658e0683e 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -76,13 +76,22 @@ internal KinesisBatchContainer(Serializer serializer private Body GetPayload() => _payload ??= this.Serializer.Deserialize(_rawRecord)!; /// - /// Gets events of a specific type from the batch. + /// Gets events of a specific type with their original event indices within the record. /// /// - /// + /// The matching events and their sequence tokens. public IEnumerable> GetEvents() { - return GetPayload().Events.OfType().Select((e, i) => Tuple.Create(e, new KinesisSequenceToken(Token.ShardSequence, Token.SequenceNumber, i))); + var events = GetPayload().Events; + for (var i = 0; i < events.Count; i++) + { + if (events[i] is T item) + { + yield return Tuple.Create( + item, + new KinesisSequenceToken(Token.ShardSequence, Token.SequenceNumber, i)); + } + } } /// diff --git a/src/Orleans.Streaming/Orleans.Streaming.csproj b/src/Orleans.Streaming/Orleans.Streaming.csproj index 5220996afe7..c2a05f3d348 100644 --- a/src/Orleans.Streaming/Orleans.Streaming.csproj +++ b/src/Orleans.Streaming/Orleans.Streaming.csproj @@ -21,6 +21,7 @@ + diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs new file mode 100644 index 00000000000..fa8c3a28f4d --- /dev/null +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs @@ -0,0 +1,296 @@ +using System.Collections.Concurrent; +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Data.Tables; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Configuration; +using Orleans.Streams; +using TestExtensions; +using Tester.AzureUtils; +using Xunit; + +namespace ServiceBus.Tests.CheckpointerTests; + +[TestSuite("Functional")] +[TestProvider("EventHub")] +[TestArea("Streaming")] +[TestCategory("EventHub"), TestCategory("AzureStorage"), TestCategory("Streaming")] +public sealed class AzureTableStreamQueueCheckpointerAzuriteTests +{ + private static CancellationToken TestCancellation => TestContext.Current.CancellationToken; + private static readonly DateTime UpdateTime = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TwoCheckpointers_InitialInsertRace_PreservesMonotonicCheckpoint(bool higherWinsInsert) + { + await using var fixture = new Fixture(); + var lowRequests = new TableRequests(fixture.TableName, holdInsert: true); + var highRequests = new TableRequests(fixture.TableName, holdInsert: true); + var low = await fixture.CreateCheckpointer(lowRequests); + var high = await fixture.CreateCheckpointer(highRequests); + Assert.Equal(string.Empty, await low.Load(TestCancellation)); + Assert.Equal(string.Empty, await high.Load(TestCancellation)); + + low.Update("9", UpdateTime, TestCancellation); + high.Update("10", UpdateTime, TestCancellation); + try + { + await Task.WhenAll( + lowRequests.InsertStarted.Task.WaitAsync(TestCancellation), + highRequests.InsertStarted.Task.WaitAsync(TestCancellation)); + var winnerRequests = higherWinsInsert ? highRequests : lowRequests; + var loserRequests = higherWinsInsert ? lowRequests : highRequests; + var winner = higherWinsInsert ? high : low; + var loser = higherWinsInsert ? low : high; + + winnerRequests.ReleaseInsert.TrySetResult(); + await winner.FlushAsync(TestCancellation); + loserRequests.ReleaseInsert.TrySetResult(); + await loser.FlushAsync(TestCancellation); + + Assert.Equal(204, Assert.Single(winnerRequests.Inserts).Status); + Assert.Equal(409, Assert.Single(loserRequests.Inserts).Status); + if (higherWinsInsert) + { + Assert.Empty(loserRequests.Updates); + } + else + { + var retry = Assert.Single(loserRequests.Updates); + Assert.Equal(204, retry.Status); + Assert.Equal(Assert.Single(winnerRequests.Inserts).ETag, retry.IfMatch); + } + + await fixture.AssertCheckpoint("10"); + var reconciled = await loser.Store.Load(TestCancellation); + Assert.Equal("10", reconciled.Checkpoint); + Assert.False(string.IsNullOrEmpty(reconciled.Version)); + Assert.NotEqual("*", reconciled.Version); + } + finally + { + lowRequests.ReleaseInsert.TrySetResult(); + highRequests.ReleaseInsert.TrySetResult(); + await Task.WhenAll(low.FlushAsync(TestCancellation), high.FlushAsync(TestCancellation)); + } + } + + [Fact] + public async Task TwoCheckpointers_StaleETagReloadsAndRetriesHigherNumericCheckpoint() + { + await using var fixture = new Fixture(); + var winner = await fixture.CreateCheckpointer(); + Assert.Equal(string.Empty, await winner.Load(TestCancellation)); + await Persist(winner, "8"); + var initial = await fixture.ReadEntity(); + var staleRequests = new TableRequests(fixture.TableName); + var stale = await fixture.CreateCheckpointer(staleRequests); + Assert.Equal("8", await stale.Load(TestCancellation)); + await Persist(winner, "10"); + var current = await fixture.ReadEntity(); + Assert.NotEqual(initial.ETag, current.ETag); + + await Persist(stale, "9"); + + var conflict = Assert.Single(staleRequests.Updates); + Assert.Equal(412, conflict.Status); + Assert.Equal(initial.ETag.ToString(), conflict.IfMatch); + Assert.Contains(staleRequests.Reads, request => request.Status == 200 && request.ETag == current.ETag.ToString()); + await fixture.AssertCheckpoint("10"); + + await Persist(stale, "11"); + + Assert.Collection(staleRequests.Updates, + request => Assert.Equal(412, request.Status), + request => + { + Assert.Equal(204, request.Status); + Assert.Equal(current.ETag.ToString(), request.IfMatch); + }); + await fixture.AssertCheckpoint("11"); + } + + [Fact] + public async Task TwoCheckpointers_StaleETagRetriesPendingHigherCheckpointWithinSameFlush() + { + await using var fixture = new Fixture(); + var first = await fixture.CreateCheckpointer(); + await first.Load(TestCancellation); + await Persist(first, "8"); + var requests = new TableRequests(fixture.TableName); + var second = await fixture.CreateCheckpointer(requests); + Assert.Equal("8", await second.Load(TestCancellation)); + await Persist(first, "9"); + var current = await fixture.ReadEntity(); + + await Persist(second, "10"); + + Assert.Collection(requests.Updates, + request => Assert.Equal(412, request.Status), + request => + { + Assert.Equal(204, request.Status); + Assert.Equal(current.ETag.ToString(), request.IfMatch); + }); + await fixture.AssertCheckpoint("10"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TwoCheckpointers_DeleteRecreateInvalidatesCachedETagAndRecovers(bool recreateBeforeUpdate) + { + await using var fixture = new Fixture(); + var oldRequests = new TableRequests(fixture.TableName); + var old = await fixture.CreateCheckpointer(oldRequests); + await old.Load(TestCancellation); + await Persist(old, "8"); + var initial = await fixture.ReadEntity(); + await fixture.Table.DeleteEntityAsync(initial.PartitionKey, initial.RowKey, initial.ETag, TestCancellation); + if (recreateBeforeUpdate) + { + var replacement = await fixture.CreateCheckpointer(); + Assert.Equal(string.Empty, await replacement.Load(TestCancellation)); + await Persist(replacement, "10"); + Assert.NotEqual(initial.ETag, (await fixture.ReadEntity()).ETag); + } + + await Persist(old, "9"); + + var conflict = Assert.Single(oldRequests.Updates); + Assert.Equal(recreateBeforeUpdate ? 412 : 404, conflict.Status); + Assert.Equal(initial.ETag.ToString(), conflict.IfMatch); + Assert.Equal(recreateBeforeUpdate ? 1 : 2, oldRequests.Inserts.Count()); + await fixture.AssertCheckpoint(recreateBeforeUpdate ? "10" : "9"); + + await Persist(old, "11"); + + Assert.Equal(204, oldRequests.Updates.Last().Status); + Assert.NotEqual(initial.ETag.ToString(), oldRequests.Updates.Last().IfMatch); + Assert.NotEqual("*", oldRequests.Updates.Last().IfMatch); + await fixture.AssertCheckpoint("11"); + } + + private static async Task Persist(IStreamQueueCheckpointer checkpointer, string checkpoint) + { + checkpointer.Update(checkpoint, UpdateTime, TestCancellation); + await checkpointer.FlushAsync(TestCancellation); + } + + private sealed class Fixture : IAsyncDisposable + { + private const string Provider = "etag-provider"; + private const string Service = "etag-service"; + private const string Partition = "shard-1"; + private readonly TableServiceClient _serviceClient; + + public Fixture() + { + if (TestDefaultConfiguration.UseAadAuthentication + ? TestDefaultConfiguration.TableEndpoint is null + : string.IsNullOrWhiteSpace(TestDefaultConfiguration.DataConnectionString)) + { + throw Xunit.Sdk.SkipException.ForSkip("Azure Table tests require the existing AzureStorage test connection configuration."); + } + + _serviceClient = AzureStorageOperationOptionsExtensions.GetTableServiceClient(); + Table = _serviceClient.GetTableClient(TableName); + } + + public string TableName { get; } = $"Checkpoint{Guid.NewGuid():N}"; + public TableClient Table { get; } + + public async Task CreateCheckpointer(TableRequests? requests = null) + { + var serviceClient = _serviceClient; + if (requests is not null) + { + var clientOptions = new TableClientOptions(); + clientOptions.Retry.MaxRetries = 2; + clientOptions.Retry.NetworkTimeout = TimeSpan.FromSeconds(5); + clientOptions.AddPolicy(requests, HttpPipelinePosition.PerCall); + serviceClient = TestDefaultConfiguration.UseAadAuthentication + ? new TableServiceClient(TestDefaultConfiguration.TableEndpoint, TestDefaultConfiguration.TokenCredential, clientOptions) + : new TableServiceClient(TestDefaultConfiguration.DataConnectionString, clientOptions); + } + + return Assert.IsType( + await AzureTableStreamQueueCheckpointer.Create( + new AzureTableStreamCheckpointerOptions + { + TableName = TableName, + TableServiceClient = serviceClient, + PersistInterval = TimeSpan.FromMinutes(1), + CheckpointComparer = StreamCheckpointComparers.Numeric, + }, + Provider, Partition, Service, NullLoggerFactory.Instance, TestCancellation)); + } + + public async Task ReadEntity() + { + var key = StreamQueueCheckpointEntity.Create(string.Empty, Provider, Service, Partition); + var response = await Table.GetEntityAsync( + key.PartitionKey, key.RowKey, cancellationToken: TestCancellation); + return response.Value; + } + + public async Task AssertCheckpoint(string expected) + { + var fresh = await CreateCheckpointer(); + Assert.Equal(expected, await fresh.Load(TestCancellation)); + Assert.True(fresh.CheckpointExists); + var entities = new List(); + await foreach (var entity in Table.QueryAsync(cancellationToken: TestCancellation)) + { + entities.Add(entity); + } + + Assert.Equal(expected, Assert.Single(entities).Offset); + } + + public async ValueTask DisposeAsync() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await _serviceClient.DeleteTableAsync(TableName, cancellation.Token); + } + } + + private sealed record TableExchange(string Method, int Status, string? IfMatch, string? ETag); + + private sealed class TableRequests(string tableName, bool holdInsert = false) : HttpPipelinePolicy + { + private readonly ConcurrentQueue _exchanges = new(); + + public TaskCompletionSource InsertStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseInsert { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public IEnumerable Inserts => _exchanges.Where(exchange => exchange.Method == "POST"); + public IEnumerable Updates => _exchanges.Where(exchange => exchange.Method == "PUT"); + public IEnumerable Reads => _exchanges.Where(exchange => exchange.Method == "GET"); + + public override void Process(HttpMessage message, ReadOnlyMemory pipeline) + => throw new InvalidOperationException("Checkpoint persistence uses asynchronous Azure Table requests."); + + public override async ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory pipeline) + { + var path = message.Request.Uri.ToUri().AbsolutePath; + var isEntityRequest = path.EndsWith($"/{tableName}", StringComparison.Ordinal) + || path.Contains($"/{tableName}(", StringComparison.Ordinal); + var method = message.Request.Method.ToString(); + if (isEntityRequest && method == "POST" && holdInsert) + { + InsertStarted.TrySetResult(); + await ReleaseInsert.Task.WaitAsync(message.CancellationToken); + } + + await ProcessNextAsync(message, pipeline); + if (isEntityRequest) + { + message.Request.Headers.TryGetValue("If-Match", out var ifMatch); + message.Response.Headers.TryGetValue("ETag", out var eTag); + _exchanges.Enqueue(new(method, message.Response.Status, ifMatch, eTag)); + } + } + } +} diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs new file mode 100644 index 00000000000..1b9c0e0fe5a --- /dev/null +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -0,0 +1,773 @@ +using System.Collections.Concurrent; +using Amazon.Kinesis; +using Amazon.Kinesis.Model; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using NSubstitute.Extensions; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; +using TestExtensions; +using Xunit; +using KinesisRecord = Amazon.Kinesis.Model.Record; + +namespace Orleans.Streaming.Kinesis.Tests; + +[TestSuite("BVT")] +[TestArea("Streaming")] +[TestProvider("Kinesis")] +[TestCategory("AWS"), TestCategory("Kinesis")] +public sealed class KinesisPooledRuntimeTests +{ + private static CancellationToken TestCancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData("")] + [InlineData("123456789012345678901234567890")] + public async Task PooledFactory_InitialIteratorUsesDurableCheckpointOrTrimHorizon(string checkpoint) + { + await using var fixture = new Fixture(checkpoint); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var record = fixture.Record("123456789012345678901234567891", "next"); + fixture.SetReads(client, () => Response("next-iterator", record)); + + var notifications = await receiver.GetQueueMessagesAsync(1, TestCancellation); + + Assert.Equal(fixture.StreamId, Assert.Single(notifications).StreamId); + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + Assert.Equal(["next"], ReadBatch(cursor).GetEvents().Select(item => item.Item1)); + Assert.False(cursor.MoveNext()); + await client.Received(1).GetShardIteratorAsync( + Arg.Is(request => + request.StreamName == "stream" + && request.ShardId == "shard-1" + && request.ShardIteratorType == (checkpoint.Length == 0 + ? ShardIteratorType.TRIM_HORIZON + : ShardIteratorType.AFTER_SEQUENCE_NUMBER) + && request.StartingSequenceNumber == (checkpoint.Length == 0 ? null : checkpoint)), + Arg.Any()); + Assert.Empty(fixture.Store.Writes); + } + + [Fact] + public async Task PooledReceiver_ExpiredIteratorResumesAfterLastAdmittedOffset() + { + await using var fixture = new Fixture("9"); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns( + new GetShardIteratorResponse { ShardIterator = "initial" }, + new GetShardIteratorResponse { ShardIterator = "renewed" }); + fixture.SetReads(client, + () => Response("expired", fixture.Record("10", "first")), + () => throw new ExpiredIteratorException("expired after cache admission"), + () => Response("tail", fixture.Record("11", "second"))); + + await receiver.GetQueueMessagesAsync(1, TestCancellation); + Assert.Equal("9", fixture.Store.Checkpoint); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(cursor), "10", 0, "first"); + AssertBatch(ReadBatch(cursor), "11", 1, "second"); + Assert.False(cursor.MoveNext()); + Assert.Equal(["initial", "expired", "renewed"], fixture.ReadRequests.Select(request => request.Iterator)); + Assert.Equal(1, fixture.Store.LoadCount); + Assert.Empty(fixture.Store.Writes); + await client.Received(1).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == ShardIteratorType.AFTER_SEQUENCE_NUMBER + && request.StartingSequenceNumber == "10"), + Arg.Any()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PooledReceiver_CacheAdmissionFailureReplaysWholeBatchWithoutAdvancingOrdinal(bool admitPrefix) + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + if (admitPrefix) + { + fixture.SetReads(client, () => Response("after-prefix", fixture.Record("9", "prefix"))); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + } + + var failure = new IOException("record body unavailable"); + var valid = fixture.Record("10", "first"); + var failed = new KinesisRecord { SequenceNumber = "11", Data = new FailingPayloadStream(failure) }; + fixture.SetReads(client, + () => Response("after-rejected-batch", valid, failed), + () => Response("tail", valid, fixture.Record("11", "second"))); + var capacity = receiver.GetMaxAddCount(); + + Assert.Same(failure, await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(2, TestCancellation))); + + Assert.Equal(capacity, receiver.GetMaxAddCount()); + Assert.Empty(fixture.Store.Writes); + await receiver.GetQueueMessagesAsync(2, TestCancellation); + + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + if (admitPrefix) + { + AssertBatch(ReadBatch(cursor), "9", 0, "prefix"); + } + + AssertBatch(ReadBatch(cursor), "10", admitPrefix ? 1 : 0, "first"); + AssertBatch(ReadBatch(cursor), "11", admitPrefix ? 2 : 1, "second"); + Assert.False(cursor.MoveNext()); + await client.Received(admitPrefix ? 1 : 2).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == (admitPrefix ? ShardIteratorType.AFTER_SEQUENCE_NUMBER : ShardIteratorType.TRIM_HORIZON) + && request.StartingSequenceNumber == (admitPrefix ? "9" : null)), + Arg.Any()); + Assert.Empty(fixture.Store.Writes); + } + + [Fact] + public async Task PooledReceiver_RepeatedThrottlingAndTransientFailuresPreserveReadPosition() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var throttled = new ProvisionedThroughputExceededException("throttled"); + var unavailable = new AmazonKinesisException("temporarily unavailable"); + fixture.SetReads(client, + () => Response("after-first", fixture.Record("10", "first")), + () => throw throttled, + () => throw throttled, + () => throw unavailable, + () => Response("tail", fixture.Record("11", "second"))); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + var capacity = receiver.GetMaxAddCount(); + + for (var attempt = 0; attempt < 2; attempt++) + { + Assert.Same(throttled, await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(1, TestCancellation))); + Assert.Equal(capacity, receiver.GetMaxAddCount()); + } + + Assert.Same(unavailable, await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(1, TestCancellation))); + Assert.Equal(capacity, receiver.GetMaxAddCount()); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + + Assert.Equal(["iterator", "after-first", "after-first", "after-first", "after-first"], + fixture.ReadRequests.Select(request => request.Iterator)); + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(cursor), "10", 0, "first"); + AssertBatch(ReadBatch(cursor), "11", 1, "second"); + Assert.False(cursor.MoveNext()); + Assert.Empty(fixture.Store.Writes); + await client.Received(1).GetShardIteratorAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task PooledReceiver_ReadRetriesFailedInitialization() + { + await using var fixture = new Fixture("9"); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var failure = new AmazonKinesisException("iterator acquisition failed"); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromException(failure), + Task.FromResult(new GetShardIteratorResponse { ShardIterator = "recovered" })); + fixture.SetReads(client, () => Response("tail", fixture.Record("10", "recovered"))); + + Assert.Same(failure, await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(1, TestCancellation))); + Assert.Empty(fixture.ReadRequests); + Assert.Empty(fixture.Store.Writes); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + + Assert.Equal(2, fixture.Store.LoadCount); + await fixture.CheckpointerFactory.Received(1).Create("shard-1", Arg.Any()); + await client.Received(2).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == ShardIteratorType.AFTER_SEQUENCE_NUMBER + && request.StartingSequenceNumber == "9"), + Arg.Any()); + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(cursor), "10", 0, "recovered"); + } + + [Fact] + public async Task PooledReceiver_OwningCallerCancellationAllowsWaitingCallerToRetry() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + fixture.CheckpointerFactory.Configure().Create("shard-1", Arg.Any()) + .Returns(async call => + { + if (Interlocked.Increment(ref calls) == 1) + { + var token = call.Arg(); + started.SetResult(token); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + } + + return fixture.CreateCheckpointer(); + }); + fixture.SetReads(Assert.Single(fixture.ReceiverClients), () => Response("tail", fixture.Record("10", "recovered"))); + using var ownerCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellation); + + var owner = receiver.GetQueueMessagesAsync(1, ownerCancellation.Token); + var initializationToken = await started.Task.WaitAsync(TestCancellation); + var waiter = receiver.GetQueueMessagesAsync(1, TestCancellation); + ownerCancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => owner); + Assert.Single(await waiter); + Assert.True(initializationToken.IsCancellationRequested); + Assert.Equal(2, calls); + Assert.Single(fixture.ReadRequests); + Assert.Equal(1, fixture.Store.LoadCount); + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(cursor), "10", 0, "recovered"); + } + + [Fact] + public async Task PooledReceiver_IndependentProviderCancellationPropagatesToBothInitializationCallers() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var creation = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.CheckpointerFactory.Configure().Create("shard-1", Arg.Any()).Returns(creation.Task); + using var providerCancellation = new CancellationTokenSource(); + + var owner = receiver.GetQueueMessagesAsync(1, TestCancellation); + var waiter = receiver.GetQueueMessagesAsync(1, TestCancellation); + providerCancellation.Cancel(); + creation.SetCanceled(providerCancellation.Token); + + await Assert.ThrowsAnyAsync(() => owner); + await Assert.ThrowsAnyAsync(() => waiter); + Assert.False(TestCancellation.IsCancellationRequested); + await fixture.CheckpointerFactory.Received(1).Create("shard-1", Arg.Any()); + await Assert.Single(fixture.ReceiverClients).DidNotReceive().GetShardIteratorAsync( + Arg.Any(), Arg.Any()); + Assert.Empty(fixture.Store.Writes); + } + + [Fact] + public async Task PooledFactory_ConcurrentReceiverAndCacheRequestsShareOneInstanceAndReassignAfterShutdown() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var queue = Assert.Single(fixture.Factory.GetStreamQueueMapper().GetAllQueues()); + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = Enumerable.Range(0, 16).Select(index => Task.Run(async () => + { + await start.Task.WaitAsync(TestCancellation); + return index % 2 == 0 + ? (object)fixture.Factory.CreateReceiver(queue) + : fixture.Factory.CreateQueueCache(queue); + }, TestCancellation)).ToArray(); + + start.SetResult(); + var results = await Task.WhenAll(calls); + var receiver = fixture.GetReceiver(); + Assert.All(results, result => Assert.Same(receiver, result)); + var firstClient = Assert.Single(fixture.ReceiverClients); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + var replacement = fixture.GetReceiver(); + + Assert.NotSame(receiver, replacement); + Assert.Same(replacement, fixture.Factory.CreateQueueCache(queue)); + Assert.Equal(2, fixture.ReceiverClients.Count); + firstClient.Received(1).Dispose(); + fixture.ReceiverClients.Last().DidNotReceive().Dispose(); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Same(replacement, fixture.GetReceiver()); + Assert.Empty(await receiver.GetQueueMessagesAsync(1, TestCancellation)); + fixture.DiscoveryClient.DidNotReceive().Dispose(); + await replacement.Shutdown(TimeSpan.FromSeconds(5)); + Assert.All(fixture.ReceiverClients, client => client.Received(1).Dispose()); + fixture.Factory.Dispose(); + fixture.Factory.Dispose(); + fixture.DiscoveryClient.Received(1).Dispose(); + } + + [Fact] + public async Task PooledFactory_ByteBudgetStagesUnreadSuffixUntilDeliveredPrefixIsPurged() + { + await using var fixture = new Fixture(maxCacheSizeBytes: 1); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + fixture.SetReads(client, () => Response("tail", + fixture.Record("10", "first"), + fixture.Record("11", "second"))); + + var firstNotification = Assert.Single(await receiver.GetQueueMessagesAsync(2, TestCancellation)); + + Assert.Equal("10", Assert.IsType(firstNotification.SequenceToken).ShardSequence); + Assert.True(receiver.IsUnderPressure()); + Assert.Equal(0, receiver.GetMaxAddCount()); + using (var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable)) + { + AssertBatch(ReadBatch(cursor), "10", 0, "first"); + Assert.False(cursor.MoveNext()); + var progress = Assert.IsAssignableFrom(cursor); + progress.RecordDeliverySuccess(); + receiver.UpdateDeliveryProgress( + Assert.IsType(progress.SafeSequenceToken), + fixture.Time.GetUtcNow().UtcDateTime); + } + + Assert.False(receiver.IsUnderPressure()); + Assert.Equal("10", fixture.Store.Checkpoint); + var secondNotification = Assert.Single(await receiver.GetQueueMessagesAsync(2, TestCancellation)); + + Assert.Equal("11", Assert.IsType(secondNotification.SequenceToken).ShardSequence); + Assert.Single(fixture.ReadRequests); + using var nextCursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(nextCursor), "11", 1, "second"); + Assert.False(nextCursor.MoveNext()); + Assert.Equal("10", fixture.Store.Checkpoint); + } + + [Fact] + public async Task PooledFactory_CanceledDiscoveryRetriesBeforePublishingReceiver() + { + await using var fixture = new Fixture(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var discoveries = 0; + fixture.DiscoveryClient.Configure().ListShardsAsync(Arg.Any(), Arg.Any()) + .Returns(async call => + { + if (Interlocked.Increment(ref discoveries) == 1) + { + started.SetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, call.Arg()); + } + + return new ListShardsResponse { Shards = [new Shard { ShardId = "shard-1" }] }; + }); + using var ownerCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellation); + var initialization = fixture.Factory.CreateAdapter(ownerCancellation.Token); + await started.Task.WaitAsync(TestCancellation); + var queue = QueueId.GetQueueId("queue", 0, 0); + Assert.Throws(() => fixture.Factory.CreateReceiver(queue)); + Assert.Throws(() => fixture.Factory.CreateQueueCache(queue)); + Assert.Empty(fixture.ReceiverClients); + + ownerCancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => initialization); + await fixture.Factory.CreateAdapter(TestCancellation); + + Assert.Equal(2, discoveries); + Assert.Same(fixture.GetReceiver(), fixture.Factory.CreateQueueCache( + Assert.Single(fixture.Factory.GetStreamQueueMapper().GetAllQueues()))); + Assert.Single(fixture.ReceiverClients); + } + + [Fact] + public async Task PooledFactory_ShutdownKeepsReceiverReservedUntilInitializationSettles() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + started.SetResult(call.Arg()); + return release.Task; + }); + var read = receiver.GetQueueMessagesAsync(1, TestCancellation); + var initializationToken = await started.Task.WaitAsync(TestCancellation); + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(10)); + try + { + Assert.True(initializationToken.IsCancellationRequested); + Assert.False(shutdown.IsCompleted); + Assert.Same(receiver, fixture.GetReceiver()); + Assert.Single(fixture.ReceiverClients); + client.DidNotReceive().Dispose(); + } + finally + { + release.TrySetResult(new GetShardIteratorResponse { ShardIterator = "late" }); + } + + await shutdown; + await Assert.ThrowsAnyAsync(() => read); + client.Received(1).Dispose(); + Assert.Empty(fixture.ReadRequests); + Assert.NotSame(receiver, fixture.GetReceiver()); + } + + [Fact] + public async Task PooledReceiver_CanceledTopologyDiscoveryReleasesMonitorForReassignedReceiver() + { + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.DiscoveryClient.Configure().ListShardsAsync(Arg.Any(), Arg.Any()) + .Returns(async call => + { + var token = call.Arg(); + started.SetResult(token); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return null!; + }); + fixture.Time.Advance(TimeSpan.FromMinutes(1)); + using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellation); + var read = receiver.GetQueueMessagesAsync(1, readCancellation.Token); + Assert.Equal(readCancellation.Token, await started.Task.WaitAsync(TestCancellation)); + + // The pulling agent owns read cancellation and settles reads before receiver shutdown. + readCancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => read); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Empty(fixture.ReadRequests); + Assert.Single(fixture.ReceiverClients).Received(1).Dispose(); + fixture.DiscoveryClient.DidNotReceive().Dispose(); + fixture.DiscoveryClient.Configure().ListShardsAsync(Arg.Any(), Arg.Any()) + .Returns(new ListShardsResponse { Shards = [new Shard { ShardId = "shard-1" }] }); + var replacement = fixture.GetReceiver(); + fixture.SetReads(fixture.ReceiverClients.Last(), () => Response("tail", fixture.Record("10", "replacement"))); + Assert.Single(await replacement.GetQueueMessagesAsync(1, TestCancellation)); + await fixture.DiscoveryClient.Received(3).ListShardsAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task PooledReceiver_MixedTypeNonzeroIndicesPersistSafeWatermarkAndRestartAtNextRecord() + { + const string previousOffset = "123456789012345678901234567889"; + const string checkpoint = "123456789012345678901234567890"; + const string nextOffset = "123456789012345678901234567891"; + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var record = fixture.Record(checkpoint, 10, "eleven", 12, "thirteen"); + var wirePayload = record.Data.ToArray(); + fixture.SetReads(client, + () => Response("mixed-record", fixture.Record(previousOffset, "prefix")), + () => Response("tail", record)); + await receiver.GetQueueMessagesAsync(1, TestCancellation); + using (var prefixCursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable)) + { + AssertBatch(ReadBatch(prefixCursor), previousOffset, 0, "prefix"); + var prefixProgress = Assert.IsAssignableFrom(prefixCursor); + prefixProgress.RecordDeliverySuccess(); + receiver.UpdateDeliveryProgress( + Assert.IsType(prefixProgress.SafeSequenceToken), + fixture.Time.GetUtcNow().UtcDateTime); + } + + Assert.Equal([previousOffset], fixture.Store.Writes); + var notification = Assert.Single(await receiver.GetQueueMessagesAsync(1, TestCancellation)); + using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + var batch = ReadBatch(cursor); + var progress = Assert.IsAssignableFrom(cursor); + + Assert.Equal([10, 12], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal([0, 2], batch.GetEvents().Select(item => item.Item2.EventIndex)); + Assert.Equal(["eleven", "thirteen"], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal([1, 3], batch.GetEvents().Select(item => item.Item2.EventIndex)); + Assert.All(batch.GetEvents(), item => + Assert.Equal(checkpoint, Assert.IsType(item.Item2).ShardSequence)); + Assert.Equal(wirePayload, record.Data.ToArray()); + Assert.True(batch.ImportRequestContext()); + try + { + Assert.Equal("trace-42", RequestContext.Get("trace-id")); + } + finally + { + RequestContext.Clear(); + } + + var enteredLastEvent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseLastEvent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var delivered = new List<(object Value, int EventIndex)>(); + var delivery = Deliver(); + await enteredLastEvent.Task.WaitAsync(TestCancellation); + try + { + Assert.Equal([0, 1, 2], delivered.Select(item => item.EventIndex)); + Assert.Null(progress.SafeSequenceToken); + await receiver.MessagesDeliveredAsync([notification], TestCancellation); + Assert.Equal([previousOffset], fixture.Store.Writes); + Assert.Equal(previousOffset, fixture.Store.Checkpoint); + } + finally + { + releaseLastEvent.TrySetResult(); + } + + await delivery; + Assert.Equal([0, 1, 2, 3], delivered.Select(item => item.EventIndex)); + Assert.Equal(new object[] { 10, "eleven", 12, "thirteen" }, delivered.Select(item => item.Value)); + progress.RecordDeliverySuccess(); + var safeToken = Assert.IsType(progress.SafeSequenceToken); + Assert.Equal(checkpoint, safeToken.ShardSequence); + receiver.UpdateDeliveryProgress(safeToken, fixture.Time.GetUtcNow().UtcDateTime); + Assert.Equal(previousOffset, fixture.Store.Checkpoint); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal([previousOffset, checkpoint], fixture.Store.Writes); + Assert.Equal(checkpoint, fixture.Store.Checkpoint); + + var replacement = fixture.GetReceiver(); + Assert.NotSame(receiver, replacement); + var replacementClient = fixture.ReceiverClients.Last(); + fixture.SetReads(replacementClient, () => Response("tail", fixture.Record(nextOffset, "next-record"))); + await replacement.GetQueueMessagesAsync(1, TestCancellation); + using var restartedCursor = replacement.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + AssertBatch(ReadBatch(restartedCursor), nextOffset, 0, "next-record"); + Assert.False(restartedCursor.MoveNext()); + await replacementClient.Received(1).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == ShardIteratorType.AFTER_SEQUENCE_NUMBER + && request.StartingSequenceNumber == checkpoint), + Arg.Any()); + Assert.Equal(2, fixture.Store.LoadCount); + + async Task Deliver() + { + foreach (var item in batch.GetEvents()) + { + if (item.Item2.EventIndex == 3) + { + enteredLastEvent.SetResult(); + await releaseLastEvent.Task.WaitAsync(TestCancellation); + } + + delivered.Add((item.Item1, item.Item2.EventIndex)); + } + } + } + + [Fact] + public async Task Source_ReadReturnsOrderedRawRecordMetadataWithoutDecodingOrMutatingWirePayload() + { + await using var fixture = new Fixture(); + var client = Substitute.For(); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(new GetShardIteratorResponse { ShardIterator = "source-iterator" }); + var first = fixture.Record("999999999999999999999999999999", 10, "eleven"); + var second = fixture.Record("1000000000000000000000000000000", "second"); + first.Data.Position = 1; + var payload = first.Data.ToArray(); + fixture.SetReads(client, () => Response("next", first, second)); + var source = new KinesisRecoverableStreamSource( + client, "stream", "shard-1", + new KinesisShardTopologyMonitor(fixture.DiscoveryClient, "stream", ["shard-1"], + TimeSpan.FromMinutes(1), fixture.Time, NullLogger.Instance), + TimeSpan.Zero, fixture.Time); + try + { + await source.Initialize(new RecoverableStreamStartPosition(null, false), TestCancellation); + var records = await source.Read(2, TestCancellation); + + Assert.Equal([0L, 1L], records.Select(record => record.SequenceNumber)); + Assert.Collection(records, + record => Assert.Same(first, record.Record), + record => Assert.Same(second, record.Record)); + Assert.All(records, record => Assert.Null(record.Body)); + Assert.Equal(payload, records[0].RawPayload); + Assert.Equal(1, first.Data.Position); + Assert.Equal("partition-key", first.PartitionKey); + Assert.Equal(fixture.Time.GetUtcNow().UtcDateTime, first.ApproximateArrivalTimestamp); + var body = fixture.Serializer.Deserialize(records[0].RawPayload); + Assert.Equal(fixture.StreamId, body.StreamId); + Assert.Equal(new object[] { 10, "eleven" }, body.Events); + Assert.Equal("trace-42", body.RequestContext!["trace-id"]); + Assert.Equal(payload, fixture.Serializer.SerializeToArray(body)); + var request = Assert.Single(fixture.ReadRequests); + Assert.Equal("source-iterator", request.Iterator); + Assert.Equal(2, request.Limit); + } + finally + { + await source.Shutdown(TestCancellation); + } + + client.Received(1).Dispose(); + } + + private static KinesisBatchContainer ReadBatch(IQueueCacheCursor cursor) + { + Assert.True(cursor.MoveNext()); + var batch = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); + return batch; + } + + private static void AssertBatch(KinesisBatchContainer batch, string offset, long ordinal, string value) + { + Assert.Equal(offset, batch.Token.ShardSequence); + Assert.Equal(ordinal, batch.Token.SequenceNumber); + Assert.Equal(value, Assert.Single(batch.GetEvents()).Item1); + } + + private static GetRecordsResponse Response(string nextIterator, params KinesisRecord[] records) + => new() { NextShardIterator = nextIterator, Records = [.. records] }; + + private sealed class FailingPayloadStream(IOException failure) : MemoryStream + { + public override byte[] ToArray() => throw failure; + } + + private sealed class CheckpointStore(string checkpoint) : IStreamCheckpointStore + { + public string Checkpoint { get; private set; } = checkpoint; + public int LoadCount { get; private set; } + public List Writes { get; } = []; + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + LoadCount++; + return ValueTask.FromResult(new StreamCheckpointStoreState(Checkpoint, Checkpoint)); + } + + public ValueTask Update(string checkpoint, string expectedVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Assert.Equal(Checkpoint, expectedVersion); + Checkpoint = checkpoint; + Writes.Add(checkpoint); + return ValueTask.FromResult(new StreamCheckpointStoreState(checkpoint, checkpoint)); + } + } + + private sealed class Fixture : IAsyncDisposable + { + private readonly ServiceProvider _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + private readonly HashSet _receivers = []; + private int _clientCount; + + public Fixture(string checkpoint = "", long? maxCacheSizeBytes = null) + { + Store = new(checkpoint); + Serializer = _services.GetRequiredService>(); + DiscoveryClient.ListShardsAsync(Arg.Any(), Arg.Any()) + .Returns(new ListShardsResponse { Shards = [new Shard { ShardId = "shard-1" }] }); + CheckpointerFactory.Create("shard-1", Arg.Any()) + .Returns(_ => Task.FromResult(CreateCheckpointer())); + var options = new KinesisStreamOptions + { + StreamName = "stream", + GetRecordsInterval = TimeSpan.Zero, + TopologyCheckInterval = TimeSpan.FromMinutes(1), + }; + if (maxCacheSizeBytes is { } byteBudget) + { + options.MaxCacheSizeBytes = byteBudget; + } + + Factory = new KinesisAdapterFactory( + "Kinesis", + options, + new SimpleQueueCacheOptions { CacheSize = 8 }, + Serializer, CheckpointerFactory, NullLoggerFactory.Instance, Time, CreateClient); + } + + public StreamId StreamId { get; } = StreamId.Create("pooled-runtime", "stream-key"); + public FakeTimeProvider Time { get; } = new(); + public IAmazonKinesis DiscoveryClient { get; } = Substitute.For(); + public ConcurrentQueue ReceiverClients { get; } = new(); + public IStreamQueueCheckpointerFactory CheckpointerFactory { get; } = Substitute.For(); + public Serializer Serializer { get; } + public KinesisAdapterFactory Factory { get; } + public CheckpointStore Store { get; } + public List<(string Iterator, int? Limit)> ReadRequests { get; } = []; + + public IStreamQueueCheckpointer CreateCheckpointer() + => new StreamQueueCheckpointer(Store, new StreamQueueCheckpointerOptions + { + PersistInterval = TimeSpan.FromMinutes(1), + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + + public KinesisPooledAdapterReceiver GetReceiver() + { + var queue = Assert.Single(Factory.GetStreamQueueMapper().GetAllQueues()); + var receiver = Assert.IsType(Factory.CreateReceiver(queue)); + Assert.Same(receiver, Factory.CreateQueueCache(queue)); + _receivers.Add(receiver); + return receiver; + } + + public KinesisRecord Record(string offset, params object[] events) + => new() + { + SequenceNumber = offset, + Data = new MemoryStream(KinesisBatchContainer.ToKinesisPayload( + Serializer, StreamId, events, new Dictionary { ["trace-id"] = "trace-42" })), + PartitionKey = "partition-key", + ApproximateArrivalTimestamp = Time.GetUtcNow().UtcDateTime, + }; + + public void SetReads(IAmazonKinesis client, params Func[] responses) + { + var pending = new Queue>(responses); + client.Configure().GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + ReadRequests.Add((request.ShardIterator, request.Limit)); + return Task.FromResult(pending.Dequeue()()); + }); + } + + private IAmazonKinesis CreateClient() + { + if (Interlocked.Increment(ref _clientCount) == 1) + { + return DiscoveryClient; + } + + var client = Substitute.For(); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(new GetShardIteratorResponse { ShardIterator = "iterator" }); + ReceiverClients.Enqueue(client); + return client; + } + + public async ValueTask DisposeAsync() + { + try + { + foreach (var receiver in _receivers) + { + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + } + finally + { + Factory.Dispose(); + await _services.DisposeAsync(); + } + } + } +} From c954014e0a4ce982aac79bc6bc518cf851ca949a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:38:04 -0700 Subject: [PATCH 55/90] test(streaming): use concrete bounded recovery consumer --- ...rsistentStreamPullingAgentRecoveryTests.cs | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs index 04fd4332669..956d3b1d99b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs @@ -76,15 +76,10 @@ public async Task FutureStartToken_BeyondFiniteCapacity_PreservesDeliveryBarrier var deliveredTokens = new List(); var attempts = 0; var firstExpectedToken = implicitSubscription ? 8 : 7; - var targetConsumer = Substitute.For(); - targetConsumer.GetSequenceToken(targetSubscription, Arg.Any()) - .Returns(Task.FromResult(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(7)))); - targetConsumer.DeliverBatch( - targetSubscription, qualifiedStream, Arg.Any(), - Arg.Any(), Arg.Any()) - .Returns(call => + var targetConsumer = new RecoveryConsumer( + StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(7)), + batch => { - var batch = call.Arg(); deliveredTokens.Add(batch.SequenceToken.SequenceNumber); if (batch.SequenceToken.SequenceNumber != firstExpectedToken) { @@ -199,6 +194,7 @@ public async Task FutureStartToken_BeyondFiniteCapacity_PreservesDeliveryBarrier Assert.Equal(8, targetData.LastProcessedToken?.SequenceNumber); Assert.Equal(9, targetData.LastSafePartitionToken?.SequenceNumber); Assert.Equal(implicitSubscription ? new long[] { 8, 8 } : new long[] { 7, 7, 8 }, deliveredTokens); + Assert.Empty(targetConsumer.Errors); Assert.Empty(slowConsumer.Errors); await UpdateProgress(); Assert.Equal(new long[] { 3, 3, 6, safeBeforeDelivery, safeBeforeDelivery, 9 }, checkpoints); @@ -247,6 +243,47 @@ async Task UpdateProgress() } } + private sealed class RecoveryConsumer( + StreamHandshakeToken startToken, + Func> deliverBatch) : IStreamConsumerExtension + { + public List Errors { get; } = []; + + public Task DeliverImmutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + public Task DeliverMutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + public Task DeliverBatch( + GuidId subscriptionId, + QualifiedStreamId streamId, + IBatchContainer item, + StreamHandshakeToken? handshakeToken, + CancellationToken cancellationToken) => deliverBatch(item); + + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToken cancellationToken) + { + Errors.Add(exc); + return Task.CompletedTask; + } + + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => Task.FromResult(startToken); + } + private sealed class BoundedRecoverySource(IReadOnlyList messages) : IRecoverableStreamSource { From f49b39333ae2eb328ffa966e3dce231e2e22bf4a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 20:39:03 -0700 Subject: [PATCH 56/90] fix(streaming): normalize legacy recovery comparisons --- docs/site/src/content/docs/host/monitoring/signals.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/site/src/content/docs/host/monitoring/signals.md b/docs/site/src/content/docs/host/monitoring/signals.md index eb474519606..fa5596d22ab 100644 --- a/docs/site/src/content/docs/host/monitoring/signals.md +++ b/docs/site/src/content/docs/host/monitoring/signals.md @@ -54,6 +54,12 @@ If spans appear as separate traces, confirm to correlate a consumer's delivery and cleanup transitions by provider, stream, subscription, consumer endpoint, and silo. includes the attempted position and original exception. reports when the pulling agent requests pubsub cleanup and when that request completes or fails. marks the successful durable removal. + +When a client route becomes stale, the runtime resolves the client through the client directory. An available gateway receives the request; an unavailable client produces , which initiates subscription cleanup. For a cleanup timeout, capture the last observed transition and its exception: a delivery failure identifies the routing or consumer failure, a pending unregistration identifies the pubsub request, and a failed unregistration identifies its storage or callback failure. Keep diagnostic capture bounded and retain positions and identifiers rather than event payloads. + ## Baseline alerts Tune alerts against normal traffic and service objectives. A practical initial set is: From 19f36127f8357278b0820eac9ae38f2a9bcbae44 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 21:01:32 -0700 Subject: [PATCH 57/90] fix(streaming): finalize recovery integration and API contracts --- .../Orleans.Streaming.Kinesis.cs | 2 + .../Orleans.Streaming.AdoNet.cs | 2 + .../Orleans.Streaming/Orleans.Streaming.cs | 49 +++++++++++++------ .../Streaming/AdoNetQueueAdapterTests.cs | 2 +- .../Streaming/AdoNetRecoverableStreamTests.cs | 4 +- .../KinesisPooledRuntimeTests.cs | 5 +- ...stentStreamPullingAgentDiagnosticsTests.cs | 44 ++++++++--------- ...rsistentStreamPullingAgentRecoveryTests.cs | 2 +- 8 files changed, 69 insertions(+), 41 deletions(-) diff --git a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs index 9ef8844adb3..954a8294197 100644 --- a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs +++ b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs @@ -106,6 +106,8 @@ public partial class KinesisStreamOptions public System.TimeSpan GetRecordsInterval { get { throw null; } set { } } + public long MaxCacheSizeBytes { get { throw null; } set { } } + public string? Region { get { throw null; } set { } } [Redact] diff --git a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs index df264c68161..363fe4d1848 100644 --- a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs +++ b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs @@ -28,6 +28,8 @@ public partial class AdoNetStreamOptions public string Invariant { get { throw null; } set { } } + public long MaxCacheSizeBytes { get { throw null; } set { } } + public System.TimeSpan? MaximumRetentionPeriod { get { throw null; } set { } } public int MaxMessagesPerRead { get { throw null; } set { } } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 2023d851482..1dd9cd14d37 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -429,7 +429,6 @@ public MemoryAdapterFactory(string providerName, Configuration.StreamCacheEvicti [System.Diagnostics.CodeAnalysis.MemberNotNull(new[] { "CacheMonitorFactory", "BlockPoolMonitorFactory", "ReceiverMonitorFactory" })] public void Init() { } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task QueueMessageBatchAsync(Runtime.StreamId streamId, System.Collections.Generic.IEnumerable events, Orleans.Streams.StreamSequenceToken? token, System.Collections.Generic.Dictionary? requestContext) { throw null; } } @@ -948,10 +947,14 @@ public sealed partial class RecoverableStreamQueueCache : IRecove { public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) { } + public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, long maxCacheSizeBytes, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) { } + public int ItemCount { get { throw null; } } public string? LastPurgedOffset { get { throw null; } } + public long SizeInBytes { get { throw null; } } + public System.Collections.Generic.IReadOnlyList Add(System.Collections.Generic.IReadOnlyList messages, System.DateTime dequeueTimeUtc) { throw null; } public void AddToCache(System.Collections.Generic.IList messages) { } @@ -960,6 +963,8 @@ public void Dispose() { } public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + public Orleans.Streams.IQueueCacheCursor GetCacheCursorAtPosition(Runtime.StreamId streamId, Orleans.Streams.StreamSubscriptionStartPosition startPosition) { throw null; } + public int GetMaxAddCount() { throw null; } public bool IsUnderPressure() { throw null; } @@ -981,9 +986,10 @@ public void AddToCache(System.Collections.Generic.IList> GetQueueMessagesAsync(int maxCount, System.Threading.CancellationToken cancellationToken) { throw null; } [System.Obsolete("Use the overload which accepts a CancellationToken.")] @@ -991,7 +997,6 @@ public void AddToCache(System.Collections.Generic.IList messages) { throw null; } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Shutdown(System.TimeSpan timeout) { throw null; } public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } @@ -1377,6 +1381,16 @@ public sealed partial class MessageDelivered : StreamingEvent public MessageDelivered(string streamProvider, Runtime.StreamId streamId, System.Guid subscriptionId, Runtime.SiloAddress? siloAddress, Runtime.IAddressable consumer, Streams.IBatchContainer batch) : base(default!, default) { } } + public sealed partial class MessageDeliveryFailed : StreamingEvent + { + public readonly Runtime.IAddressable Consumer; + public readonly System.Exception Exception; + public readonly Streams.StreamSequenceToken SequenceToken; + public readonly Runtime.StreamId StreamId; + public readonly System.Guid SubscriptionId; + public MessageDeliveryFailed(string streamProvider, Runtime.StreamId streamId, System.Guid subscriptionId, Runtime.SiloAddress? siloAddress, Runtime.IAddressable consumer, Streams.StreamSequenceToken sequenceToken, System.Exception exception) : base(default!, default) { } + } + public sealed partial class ProducerRegistered : StreamingEvent { public readonly Runtime.GrainId ProducerGrainId; @@ -1508,6 +1522,23 @@ public sealed partial class SubscriptionUnregistered : StreamingEvent public readonly System.Guid SubscriptionId; public SubscriptionUnregistered(string streamProvider, Runtime.StreamId streamId, System.Guid subscriptionId, Runtime.SiloAddress? siloAddress) : base(default!, default) { } } + + public sealed partial class SubscriptionUnregistration : StreamingEvent + { + public readonly Runtime.IAddressable Consumer; + public readonly System.Exception? Exception; + public readonly SubscriptionUnregistrationStage Stage; + public readonly Runtime.StreamId StreamId; + public readonly System.Guid SubscriptionId; + public SubscriptionUnregistration(string streamProvider, Runtime.StreamId streamId, System.Guid subscriptionId, Runtime.SiloAddress? siloAddress, Runtime.IAddressable consumer, SubscriptionUnregistrationStage stage, System.Exception? exception) : base(default!, default) { } + } + + public enum SubscriptionUnregistrationStage + { + Requested = 0, + Completed = 1, + Failed = 2 + } } } @@ -1658,17 +1689,13 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain) { } public bool CheckpointExists { get { throw null; } } - [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options, System.Threading.CancellationToken cancellationToken) { throw null; } - [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options) { throw null; } - [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, System.Threading.CancellationToken cancellationToken) { throw null; } - [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient) { throw null; } @@ -1995,12 +2022,10 @@ public LeaseBasedQueueBalancer(string name, Configuration.LeaseBasedQueueBalance public override System.Collections.Generic.IEnumerable GetMyQueues() { throw null; } - [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Initialize(IStreamQueueMapper queueMapper) { throw null; } protected override void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos) { } - [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Shutdown() { throw null; } } @@ -2131,7 +2156,6 @@ protected QueueBalancerBase(System.IServiceProvider sp, Microsoft.Extensions.Log protected System.Threading.Tasks.Task NotifyListeners() { throw null; } protected abstract void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos); - [System.Diagnostics.DebuggerStepThrough] public virtual System.Threading.Tasks.Task Shutdown() { throw null; } public bool SubscribeToQueueDistributionChangeEvents(IStreamQueueBalanceListener observer) { throw null; } @@ -2344,7 +2368,6 @@ public StreamCheckpointGrain(Runtime.IPersistentState Load(System.Threading.CancellationToken cancellationToken) { throw null; } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.ValueTask Update(string offset, string expectedCheckpoint, System.Threading.CancellationToken cancellationToken) { throw null; } } @@ -2449,13 +2472,11 @@ public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpoi public bool CheckpointExists { get { throw null; } } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 87d92d8f34e..aa7b3acfb7b 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -308,7 +308,7 @@ public async Task AcquisitionCancellation_BlockedCommandSettlesBeforeReplacement await first.Shutdown(TimeSpan.FromSeconds(20)); if (read is not null) { - await ((Task)read).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + await ((Task)read).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 138d31433f5..9a750f6d287 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -547,9 +547,9 @@ public async Task Read_CheckpointedHistoryHolesPreserveUnreadContinuity() [Theory] [InlineData(0, 4, null, null, true)] - [InlineData(2, 4, 1, 2, true)] + [InlineData(2, 4, 1L, 2L, true)] [InlineData(3, 4, null, null, false)] - [InlineData(3, 5, 4, 4, false)] + [InlineData(3, 5, 4L, 4L, false)] public async Task Read_EmptyBatchReconcilesDeletedHistoryAndConcurrentArrivals( long checkpoint, long nextMessageId, long? earliest, long? tail, bool gap) { diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs index 1b9c0e0fe5a..ce7b8456fd9 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -439,10 +439,12 @@ public async Task PooledReceiver_CanceledTopologyDiscoveryReleasesMonitorForReas fixture.Time.Advance(TimeSpan.FromMinutes(1)); using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellation); var read = receiver.GetQueueMessagesAsync(1, readCancellation.Token); - Assert.Equal(readCancellation.Token, await started.Task.WaitAsync(TestCancellation)); + var discoveryCancellation = await started.Task.WaitAsync(TestCancellation); + Assert.True(discoveryCancellation.CanBeCanceled); // The pulling agent owns read cancellation and settles reads before receiver shutdown. readCancellation.Cancel(); + Assert.True(discoveryCancellation.IsCancellationRequested); await Assert.ThrowsAnyAsync(() => read); await receiver.Shutdown(TimeSpan.FromSeconds(5)); @@ -598,6 +600,7 @@ record => Assert.Same(second, record.Record)); Assert.Equal("partition-key", first.PartitionKey); Assert.Equal(fixture.Time.GetUtcNow().UtcDateTime, first.ApproximateArrivalTimestamp); var body = fixture.Serializer.Deserialize(records[0].RawPayload); + Assert.NotNull(body); Assert.Equal(fixture.StreamId, body.StreamId); Assert.Equal(new object[] { 10, "eleven" }, body.Events); Assert.Equal("trace-42", body.RequestContext!["trace-id"]); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 7a1267be840..b3a2141f1d4 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -49,28 +49,6 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking { outcome.TrySetResult(registration); } - - private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension - { - public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, - StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - => throw new NotSupportedException(); - - public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, - StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - => throw new NotSupportedException(); - - public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, - StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - => Task.FromException(exception); - - public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task ErrorInStream(GuidId subscriptionId, Exception error, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) - => Task.FromResult(null); - } } }); @@ -112,4 +90,26 @@ private sealed class UnavailableConsumer(ClientNotAvailableException exception) await accessor.Shutdown(); } } + + private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension + { + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, + StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => Task.FromException(exception); + + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception error, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => Task.FromResult(null); + } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs index 956d3b1d99b..2f97ca6cd46 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs @@ -244,7 +244,7 @@ async Task UpdateProgress() } private sealed class RecoveryConsumer( - StreamHandshakeToken startToken, + StreamHandshakeToken? startToken, Func> deliverBatch) : IStreamConsumerExtension { public List Errors { get; } = []; From b39e144fc8eb0a877eaeb2069c47ac06714d1626 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 21:25:59 -0700 Subject: [PATCH 58/90] fix(streaming): harden cross-framework and cancellation validation --- .../AdoNetRecoverableStream.cs | 11 ++++- .../Streaming/AdoNetQueueAdapterTests.cs | 19 ++++++--- .../Streaming/AdoNetRecoverableStreamTests.cs | 36 +++++++++++++++++ .../RecoveryCacheMemoryOptionsTests.cs | 38 +++++++++++++----- ...ableStreamQueueCheckpointerAzuriteTests.cs | 40 +++++++++++++------ ...stentStreamPullingAgentDiagnosticsTests.cs | 15 ++++++- .../RecoveryCacheMemoryTests.cs | 4 +- 7 files changed, 129 insertions(+), 34 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index 93c48f28754..a00b0c4d96f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -1,3 +1,4 @@ +using System.Data.Common; using System.Globalization; using Orleans.Providers.Streams.Common; @@ -34,7 +35,15 @@ public async ValueTask Load(CancellationToken cancel options.StartFromNow, cancellationToken); Volatile.Write(ref _acquisitionTask, acquisitionTask); - var partition = await acquisitionTask; + AdoNetStreamPartitionState partition; + try + { + partition = await acquisitionTask; + } + catch (DbException exception) when (cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException("ADO.NET stream partition acquisition was canceled.", exception, cancellationToken); + } cancellationToken.ThrowIfCancellationRequested(); _partition = partition; _readOffset = partition.Checkpoint ?? 0; diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index aa7b3acfb7b..6906d908d01 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -237,17 +237,20 @@ public async Task AcquisitionCancellation_BlockedCommandSettlesBeforeReplacement Assert.Equal(1, await command.ExecuteNonQueryAsync(cancellationToken)); } - await using var observer = lockConnection.CreateCommand(); - observer.Transaction = transaction; - observer.CommandText = invariant switch + await using var ownerCommand = lockConnection.CreateCommand(); + ownerCommand.Transaction = transaction; + ownerCommand.CommandText = invariant switch { AdoNetInvariants.InvariantNameSqlServer => "SELECT @@SPID", AdoNetInvariants.InvariantNameMySql => "SELECT CONNECTION_ID()", AdoNetInvariants.InvariantNamePostgreSql => "SELECT pg_backend_pid()", _ => throw new NotSupportedException(invariant), }; - var lockOwner = Convert.ToInt64(await observer.ExecuteScalarAsync(cancellationToken), CultureInfo.InvariantCulture); - observer.CommandText = GetBlockedAcquisitionQuery(lockConnection); + var lockOwner = Convert.ToInt64(await ownerCommand.ExecuteScalarAsync(cancellationToken), CultureInfo.InvariantCulture); + await using var observerConnection = CreateConnection(); + await observerConnection.OpenAsync(cancellationToken); + await using var observer = observerConnection.CreateCommand(); + observer.CommandText = GetBlockedAcquisitionQuery(observerConnection); AddParameter(observer, "LockOwner", lockOwner); using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var lockReleased = false; @@ -316,6 +319,12 @@ Task WaitForBlockedAcquisitions(int expected) => TestingUtils.WaitUntilAsync( async (lastTry, token) => { + if (expected > 0 && read is { IsCompleted: true }) + { + await read; + Assert.Fail("Acquisition completed before the held partition lock was released."); + } + var actual = Convert.ToInt32(await observer.ExecuteScalarAsync(token), CultureInfo.InvariantCulture); if (lastTry) { diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 9a750f6d287..8cc19b2ff6b 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -1,4 +1,5 @@ using System.Data; +using System.Data.Common; using System.Reflection; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging.Abstractions; @@ -18,6 +19,37 @@ namespace Tester.AdoNet.Streaming; [TestArea("Streaming")] public class AdoNetRecoverableStreamTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Load_ProviderDatabaseFailureHonorsCancellationWithoutHidingOtherFailures(bool canceled) + { + var storage = new BlockingRelationalStorage(); + var source = CreateSource(storage); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var load = source.Load(cancellation.Token).AsTask(); + await storage.AcquisitionStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var databaseFailure = new AcquisitionDatabaseException("Provider command canceled."); + if (canceled) + { + cancellation.Cancel(); + } + + storage.FailAcquisition(databaseFailure); + if (canceled) + { + var exception = await Assert.ThrowsAnyAsync(() => load); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Same(databaseFailure, exception.InnerException); + } + else + { + Assert.Same(databaseFailure, await Assert.ThrowsAsync(() => load)); + } + + Assert.True(source.AcquisitionCompletion.IsCompleted); + } + [Fact] public async Task ConcurrentCreateAdapterCallsConstructOnce() { @@ -675,6 +707,8 @@ internal override async ValueTask CreateAdapterCore(RelationalOrl } } + private sealed class AcquisitionDatabaseException(string message) : DbException(message); + private sealed class BlockingRelationalStorage : IRelationalStorage { private readonly TaskCompletionSource acquisition = @@ -689,6 +723,8 @@ private sealed class BlockingRelationalStorage : IRelationalStorage public string ConnectionString => string.Empty; + public void FailAcquisition(Exception exception) => acquisition.SetException(exception); + public void CompleteAcquisition(long ownerEpoch) => acquisition.SetResult(new( "service", diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs index a9fc42bbe85..56cd5cf4e34 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs @@ -190,10 +190,14 @@ public async Task> ReadAsync( case nameof(DbStoredQueries.GetStreamPartitionBoundsKey): rows.Add(new() { - ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, - ["OwnerEpoch"] = 1L, ["NextMessageId"] = (long)partition.Payloads.Length + 1, + ["ServiceId"] = "service", + ["ProviderId"] = "provider", + ["QueueId"] = queue, + ["OwnerEpoch"] = 1L, + ["NextMessageId"] = (long)partition.Payloads.Length + 1, ["Checkpoint"] = Checkpoints.GetValueOrDefault(queue), - ["EarliestMessageId"] = 1L, ["TailMessageId"] = (long)partition.Payloads.Length, + ["EarliestMessageId"] = 1L, + ["TailMessageId"] = (long)partition.Payloads.Length, }); break; case nameof(DbStoredQueries.ReadStreamMessagesKey): @@ -204,11 +208,14 @@ public async Task> ReadAsync( { rows.Add(new() { - ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, + ["ServiceId"] = "service", + ["ProviderId"] = "provider", + ["QueueId"] = queue, ["MessageId"] = (long)i + 1, ["StreamIdBytes"] = partition.Stream.FullKey.ToArray(), ["StreamNamespaceLength"] = partition.Stream.Namespace.Length, - ["CreatedOn"] = DateTime.UnixEpoch, ["Payload"] = partition.Payloads[i], + ["CreatedOn"] = DateTime.UnixEpoch, + ["Payload"] = partition.Payloads[i], }); } @@ -216,10 +223,15 @@ public async Task> ReadAsync( case nameof(DbStoredQueries.CleanupStreamMessagesKey): rows.Add(new() { - ["Ran"] = false, ["DeletedCount"] = 0, ["DeletedThroughMessageId"] = null, - ["HardDeletedCount"] = 0, ["HardDeletedFromMessageId"] = null, - ["HardDeletedThroughMessageId"] = null, ["Checkpoint"] = Checkpoints.GetValueOrDefault(queue), - ["EarliestMessageId"] = 1L, ["TailMessageId"] = (long)partition.Payloads.Length, + ["Ran"] = false, + ["DeletedCount"] = 0, + ["DeletedThroughMessageId"] = null, + ["HardDeletedCount"] = 0, + ["HardDeletedFromMessageId"] = null, + ["HardDeletedThroughMessageId"] = null, + ["Checkpoint"] = Checkpoints.GetValueOrDefault(queue), + ["EarliestMessageId"] = 1L, + ["TailMessageId"] = (long)partition.Payloads.Length, }); break; case nameof(DbStoredQueries.AdvanceStreamCheckpointKey): @@ -227,8 +239,12 @@ public async Task> ReadAsync( Checkpoints[queue] = checkpoint; rows.Add(new() { - ["ServiceId"] = "service", ["ProviderId"] = "provider", ["QueueId"] = queue, - ["OwnerEpoch"] = 1L, ["Checkpoint"] = checkpoint, ["Updated"] = true, + ["ServiceId"] = "service", + ["ProviderId"] = "provider", + ["QueueId"] = queue, + ["OwnerEpoch"] = 1L, + ["Checkpoint"] = checkpoint, + ["Updated"] = true, }); break; default: diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs index fa8c3a28f4d..dd003979300 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerAzuriteTests.cs @@ -15,17 +15,28 @@ namespace ServiceBus.Tests.CheckpointerTests; [TestProvider("EventHub")] [TestArea("Streaming")] [TestCategory("EventHub"), TestCategory("AzureStorage"), TestCategory("Streaming")] -public sealed class AzureTableStreamQueueCheckpointerAzuriteTests +public sealed class AzureTableStreamQueueCheckpointerAzuriteTests : IDisposable { - private static CancellationToken TestCancellation => TestContext.Current.CancellationToken; + private readonly CancellationTokenSource _operationCancellation = + CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + private CancellationToken TestCancellation => _operationCancellation.Token; private static readonly DateTime UpdateTime = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + public AzureTableStreamQueueCheckpointerAzuriteTests() + => _operationCancellation.CancelAfter(TimeSpan.FromMinutes(1)); + + public void Dispose() + { + _operationCancellation.Cancel(); + _operationCancellation.Dispose(); + } + [Theory] [InlineData(false)] [InlineData(true)] public async Task TwoCheckpointers_InitialInsertRace_PreservesMonotonicCheckpoint(bool higherWinsInsert) { - await using var fixture = new Fixture(); + await using var fixture = new Fixture(TestCancellation); var lowRequests = new TableRequests(fixture.TableName, holdInsert: true); var highRequests = new TableRequests(fixture.TableName, holdInsert: true); var low = await fixture.CreateCheckpointer(lowRequests); @@ -73,14 +84,15 @@ await Task.WhenAll( { lowRequests.ReleaseInsert.TrySetResult(); highRequests.ReleaseInsert.TrySetResult(); - await Task.WhenAll(low.FlushAsync(TestCancellation), high.FlushAsync(TestCancellation)); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await Task.WhenAll(low.FlushAsync(cleanup.Token), high.FlushAsync(cleanup.Token)); } } [Fact] public async Task TwoCheckpointers_StaleETagReloadsAndRetriesHigherNumericCheckpoint() { - await using var fixture = new Fixture(); + await using var fixture = new Fixture(TestCancellation); var winner = await fixture.CreateCheckpointer(); Assert.Equal(string.Empty, await winner.Load(TestCancellation)); await Persist(winner, "8"); @@ -115,7 +127,7 @@ public async Task TwoCheckpointers_StaleETagReloadsAndRetriesHigherNumericCheckp [Fact] public async Task TwoCheckpointers_StaleETagRetriesPendingHigherCheckpointWithinSameFlush() { - await using var fixture = new Fixture(); + await using var fixture = new Fixture(TestCancellation); var first = await fixture.CreateCheckpointer(); await first.Load(TestCancellation); await Persist(first, "8"); @@ -142,7 +154,7 @@ public async Task TwoCheckpointers_StaleETagRetriesPendingHigherCheckpointWithin [InlineData(true)] public async Task TwoCheckpointers_DeleteRecreateInvalidatesCachedETagAndRecovers(bool recreateBeforeUpdate) { - await using var fixture = new Fixture(); + await using var fixture = new Fixture(TestCancellation); var oldRequests = new TableRequests(fixture.TableName); var old = await fixture.CreateCheckpointer(oldRequests); await old.Load(TestCancellation); @@ -173,7 +185,7 @@ public async Task TwoCheckpointers_DeleteRecreateInvalidatesCachedETagAndRecover await fixture.AssertCheckpoint("11"); } - private static async Task Persist(IStreamQueueCheckpointer checkpointer, string checkpoint) + private async Task Persist(IStreamQueueCheckpointer checkpointer, string checkpoint) { checkpointer.Update(checkpoint, UpdateTime, TestCancellation); await checkpointer.FlushAsync(TestCancellation); @@ -185,8 +197,9 @@ private sealed class Fixture : IAsyncDisposable private const string Service = "etag-service"; private const string Partition = "shard-1"; private readonly TableServiceClient _serviceClient; + private readonly CancellationToken _cancellationToken; - public Fixture() + public Fixture(CancellationToken cancellationToken) { if (TestDefaultConfiguration.UseAadAuthentication ? TestDefaultConfiguration.TableEndpoint is null @@ -195,6 +208,7 @@ public Fixture() throw Xunit.Sdk.SkipException.ForSkip("Azure Table tests require the existing AzureStorage test connection configuration."); } + _cancellationToken = cancellationToken; _serviceClient = AzureStorageOperationOptionsExtensions.GetTableServiceClient(); Table = _serviceClient.GetTableClient(TableName); } @@ -225,24 +239,24 @@ await AzureTableStreamQueueCheckpointer.Create( PersistInterval = TimeSpan.FromMinutes(1), CheckpointComparer = StreamCheckpointComparers.Numeric, }, - Provider, Partition, Service, NullLoggerFactory.Instance, TestCancellation)); + Provider, Partition, Service, NullLoggerFactory.Instance, _cancellationToken)); } public async Task ReadEntity() { var key = StreamQueueCheckpointEntity.Create(string.Empty, Provider, Service, Partition); var response = await Table.GetEntityAsync( - key.PartitionKey, key.RowKey, cancellationToken: TestCancellation); + key.PartitionKey, key.RowKey, cancellationToken: _cancellationToken); return response.Value; } public async Task AssertCheckpoint(string expected) { var fresh = await CreateCheckpointer(); - Assert.Equal(expected, await fresh.Load(TestCancellation)); + Assert.Equal(expected, await fresh.Load(_cancellationToken)); Assert.True(fresh.CheckpointExists); var entities = new List(); - await foreach (var entity in Table.QueryAsync(cancellationToken: TestCancellation)) + await foreach (var entity in Table.QueryAsync(cancellationToken: _cancellationToken)) { entities.Add(entity); } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index b3a2141f1d4..a708a88ede4 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Net; using System.Reactive.Linq; using NSubstitute; using Orleans.Configuration; @@ -22,6 +23,7 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking { var streamId = new QualifiedStreamId("provider", StreamId.Create("unregister", Guid.NewGuid())); var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var token = new EventSequenceTokenV2(1); var cache = new ScriptedQueueCache(); cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); @@ -61,8 +63,15 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking Assert.Same(consumer, failed.Consumer); Assert.Equal(token, failed.SequenceToken); Assert.Equal(streamId.StreamId, failed.StreamId); - Assert.Equal(StreamingEvents.SubscriptionUnregistrationStage.Requested, - Assert.Single(events.OfType()).Stage); + Assert.Equal(streamId.ProviderName, failed.StreamProvider); + Assert.Equal(siloAddress, failed.SiloAddress); + var requested = Assert.Single(events.OfType()); + Assert.Equal(StreamingEvents.SubscriptionUnregistrationStage.Requested, requested.Stage); + Assert.Equal(streamId.ProviderName, requested.StreamProvider); + Assert.Equal(streamId.StreamId, requested.StreamId); + Assert.Equal(siloAddress, requested.SiloAddress); + Assert.Same(consumer, requested.Consumer); + Assert.Null(requested.Exception); Assert.False(outcome.Task.IsCompleted); var exception = new InvalidOperationException("unregistration storage failure"); @@ -80,6 +89,8 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking failUnregistration ? StreamingEvents.SubscriptionUnregistrationStage.Failed : StreamingEvents.SubscriptionUnregistrationStage.Completed, completed.Stage); Assert.Equal(streamId.StreamId, completed.StreamId); + Assert.Equal(streamId.ProviderName, completed.StreamProvider); + Assert.Equal(siloAddress, completed.SiloAddress); Assert.Same(consumer, completed.Consumer); Assert.Same(failUnregistration ? exception : null, completed.Exception); _ = pubSub.Received(1).UnregisterConsumer(subscriptionId, streamId, Arg.Any()); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs index f2da41d49cc..c111db00348 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs @@ -401,13 +401,13 @@ public async Task Receiver_ShutdownPreventsLateSourceBatchFromRepopulatingCache( using var cache = CreateCache(64, adapter, pool); var receiver = new RecoverableStreamReceiver(source, adapter, cache, new Checkpointer(), false); var read = receiver.GetQueueMessagesAsync(1000, CancellationToken.None); - await source.Started.Task; + await source.Started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); await receiver.Shutdown(Timeout.InfiniteTimeSpan); Assert.True(source.ReadCancellation.IsCancellationRequested); source.Completion.SetResult([Record(1, 40), Record(2, 40)]); - Assert.Empty(await read); + Assert.Empty(await read.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); Assert.Equal(0, adapter.PackCount); Assert.Equal(0, cache.SizeInBytes); Assert.Equal(0, pool.AllocationCount); From 1e5b1f42e7e022d166681d5234a370e527e4387a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 22:00:03 -0700 Subject: [PATCH 59/90] test(streaming): expose acquisition lock diagnostics --- .../Streaming/AdoNetQueueAdapterTests.cs | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 6906d908d01..98e4b28587e 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -250,7 +250,8 @@ public async Task AcquisitionCancellation_BlockedCommandSettlesBeforeReplacement await using var observerConnection = CreateConnection(); await observerConnection.OpenAsync(cancellationToken); await using var observer = observerConnection.CreateCommand(); - observer.CommandText = GetBlockedAcquisitionQuery(observerConnection); + var serverVersion = observerConnection.ServerVersion; + observer.CommandText = GetBlockedAcquisitionQuery(serverVersion); AddParameter(observer, "LockOwner", lockOwner); using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var lockReleased = false; @@ -325,11 +326,17 @@ Task WaitForBlockedAcquisitions(int expected) Assert.Fail("Acquisition completed before the held partition lock was released."); } - var actual = Convert.ToInt32(await observer.ExecuteScalarAsync(token), CultureInfo.InvariantCulture); + var result = await observer.ExecuteScalarAsync(token); + Assert.NotNull(result); + var actual = Convert.ToInt32(result, CultureInfo.InvariantCulture); if (lastTry) { + var diagnostics = invariant == AdoNetInvariants.InvariantNameMySql && actual != expected + ? await GetMySqlLockDiagnostics(token) + : string.Empty; Assert.True(actual == expected, - $"{invariant} partition {serviceId}/{providerId}/{partitionId}: expected {expected} acquisitions blocked by session {lockOwner}, observed {actual}."); + $"{invariant} server {serverVersion}, partition {serviceId}/{providerId}/{partitionId}: expected {expected} acquisitions blocked by session {lockOwner}, observed {actual}. " + + $"Read status: {read?.Status}; worker threads: {ThreadPool.ThreadCount}; queued work: {ThreadPool.PendingWorkItemCount}.{Environment.NewLine}{diagnostics}"); } return actual == expected; @@ -338,6 +345,42 @@ Task WaitForBlockedAcquisitions(int expected) TimeSpan.FromMilliseconds(50), cancellationToken, predicateExpression: $"{invariant} acquisition lock wait count for session {lockOwner} becomes {expected}"); + + async Task GetMySqlLockDiagnostics(CancellationToken token) + { + var rows = new List(); + await using var command = observerConnection.CreateCommand(); + command.CommandTimeout = 5; + AddParameter(command, "LockOwner", lockOwner); + command.CommandText = + """ + SELECT ID, COMMAND, STATE, TIME, LEFT(INFO, 256) + FROM information_schema.PROCESSLIST + WHERE DB = DATABASE() AND ID <> CONNECTION_ID() + AND (ID = @LockOwner OR INFO LIKE '%StreamPartition%') + LIMIT 10 + """; + await ReadRows(); + command.CommandText = + """ + SELECT trx_id, trx_state, trx_mysql_thread_id, trx_requested_lock_id, LEFT(trx_query, 256) + FROM information_schema.INNODB_TRX + WHERE trx_mysql_thread_id = @LockOwner + OR (trx_state = 'LOCK WAIT' AND trx_query LIKE '%StreamPartition%') + LIMIT 10 + """; + await ReadRows(); + return string.Join(Environment.NewLine, rows); + + async Task ReadRows() + { + await using var reader = await command.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) + { + rows.Add(string.Join(" | ", Enumerable.Range(0, reader.FieldCount).Select(reader.GetValue))); + } + } + } } private DbConnection CreateConnection() @@ -349,14 +392,14 @@ private DbConnection CreateConnection() _ => throw new NotSupportedException(invariant), }; - private string GetBlockedAcquisitionQuery(DbConnection connection) + private string GetBlockedAcquisitionQuery(string serverVersion) => invariant switch { AdoNetInvariants.InvariantNameSqlServer => "SELECT COUNT(*) FROM sys.dm_exec_requests WHERE blocking_session_id = @LockOwner", AdoNetInvariants.InvariantNamePostgreSql => "SELECT COUNT(*) FROM pg_locks WHERE NOT granted AND @LockOwner = ANY(pg_blocking_pids(pid))", - AdoNetInvariants.InvariantNameMySql when connection.ServerVersion.Contains("MariaDB", StringComparison.OrdinalIgnoreCase) => + AdoNetInvariants.InvariantNameMySql when serverVersion.Contains("MariaDB", StringComparison.OrdinalIgnoreCase) => """ SELECT COUNT(*) FROM information_schema.INNODB_LOCK_WAITS AS W INNER JOIN information_schema.INNODB_TRX AS B ON B.trx_id = W.blocking_trx_id From 642a8ce5f4233f5b1dd2a3639eefc1bfa3191d1a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 21:51:02 -0700 Subject: [PATCH 60/90] fix: refresh stale client routes at gateway owners --- .../GrainDirectory/ClientDirectory.cs | 100 ++++++++++- .../GrainDirectory/ClientGrainLocator.cs | 12 +- .../GrainDirectory/ILocalClientDirectory.cs | 1 + .../Directory/ClientDirectoryTests.cs | 163 ++++++++++++++++++ .../ClientObserverRoutingTests.cs | 120 +++++++++++++ 5 files changed, 392 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index 5d6d2a376dc..4e4cae017d6 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -56,6 +56,7 @@ internal sealed partial class ClientDirectory : SystemTarget, ILocalClientDirect private ImmutableHashSet _localClients = ImmutableHashSet.Empty; private ImmutableDictionary> _currentSnapshot = ImmutableDictionary>.Empty; private ImmutableDictionary ConnectedClients, long Version)> _table = ImmutableDictionary ConnectedClients, long Version)>.Empty; + private volatile ImmutableDictionary _pendingRefreshes = ImmutableDictionary.Empty; // For synchronization with remote silos. private readonly AdmissionGate _publicationGate = new(); @@ -102,6 +103,17 @@ public ValueTask> Lookup(GrainId grainId) async ValueTask> LookupClientAsync(GrainId grainId) { + await RefreshInvalidatedRoutes(); + if (TryLocalLookup(grainId, out var refreshedRoutes)) + { + if (ShouldPublish()) + { + _schedulePublishUpdate(); + } + + return refreshedRoutes; + } + var seed = Random.Shared.Next(); var attemptsRemaining = 5; List? result = null; @@ -153,8 +165,27 @@ async ValueTask> LookupClientAsync(GrainId grainId) public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List? addresses) { EnsureRefreshed(); + var pendingRefreshes = _pendingRefreshes; if (_currentSnapshot.TryGetValue(grainId, out var clientRoutes) && clientRoutes.Count > 0) { + if (pendingRefreshes.Count > 0) + { + foreach (var route in clientRoutes) + { + if (pendingRefreshes.ContainsKey(route.SiloAddress!)) + { + clientRoutes = clientRoutes.FindAll(candidate => !pendingRefreshes.ContainsKey(candidate.SiloAddress!)); + break; + } + } + } + + if (clientRoutes.Count == 0) + { + addresses = null; + return false; + } + addresses = clientRoutes; return true; } @@ -163,6 +194,65 @@ public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List 0 } pending) + { + foreach (var entry in pending) + { + ImmutableDictionary versionVector; + lock (_lockObj) + { + EnsureRefreshed(); + if (!_pendingRefreshes.TryGetValue(entry.Key, out var token) || !ReferenceEquals(token, entry.Value)) + { + continue; + } + + versionVector = _table.ToImmutableDictionary(e => e.Key, e => e.Value.Version); + } + + var remote = _grainFactory.GetSystemTarget(Constants.ClientDirectoryType, entry.Key); + var delta = await remote.GetClientRoutes(versionVector, _stoppingCts.Token); + lock (_lockObj) + { + UpdateRoutingTable(delta); + + // Keep the versioned row while refreshing: a same-version response confirms it, + // and a delayed response only completes the invalidation which initiated its read. + if (_pendingRefreshes.TryGetValue(entry.Key, out var token) && ReferenceEquals(token, entry.Value)) + { + _pendingRefreshes = _pendingRefreshes.Remove(entry.Key); + } + } + } + } + } + private void EnsureRefreshed() { if (IsStale()) @@ -207,10 +297,14 @@ public Task OnUpdateClientRoutes( CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - EnsureRefreshed(); + ImmutableDictionary ConnectedClients, long Version)> table; + lock (_lockObj) + { + EnsureRefreshed(); + table = _table; + } // Return a collection containing all missing or out-dated routes, based on the known-routes version vector provided by the caller. - var table = _table; var resultBuilder = ImmutableDictionary.CreateBuilder ConnectedClients, long Version)>(); foreach (var entry in table) { @@ -273,6 +367,8 @@ private void UpdateRoutingTable(ImmutableDictionary InvalidateCache(address.GrainId); public bool TryLookupInCache(GrainId grainId, [NotNullWhen(true)] out GrainAddress? address) { diff --git a/src/Orleans.Runtime/GrainDirectory/ILocalClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ILocalClientDirectory.cs index e9dc3aa625e..06c6ccdb144 100644 --- a/src/Orleans.Runtime/GrainDirectory/ILocalClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ILocalClientDirectory.cs @@ -8,5 +8,6 @@ internal interface ILocalClientDirectory { bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List? addresses); ValueTask> Lookup(GrainId grainId); + void InvalidateCache(GrainId grainId); } } diff --git a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs index a81b29072e6..7e67e26b019 100644 --- a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs +++ b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs @@ -333,6 +333,157 @@ IRemoteClientDirectory CreateRemoteDirectory() } } + [Fact] + public async Task InvalidationRefreshesAllCachedGateways() + { + var clientId = Client("dropped"); + var observerId = ObserverGrainId.Create(ClientGrainId.Create("dropped")).GrainId; + var remoteSilos = Enumerable.Range(1, 6).Select(index => Silo($"127.0.0.1:{200 + index}@100")).ToArray(); + var refreshed = new HashSet(); + foreach (var silo in remoteSilos) + { + var remote = await AddRemoteClient(silo, clientId); + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(info => + { + var versions = info.ArgAt>(0); + Assert.Equal(refreshed.Add(silo) ? 2 : 3, versions[silo]); + return Task.FromResult(CreateRoutes(silo, 3)); + }); + } + + Assert.True(_directory.TryLocalLookup(clientId, out var cached)); + Assert.Equal(remoteSilos.Length, cached.Count); + var locator = new ClientGrainLocator(_localSiloDetails, _directory); + locator.InvalidateCache(new GrainAddress { GrainId = observerId, SiloAddress = _localSilo }); + Assert.False(locator.TryLookupInCache(observerId, out _)); + + Assert.Null(await locator.Lookup(observerId)); + Assert.Equal(remoteSilos.OrderBy(silo => silo.ToString()), refreshed.OrderBy(silo => silo.ToString())); + + // The newer owner versions also protect against delayed gossip of the dropped routes. + foreach (var silo in remoteSilos) + { + await _directory.OnUpdateClientRoutes(CreateRoutes(silo, 2, clientId), TestContext.Current.CancellationToken); + } + + Assert.False(locator.TryLookupInCache(observerId, out _)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InvalidationRevalidatesReconnectedGateway(bool sameVersion) + { + var clientId = Client("reconnected"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(info => + { + var versions = info.ArgAt>(0); + Assert.Equal(2, versions[remoteSilo]); + return Task.FromResult(sameVersion + ? ImmutableDictionary, long)>.Empty + : CreateRoutes(remoteSilo, 4, clientId)); + }); + + var locator = new ClientGrainLocator(_localSiloDetails, _directory); + locator.InvalidateCache(clientId); + Assert.False(locator.TryLookupInCache(clientId, out _)); + + var address = await locator.Lookup(clientId); + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), address); + Assert.True(locator.TryLookupInCache(clientId, out var cached)); + Assert.Equal(address, cached); + _ = remote.Received(1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task RefreshRetainsConcurrentInvalidation() + { + var clientId = Client("reconnected"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + var firstResponse = new TaskCompletionSource, long)>>( + TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(_ => + Interlocked.Increment(ref calls) == 1 ? firstResponse.Task : Task.FromResult(CreateRoutes(remoteSilo, 4, clientId))); + + _directory.InvalidateCache(clientId); + var lookup = _directory.Lookup(clientId).AsTask(); + Assert.Equal(1, calls); + Assert.False(lookup.IsCompleted); + + _directory.InvalidateCache(clientId); + firstResponse.SetResult(ImmutableDictionary, long)>.Empty); + var address = Assert.Single(await lookup.WaitAsync(TestContext.Current.CancellationToken)); + + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), address); + _ = remote.Received(2).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task InvalidatedGatewayRefreshFailureRemainsRetryable() + { + var clientId = Client("remote"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + var failure = new TimeoutException("Client directory owner did not respond."); + var respond = false; + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(_ => + respond ? Task.FromResult(CreateRoutes(remoteSilo, 4, clientId)) : throw failure); + + _directory.InvalidateCache(clientId); + Assert.Same(failure, await Assert.ThrowsAsync(() => _directory.Lookup(clientId).AsTask())); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + + respond = true; + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); + } + + [Fact] + public async Task InvalidationKeepsLocallyConnectedClientAvailable() + { + var clientId = Client("multiple-gateways"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + remote.GetClientRoutes(default!, Arg.Any()) + .ReturnsForAnyArgs(Task.FromResult(ImmutableDictionary, long)>.Empty)); + SetLocalClients([clientId]); + + var locator = new ClientGrainLocator(_localSiloDetails, _directory); + locator.InvalidateCache(clientId); + Assert.Equal(Gateway.GetClientActivationAddress(clientId, _localSilo), await locator.Lookup(clientId)); + _ = remote.DidNotReceive().GetClientRoutes( + Arg.Any>(), + Arg.Any()); + + SetLocalClients([]); + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), await locator.Lookup(clientId)); + _ = remote.Received(1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task InvalidationRemovesTerminatedGateway() + { + var clientId = Client("remote"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + _directory.InvalidateCache(clientId); + + _clusterMembershipService.UpdateSiloStatus(remoteSilo, SiloStatus.Dead, "remoteSilo"); + Assert.Empty(await _directory.Lookup(clientId)); + _ = remote.DidNotReceive().GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + [Fact] public async Task PublishChangesSuccessTests() { @@ -829,6 +980,18 @@ public async Task PublishingStopsBeforeMembershipShutdownBegins() private static GrainId Client(string id) => ClientGrainId.Create(id).GrainId; + private async Task AddRemoteClient(SiloAddress silo, GrainId clientId) + { + _clusterMembershipService.UpdateSiloStatus(silo, SiloStatus.Active, silo.ToString()); + await _directory.OnUpdateClientRoutes(CreateRoutes(silo, 2, clientId), TestContext.Current.CancellationToken); + return _remoteDirectories[silo]; + } + + private static ImmutableDictionary, long)> CreateRoutes( + SiloAddress silo, long version, params GrainId[] clients) + => ImmutableDictionary, long)>.Empty.Add( + silo, (clients.ToImmutableHashSet().Add(HostedClient.CreateHostedClientGrainId(silo).GrainId), version)); + private long SetLocalClients(List clients) { var clientCollectionVersion = ++_expectedConnectedClientsVersion; diff --git a/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs index 380aea9ec7b..05d0704e25c 100644 --- a/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs +++ b/test/Orleans.Runtime.Tests/ClientConnectionTests/ClientObserverRoutingTests.cs @@ -1,8 +1,10 @@ +using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Orleans; using Orleans.Runtime; using Orleans.Runtime.GrainDirectory; +using Orleans.Runtime.Messaging; using Orleans.TestingHost; using TestExtensions; using Xunit; @@ -18,6 +20,101 @@ public class ClientObserverRoutingTests public static ClientNotAvailableException CreateUnavailableClientException() => new(ClientGrainId.Create().GrainId); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PersistentlyStaleClientRoutesRefreshGatewayOwners(bool unavailable) + { + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureHost(hostBuilder => TestDefaultConfiguration.ConfigureHostConfiguration(hostBuilder.Configuration)); + builder.ConfigureSilo((_, siloBuilder) => siloBuilder.ConfigureServices(services => + { + services.AddSingleton(services => + new(Assert.IsType(services.GetRequiredService().Gateway))); + services.AddSingleton(services => services.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(services => services.GetRequiredService()); + })); + await using var cluster = builder.Build(); + await cluster.DeployAsync(TestContext.Current.CancellationToken); + + var sender = cluster.Silos[0].ServiceProvider; + var receiver = cluster.Silos[2].ServiceProvider; + var observer = new EchoObserver(); + var target = unavailable + ? ObserverGrainId.Create(ClientGrainId.Create()).GrainId + : receiver.GetRequiredService().CreateObjectReference(observer).GetGrainId(); + Assert.True(ClientGrainId.TryParse(target, out var clientId)); + var directories = cluster.Silos.Select(silo => silo.ServiceProvider.GetRequiredService()).ToArray(); + var connections = cluster.Silos.Select(silo => silo.ServiceProvider.GetRequiredService()).ToArray(); + + // Hold publication while real directories exchange the connected snapshots, then drop the + // simulated gateway connections. Each replica retains the other gateways' stale routes. + foreach (var directory in directories) + { + new ClientDirectory.TestAccessor(directory).SchedulePublishUpdate = static () => { }; + } + + foreach (var connection in connections) + { + connection.SetAdditionalClients([clientId.GrainId]); + } + + var routes = ImmutableDictionary, long)>.Empty; + for (var i = 0; i < directories.Length; i++) + { + var snapshot = await directories[i].GetClientRoutes( + ImmutableDictionary.Empty, TestContext.Current.CancellationToken); + var silo = cluster.Silos[i].SiloAddress; + routes = routes.Add(silo, snapshot[silo]); + } + + foreach (var directory in directories) + { + await directory.OnUpdateClientRoutes(routes, TestContext.Current.CancellationToken); + } + + foreach (var connection in connections) + { + connection.SetAdditionalClients([]); + } + + for (var i = 0; i < directories.Length; i++) + { + Assert.True(directories[i].TryLocalLookup(clientId.GrainId, out var staleRoutes)); + var expectedSilos = cluster.Silos + .Where((_, index) => index != i || !unavailable && index == 2) + .Select(silo => silo.SiloAddress) + .OrderBy(silo => silo.ToString()); + Assert.Equal(expectedSilos, staleRoutes.Select(route => route.SiloAddress!).OrderBy(silo => silo.ToString())); + } + + var redirect = sender.GetRequiredService(); + redirect.Arm(clientId.GrainId, cluster.Silos[0].SiloAddress); + var reference = sender.GetRequiredService().GetGrain(target); + if (unavailable) + { + await Assert.ThrowsAsync( + () => reference.Echo(42).WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken)); + Assert.Empty(await directories[0].Lookup(clientId.GrainId)); + Assert.Equal(0, observer.CallCount); + } + else + { + Assert.Equal(42, await reference.Echo(42).WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken)); + var liveRoute = Assert.Single(await directories[0].Lookup(clientId.GrainId)); + Assert.Equal(cluster.Silos[2].SiloAddress, liveRoute.SiloAddress); + Assert.Equal(43, await reference.Echo(43).WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken)); + Assert.Equal(2, observer.CallCount); + } + + Assert.Equal(1, redirect.StaleRoutesReturned); + foreach (var silo in cluster.Silos) + { + Assert.Null(silo.ServiceProvider.GetRequiredService().FindTarget(target)); + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -92,6 +189,27 @@ public Task Echo(int value) } } + private sealed class ControlledConnectedClients(IConnectedClientCollection inner) : IConnectedClientCollection + { + private ImmutableList _additionalClients = []; + private long _version; + + public long Version => inner.Version + Interlocked.Read(ref _version); + + public List GetConnectedClientIds() + { + var clients = inner.GetConnectedClientIds(); + clients.AddRange(Volatile.Read(ref _additionalClients)); + return clients; + } + + public void SetAdditionalClients(ImmutableList clients) + { + Volatile.Write(ref _additionalClients, clients); + Interlocked.Increment(ref _version); + } + } + private sealed class RedirectOnceClientDirectory(ClientDirectory inner) : ILocalClientDirectory { private GrainId _clientId; @@ -117,6 +235,8 @@ public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List> Lookup(GrainId grainId) => inner.Lookup(grainId); + + public void InvalidateCache(GrainId grainId) => inner.InvalidateCache(grainId); } } From 7d2608181332cef64f1138ecc893a8c57acba869 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 21:57:09 -0700 Subject: [PATCH 61/90] fix: scope client route refreshes to requested clients --- .../GrainDirectory/ClientDirectory.cs | 72 ++++++++--------- .../Directory/ClientDirectoryTests.cs | 80 +++++++++++++++++++ 2 files changed, 116 insertions(+), 36 deletions(-) diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index 4e4cae017d6..5d54bab7435 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -103,20 +103,19 @@ public ValueTask> Lookup(GrainId grainId) async ValueTask> LookupClientAsync(GrainId grainId) { - await RefreshInvalidatedRoutes(); - if (TryLocalLookup(grainId, out var refreshedRoutes)) + var result = await RefreshInvalidatedRoutes(grainId); + if (result is not null) { if (ShouldPublish()) { _schedulePublishUpdate(); } - return refreshedRoutes; + return result; } var seed = Random.Shared.Next(); var attemptsRemaining = 5; - List? result = null; while (attemptsRemaining-- > 0 && _remoteDirectories is var remoteDirectories && remoteDirectories.Length > 0) { try @@ -139,26 +138,21 @@ async ValueTask> LookupClientAsync(GrainId grainId) LogErrorCallingRemoteClientDirectory(exception); } - // Try again to find the requested client's routes. - // Note that this occurs whether the remote update call succeeded or failed. - if (TryLocalLookup(grainId, out result) && result.Count > 0) + // Discovery can reveal a route whose owner is already pending refresh. + result = await RefreshInvalidatedRoutes(grainId); + if (result is not null) { break; } } + result ??= await RefreshInvalidatedRoutes(grainId); if (ShouldPublish()) { _schedulePublishUpdate(); } - // Try one last time to find the requested client's routes. - if (result is null && !TryLocalLookup(grainId, out result)) - { - result = []; - } - - return result; + return result ?? []; } } @@ -218,36 +212,42 @@ public void InvalidateCache(GrainId grainId) } } - private async Task RefreshInvalidatedRoutes() + private async ValueTask?> RefreshInvalidatedRoutes(GrainId grainId) { - while (_pendingRefreshes is { Count: > 0 } pending) + while (true) { - foreach (var entry in pending) + SiloAddress silo; + object token; + ImmutableDictionary versionVector; + lock (_lockObj) { - ImmutableDictionary versionVector; - lock (_lockObj) + if (TryLocalLookup(grainId, out var addresses)) { - EnsureRefreshed(); - if (!_pendingRefreshes.TryGetValue(entry.Key, out var token) || !ReferenceEquals(token, entry.Value)) - { - continue; - } - - versionVector = _table.ToImmutableDictionary(e => e.Key, e => e.Value.Version); + return addresses; } - var remote = _grainFactory.GetSystemTarget(Constants.ClientDirectoryType, entry.Key); - var delta = await remote.GetClientRoutes(versionVector, _stoppingCts.Token); - lock (_lockObj) + if (!_currentSnapshot.TryGetValue(grainId, out var candidates)) { - UpdateRoutingTable(delta); + return null; + } - // Keep the versioned row while refreshing: a same-version response confirms it, - // and a delayed response only completes the invalidation which initiated its read. - if (_pendingRefreshes.TryGetValue(entry.Key, out var token) && ReferenceEquals(token, entry.Value)) - { - _pendingRefreshes = _pendingRefreshes.Remove(entry.Key); - } + // A cached candidate excluded by TryLocalLookup has a pending owner refresh. + silo = candidates[0].SiloAddress!; + token = _pendingRefreshes[silo]; + versionVector = _table.ToImmutableDictionary(e => e.Key, e => e.Value.Version); + } + + var remote = _grainFactory.GetSystemTarget(Constants.ClientDirectoryType, silo); + var delta = await remote.GetClientRoutes(versionVector, _stoppingCts.Token); + lock (_lockObj) + { + UpdateRoutingTable(delta); + + // Keep the versioned row while refreshing: a same-version response confirms it, + // and a delayed response only completes the invalidation which initiated its read. + if (_pendingRefreshes.TryGetValue(silo, out var currentToken) && ReferenceEquals(currentToken, token)) + { + _pendingRefreshes = _pendingRefreshes.Remove(silo); } } } diff --git a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs index 7e67e26b019..1a1ae422e01 100644 --- a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs +++ b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs @@ -369,6 +369,86 @@ public async Task InvalidationRefreshesAllCachedGateways() Assert.False(locator.TryLookupInCache(observerId, out _)); } + [Theory] + [InlineData(1, false)] + [InlineData(1, true)] + [InlineData(5, false)] + [InlineData(5, true)] + public async Task DiscoveryRefreshesNewlyRelevantGateway(int discoveryAttempt, bool refreshFails) + { + var clientId = Client("discovered"); + var otherClientId = Client("already-invalidated"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, otherClientId); + var failure = new TimeoutException("Client directory owner did not respond."); + var calls = 0; + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(info => + { + var versions = info.ArgAt>(0); + var call = Interlocked.Increment(ref calls); + if (call <= discoveryAttempt) + { + Assert.Equal(2, versions[remoteSilo]); + return Task.FromResult(call == discoveryAttempt + ? CreateRoutes(remoteSilo, 3, clientId, otherClientId) + : ImmutableDictionary, long)>.Empty); + } + + Assert.Equal(discoveryAttempt + 1, call); + Assert.Equal(3, versions[remoteSilo]); + return refreshFails + ? throw failure + : Task.FromResult(ImmutableDictionary, long)>.Empty); + }); + + _directory.InvalidateCache(otherClientId); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + if (refreshFails) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => _directory.Lookup(clientId).AsTask())); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + } + else + { + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); + Assert.True(_directory.TryLocalLookup(clientId, out _)); + } + + _ = remote.Received(discoveryAttempt + 1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task InvalidationRefreshesOnlyRequestedClientGateways() + { + var clientId = Client("requested"); + var otherClientId = Client("unrelated"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var otherSilo = Silo("127.0.0.1:333@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + var otherRemote = await AddRemoteClient(otherSilo, otherClientId); + remote.GetClientRoutes(default!, Arg.Any()) + .ReturnsForAnyArgs(Task.FromResult(ImmutableDictionary, long)>.Empty)); + otherRemote.GetClientRoutes(default!, Arg.Any()) + .ReturnsForAnyArgs(_ => throw new TimeoutException("The unrelated client's directory owner did not respond.")); + + _directory.InvalidateCache(clientId); + _directory.InvalidateCache(otherClientId); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + Assert.False(_directory.TryLocalLookup(otherClientId, out _)); + + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); + Assert.True(_directory.TryLocalLookup(clientId, out _)); + Assert.False(_directory.TryLocalLookup(otherClientId, out _)); + _ = remote.Received(1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + _ = otherRemote.DidNotReceive().GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + [Theory] [InlineData(false)] [InlineData(true)] From 57872a7932318800b13d89f9c1b6dddeb03a3002 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 22:08:56 -0700 Subject: [PATCH 62/90] test: finalize authoritative client routing regressions --- docs/site/src/content/docs/host/monitoring/signals.md | 2 +- test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/site/src/content/docs/host/monitoring/signals.md b/docs/site/src/content/docs/host/monitoring/signals.md index fa5596d22ab..5102cfda951 100644 --- a/docs/site/src/content/docs/host/monitoring/signals.md +++ b/docs/site/src/content/docs/host/monitoring/signals.md @@ -58,7 +58,7 @@ Avoid recording grain keys and state values by default. They can be high-cardina Subscribe to to correlate a consumer's delivery and cleanup transitions by provider, stream, subscription, consumer endpoint, and silo. includes the attempted position and original exception. reports when the pulling agent requests pubsub cleanup and when that request completes or fails. marks the successful durable removal. -When a client route becomes stale, the runtime resolves the client through the client directory. An available gateway receives the request; an unavailable client produces , which initiates subscription cleanup. For a cleanup timeout, capture the last observed transition and its exception: a delivery failure identifies the routing or consumer failure, a pending unregistration identifies the pubsub request, and a failed unregistration identifies its storage or callback failure. Keep diagnostic capture bounded and retain positions and identifiers rather than event payloads. +When a client route becomes stale, the runtime refreshes its candidate routes at the owning gateways while preserving the directory's versioned routing state. An available gateway receives the request; an unavailable client produces , which initiates subscription cleanup. Owner-refresh failures retain the pending refresh so that a subsequent lookup can retry. For a cleanup timeout, capture the last observed transition and its exception: a delivery failure identifies the routing or consumer failure, a pending unregistration identifies the pubsub request, and a failed unregistration identifies its storage or callback failure. Keep diagnostic capture bounded and retain positions and identifiers rather than event payloads. ## Baseline alerts diff --git a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs index 1a1ae422e01..61f4d5617bb 100644 --- a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs +++ b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs @@ -431,7 +431,8 @@ public async Task InvalidationRefreshesOnlyRequestedClientGateways() remote.GetClientRoutes(default!, Arg.Any()) .ReturnsForAnyArgs(Task.FromResult(ImmutableDictionary, long)>.Empty)); otherRemote.GetClientRoutes(default!, Arg.Any()) - .ReturnsForAnyArgs(_ => throw new TimeoutException("The unrelated client's directory owner did not respond.")); + .ReturnsForAnyArgs(_ => Task.FromException, long)>>( + new TimeoutException("The unrelated client's directory owner did not respond."))); _directory.InvalidateCache(clientId); _directory.InvalidateCache(otherClientId); From 59ee41bdd5c3b05b6224c97ad5bb24cf5cbef573 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 4 Sep 2026 22:22:40 -0700 Subject: [PATCH 63/90] test(streaming): allow MariaDB lock snapshots to refresh --- .../Streaming/AdoNetQueueAdapterTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 98e4b28587e..b5ab37c2db2 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -253,6 +253,11 @@ public async Task AcquisitionCancellation_BlockedCommandSettlesBeforeReplacement var serverVersion = observerConnection.ServerVersion; observer.CommandText = GetBlockedAcquisitionQuery(serverVersion); AddParameter(observer, "LockOwner", lockOwner); + // MariaDB refreshes its InnoDB diagnostic cache only after more than 100ms without a reader. + // Polling every 50ms would keep the initial, pre-acquisition snapshot alive. + var observationInterval = serverVersion.Contains("MariaDB", StringComparison.OrdinalIgnoreCase) + ? TimeSpan.FromMilliseconds(250) + : TimeSpan.FromMilliseconds(50); using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var lockReleased = false; Task> read = null!; @@ -342,7 +347,7 @@ Task WaitForBlockedAcquisitions(int expected) return actual == expected; }, TimeSpan.FromSeconds(15), - TimeSpan.FromMilliseconds(50), + observationInterval, cancellationToken, predicateExpression: $"{invariant} acquisition lock wait count for session {lockOwner} becomes {expected}"); From 123ff5eca353a8caeb6ed28d3e3d53d66bad0a4c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 6 Sep 2026 09:51:26 -0700 Subject: [PATCH 64/90] fix(streaming): reconcile recovery with typed cursor results --- .../Streams/KinesisPooledAdapterReceiver.cs | 10 +- .../AdoNetQueueAdapterReceiver.cs | 10 +- .../StreamCheckpointStoreState.cs | 7 +- .../Common/PooledCache/PooledQueueCache.cs | 9 +- .../RecoverableStreamQueueCache.cs | 60 +++++++--- .../RecoverableStreamReceiver.cs | 9 +- .../RecoverableStreamStartPosition.cs | 21 +++- .../PersistentStreamPullingAgent.cs | 78 +++++++----- .../Orleans.Streaming.EventHubs.cs | 5 - .../Orleans.Streaming/Orleans.Streaming.cs | 26 +++- .../RecoveryCacheMemoryOptionsTests.cs | 4 +- .../KinesisAdapterTests.cs | 6 +- .../KinesisPooledRuntimeTests.cs | 49 +++++--- .../Streams/PooledQueueCacheTests.cs | 5 +- .../PersistentStreamPullingAgentTests.cs | 5 +- .../RecoverableStreamReceiverTests.cs | 113 ++++++++++-------- .../RecoveryCacheMemoryTests.cs | 8 +- 17 files changed, 271 insertions(+), 154 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index fb5bbc020bc..0dc58fff361 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -189,13 +189,19 @@ public void AddToCache(IList messages) public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) => _cache.TryPurgeFromCache(out purgedItems); + [Obsolete("Use TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _cache.GetCacheCursor(streamId, token); - public IQueueCacheCursor GetCacheCursorAtPosition( + /// + public QueueCacheCursorResult TryGetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.TryGetCacheCursor(streamId, token); + + /// + public QueueCacheCursorResult TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) - => _cache.GetCacheCursorAtPosition(streamId, startPosition); + => _cache.TryGetCacheCursorAtPosition(streamId, startPosition); public bool IsUnderPressure() => _cache.IsUnderPressure(); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 79638afc1fa..00562c7e214 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -124,13 +124,19 @@ public Task MessagesDeliveredAsync( public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) => _inner.TryPurgeFromCache(out purgedItems); + [Obsolete("Use TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _inner.GetCacheCursor(streamId, token); - public IQueueCacheCursor GetCacheCursorAtPosition( + /// + public QueueCacheCursorResult TryGetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _inner.TryGetCacheCursor(streamId, token); + + /// + public QueueCacheCursorResult TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) - => _inner.GetCacheCursorAtPosition(streamId, startPosition); + => _inner.TryGetCacheCursorAtPosition(streamId, startPosition); public bool IsUnderPressure() => _inner.IsUnderPressure(); diff --git a/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs b/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs index 67dd2a6751b..4d92146715f 100644 --- a/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs +++ b/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs @@ -33,16 +33,17 @@ public StreamCheckpointStoreState(string checkpoint, string version) /// public bool Equals(StreamCheckpointStoreState other) => string.Equals(Checkpoint, other.Checkpoint, StringComparison.Ordinal) - && string.Equals(Version, other.Version, StringComparison.Ordinal); + && string.Equals(Version, other.Version, StringComparison.Ordinal); /// - public override bool Equals(object? obj) - => obj is StreamCheckpointStoreState other && Equals(other); + public override bool Equals(object? obj) => obj is StreamCheckpointStoreState other && Equals(other); /// public override int GetHashCode() => HashCode.Combine(Checkpoint, Version); + /// Determines whether two persisted checkpoint states are equal. public static bool operator ==(StreamCheckpointStoreState left, StreamCheckpointStoreState right) => left.Equals(right); + /// Determines whether two persisted checkpoint states differ. public static bool operator !=(StreamCheckpointStoreState left, StreamCheckpointStoreState right) => !left.Equals(right); } diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 7c06bda23e0..0fc0f54a6b9 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -386,14 +386,14 @@ private bool TryGetCacheMiss( var newestBlock = messageBlocks.First!; var newestMessage = newestBlock.Value.NewestMessage; - if (newestMessage.Compare(sequenceToken) < 0) + if (cacheDataAdapter.Compare(ref newestMessage, sequenceToken) < 0) { cacheMiss = default; return false; } var oldestMessage = messageBlocks.Last!.Value.OldestMessage; - if (oldestMessage.Compare(sequenceToken) <= 0 + if (cacheDataAdapter.Compare(ref oldestMessage, sequenceToken) <= 0 || lastPurgedToken.TryGetValue(streamId, out var entry) && EventSequenceTokenCompatibility.Compare(sequenceToken, entry.Token) >= 0) { @@ -628,7 +628,10 @@ internal void RecordDeliveryFailure(object cursorObj) cursor.State = CursorStates.Unset; cursor.CurrentBlock = null; cursor.SequenceToken = retryToken; - SetCursor(cursor, retryToken); + if (SetCursor(cursor, retryToken) is { } cacheMiss) + { + throw cacheMiss.ToException(); + } } private Cursor GetCursor(object cursorObj) diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs index b4909eaa9f0..a0ee9151709 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -320,14 +320,36 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList } /// + [Obsolete("Use TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) - => new Cursor(_cache, streamId, token); + { + var result = TryGetCacheCursor(streamId, token); + return result.Kind switch + { + QueueCacheCursorResultKind.Success => result.Cursor!, + QueueCacheCursorResultKind.CacheMiss => throw result.CacheMiss!.Value.ToException(), + _ => throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."), + }; + } /// - public IQueueCacheCursor GetCacheCursorAtPosition( + public QueueCacheCursorResult TryGetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => WrapCursorResult(_cache.TryGetCursor(streamId, token)); + + /// + public QueueCacheCursorResult TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) - => new Cursor(_cache, streamId, startPosition); + => WrapCursorResult(_cache.TryGetCursorAtPosition(streamId, startPosition)); + + private QueueCacheCursorResult WrapCursorResult(QueueCacheCursorResult result) + => result.Kind switch + { + QueueCacheCursorResultKind.Success => QueueCacheCursorResult.FromCursor(new Cursor(_cache, result.Cursor!)), + QueueCacheCursorResultKind.CacheMiss => QueueCacheCursorResult.FromCacheMiss(result.CacheMiss!.Value), + QueueCacheCursorResultKind.NotSupported => QueueCacheCursorResult.NotSupported, + _ => throw new InvalidOperationException("The cursor result is not initialized."), + }; /// public bool IsUnderPressure() => GetMaxAddCount() <= 0; @@ -406,19 +428,10 @@ private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress private readonly object _cursor; private IBatchContainer? _current; - public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? token) + public Cursor(PooledQueueCache cache, object cursor) { _cache = cache; - _cursor = cache.GetCursor(streamId, token); - } - - public Cursor( - PooledQueueCache cache, - StreamId streamId, - StreamSubscriptionStartPosition startPosition) - { - _cache = cache; - _cursor = cache.GetCursorAtPosition(streamId, startPosition); + _cursor = cursor; } public void Dispose() @@ -436,15 +449,24 @@ public void SetDeliveredThrough(StreamSequenceToken token) return _current; } + [Obsolete("Use MoveNextWithResult instead.")] public bool MoveNext() { - if (!_cache.TryGetNextMessage(_cursor, out var next)) + var result = MoveNextWithResult(); + return result.Kind switch { - return false; - } + QueueCacheCursorMoveResultKind.Success => true, + QueueCacheCursorMoveResultKind.NoData => false, + QueueCacheCursorMoveResultKind.CacheMiss => throw result.CacheMiss!.Value.ToException(), + _ => throw new InvalidOperationException("The cursor move result is not initialized."), + }; + } - _current = next; - return true; + public QueueCacheCursorMoveResult MoveNextWithResult() + { + var result = _cache.TryGetNextMessageWithResult(_cursor, out var next); + _current = result.Kind == QueueCacheCursorMoveResultKind.Success ? next : null; + return result; } public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 85441e14584..96e656c4c1c 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -327,14 +327,19 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList => _cache.TryPurgeFromCache(out purgedItems); /// + [Obsolete("Use TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => _cache.GetCacheCursor(streamId, token); /// - public IQueueCacheCursor GetCacheCursorAtPosition( + public QueueCacheCursorResult TryGetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.TryGetCacheCursor(streamId, token); + + /// + public QueueCacheCursorResult TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) - => _cache.GetCacheCursorAtPosition(streamId, startPosition); + => _cache.TryGetCacheCursorAtPosition(streamId, startPosition); /// public bool IsUnderPressure() => _cache.IsUnderPressure(); diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs index 322810d7a6a..8a7e4ea5bcc 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs @@ -1,9 +1,11 @@ +using System; + namespace Orleans.Providers.Streams.Common; /// /// Describes where a recoverable stream source begins reading. /// -public readonly struct RecoverableStreamStartPosition +public readonly struct RecoverableStreamStartPosition : IEquatable { /// /// Initializes a new instance of the struct. @@ -25,4 +27,21 @@ public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) /// Gets a value indicating whether a source without a checkpoint starts at its current tail. /// public bool StartFromNow { get; } + + /// + public bool Equals(RecoverableStreamStartPosition other) + => string.Equals(Checkpoint, other.Checkpoint, StringComparison.Ordinal) + && StartFromNow == other.StartFromNow; + + /// + public override bool Equals(object? obj) => obj is RecoverableStreamStartPosition other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(Checkpoint, StartFromNow); + + /// Determines whether two recovery start positions are equal. + public static bool operator ==(RecoverableStreamStartPosition left, RecoverableStreamStartPosition right) => left.Equals(right); + + /// Determines whether two recovery start positions differ. + public static bool operator !=(RecoverableStreamStartPosition left, RecoverableStreamStartPosition right) => !left.Equals(right); } diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index d92b827d4ff..c107a370f6e 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -494,7 +494,9 @@ private async Task DoHandshakeWithConsumer( requestedToken, cacheToken, out consumerData.PendingBatch, + out var effectiveStartToken, cursorStartToken); + cursorStartToken = effectiveStartToken; cursorRepositioned = true; } else @@ -729,15 +731,27 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( StreamSequenceToken token, StreamSequenceToken? fallbackToken, out IBatchContainer? pendingBatch, + out StreamSequenceToken? effectiveStartToken, StreamSequenceToken? startToken = null) { pendingBatch = null; + effectiveStartToken = startToken ?? token; var acquisitionResult = queueCache!.TryGetCacheCursor(streamId, startToken ?? token); if (acquisitionResult.Kind == QueueCacheCursorResultKind.CacheMiss) { - return fallbackToken is not null - ? GetCacheCursorOrThrow(streamId, fallbackToken) - : throw acquisitionResult.CacheMiss!.Value.ToException(); + if (fallbackToken is null) + { + throw acquisitionResult.CacheMiss!.Value.ToException(); + } + + effectiveStartToken = fallbackToken; + var fallbackCursor = GetCacheCursorOrThrow(streamId, fallbackToken); + if (fallbackCursor is IQueueCacheCursorProgress fallbackProgress) + { + fallbackProgress.SetDeliveredThrough(token); + } + + return fallbackCursor; } if (acquisitionResult.Kind != QueueCacheCursorResultKind.Success) @@ -755,7 +769,7 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( if (startToken is not null && !Equals(startToken, token)) { cursor.Dispose(); - return GetAdvancedCursorOrFallback(streamId, token, fallbackToken, out pendingBatch); + return GetAdvancedCursorOrFallback(streamId, token, fallbackToken, out pendingBatch, out effectiveStartToken); } var retainCursor = false; @@ -764,6 +778,7 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( var moveResult = AdvanceCursorPastToken(cursor, token, out pendingBatch); if (moveResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { + effectiveStartToken = fallbackToken; return fallbackToken is not null ? GetCacheCursorOrThrow(streamId, fallbackToken) : throw moveResult.CacheMiss!.Value.ToException(); @@ -791,6 +806,13 @@ private static bool TryAdvanceRecoveryCursor( StreamSequenceToken token, out IBatchContainer? pendingBatch) { + pendingBatch = null; + if (cursor is IQueueCacheCursorProgress progressCursor) + { + progressCursor.SetDeliveredThrough(token); + return true; + } + var retainCursor = false; try { @@ -822,38 +844,32 @@ private IQueueCacheCursor GetCursorAfterProcessedToken( StreamSequenceToken processedToken) { var restartToken = consumerData.LastSafePartitionToken ?? processedToken; - IQueueCacheCursor cursor; - try + var result = queueCache!.TryGetCacheCursor(consumerData.StreamId, restartToken); + if (result.Kind == QueueCacheCursorResultKind.CacheMiss) { - cursor = queueCache!.GetCacheCursor(consumerData.StreamId, restartToken); - } - catch (QueueCacheMissException) - { - try - { - cursor = queueCache!.GetCacheCursorAtPosition( - consumerData.StreamId, - StreamSubscriptionStartPosition.EarliestAvailable); - } - catch (NotSupportedException) + result = queueCache.TryGetCacheCursorAtPosition( + consumerData.StreamId, + StreamSubscriptionStartPosition.EarliestAvailable); + if (result.Kind == QueueCacheCursorResultKind.NotSupported) { - cursor = queueCache!.GetCacheCursor(consumerData.StreamId, null); + result = queueCache.TryGetCacheCursor(consumerData.StreamId, null); } } - if (cursor is IQueueCacheCursorProgress progressCursor) + if (result.Kind == QueueCacheCursorResultKind.CacheMiss) { - progressCursor.SetDeliveredThrough(processedToken); + throw result.CacheMiss!.Value.ToException(); } - else + + if (result.Kind != QueueCacheCursorResultKind.Success) { - if (!Equals(restartToken, processedToken)) - { - cursor.Dispose(); - cursor = queueCache.GetCacheCursor(consumerData.StreamId, processedToken); - } + throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."); + } - cursor.MoveNext(); + var cursor = result.Cursor!; + if (!TryAdvanceRecoveryCursor(cursor, processedToken, out consumerData.PendingBatch)) + { + return GetCacheCursorOrThrow(consumerData.StreamId, null); } return cursor; @@ -1518,7 +1534,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; - var resumedFromFallback = false; + StreamSequenceToken? cursorStartToken = newToken.Token; if (newToken is StartPositionToken startPositionToken) { consumerData.LastProcessedToken = null; @@ -1548,7 +1564,8 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.StreamId, sequenceToken, batch.SequenceToken, - out pendingBatch); + out pendingBatch, + out cursorStartToken); } else { @@ -1568,6 +1585,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati sequenceToken, batch.SequenceToken, out pendingBatch, + out cursorStartToken, previousSafePartitionToken); } else @@ -1579,7 +1597,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.SafeDisposeCursor(logger); consumerData.Cursor = newCursor; consumerData.PendingBatch = pendingBatch; - consumerData.CursorStartToken = resumedFromFallback ? batch.SequenceToken : newToken.Token; + consumerData.CursorStartToken = cursorStartToken; if (newToken is DeliveryToken deliveryToken) { consumerData.LastProcessedToken = deliveryToken.Token; diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index b3950f63988..4cbf5022adb 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -303,7 +303,6 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public static EventHubAdapterFactory Create(System.IServiceProvider services, string name) { throw null; } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task CreateAdapter() { throw null; } protected virtual IEventHubQueueCacheFactory CreateCacheFactory(Configuration.EventHubStreamCachePressureOptions eventHubCacheOptions) { throw null; } @@ -314,7 +313,6 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public System.Threading.Tasks.Task GetDeliveryFailureHandler(Streams.QueueId queueId) { throw null; } - [System.Diagnostics.DebuggerStepThrough] protected virtual System.Threading.Tasks.Task GetPartitionIdsAsync() { throw null; } public Streams.IQueueAdapterCache GetQueueAdapterCache() { throw null; } @@ -370,10 +368,8 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } - [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } - [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -381,7 +377,6 @@ internal EventHubCheckpointer() { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } - [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index 1dd9cd14d37..32540290221 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -961,14 +961,17 @@ public void AddToCache(System.Collections.Generic.IList TryGetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public Orleans.Streams.QueueCacheCursorResult TryGetCacheCursorAtPosition(Runtime.StreamId streamId, Orleans.Streams.StreamSubscriptionStartPosition startPosition) { throw null; } + public bool TryGetNewestPosition(out Orleans.Streams.StreamSequenceToken? token, out string? offset) { throw null; } public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } @@ -984,10 +987,9 @@ public RecoverableStreamReceiver(IRecoverableStreamSource source, public void AddToCache(System.Collections.Generic.IList messages) { } + [System.Obsolete("Use TryGetCacheCursor instead.")] public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } - public Orleans.Streams.IQueueCacheCursor GetCacheCursorAtPosition(Runtime.StreamId streamId, Orleans.Streams.StreamSubscriptionStartPosition startPosition) { throw null; } - public int GetMaxAddCount() { throw null; } public System.Threading.Tasks.Task> GetQueueMessagesAsync(int maxCount, System.Threading.CancellationToken cancellationToken) { throw null; } @@ -1008,12 +1010,16 @@ public void AddToCache(System.Collections.Generic.IList TryGetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public Orleans.Streams.QueueCacheCursorResult TryGetCacheCursorAtPosition(Runtime.StreamId streamId, Orleans.Streams.StreamSubscriptionStartPosition startPosition) { throw null; } + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } } - public readonly partial struct RecoverableStreamStartPosition + public readonly partial struct RecoverableStreamStartPosition : System.IEquatable { private readonly object _dummy; private readonly int _dummyPrimitive; @@ -1022,6 +1028,16 @@ public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) { } public string? Checkpoint { get { throw null; } } public bool StartFromNow { get { throw null; } } + + public readonly bool Equals(RecoverableStreamStartPosition other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public static bool operator ==(RecoverableStreamStartPosition left, RecoverableStreamStartPosition right) { throw null; } + + public static bool operator !=(RecoverableStreamStartPosition left, RecoverableStreamStartPosition right) { throw null; } } public static partial class SegmentBuilder diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs index 56cd5cf4e34..fb4e2648c0b 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RecoveryCacheMemoryOptionsTests.cs @@ -145,8 +145,8 @@ private static AdoNetStreamOptions Options() private static void AssertEvent(IQueueCache receiver, IBatchContainer notification, string expected) { - using var cursor = receiver.GetCacheCursor(notification.StreamId, notification.SequenceToken); - Assert.True(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(receiver.TryGetCacheCursor(notification.StreamId, notification.SequenceToken).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); var batch = cursor.GetCurrent(out var exception); Assert.Null(exception); var item = Assert.Single(Assert.IsType(batch).GetEvents()); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs index 346fb6b099f..5d6a0fd9410 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs @@ -125,10 +125,10 @@ await Task.WhenAll(Enumerable.Range(0, NumBatches) var messages = (await receiver.GetQueueMessagesAsync(10, cancellationToken)).ToArray(); foreach (var notification in messages) { - using var cursor = caches[queueId].GetCacheCursor( + using var cursor = Assert.IsAssignableFrom(caches[queueId].TryGetCacheCursor( notification.StreamId, - notification.SequenceToken); - Assert.True(cursor.MoveNext()); + notification.SequenceToken).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); var message = Assert.IsType(cursor.GetCurrent(out var exception)); Assert.Null(exception); output.WriteLine($"Queue {queueId} received message on stream {message.StreamId}"); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs index ce7b8456fd9..1e1f2c6dbcf 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -40,9 +40,10 @@ public async Task PooledFactory_InitialIteratorUsesDurableCheckpointOrTrimHorizo var notifications = await receiver.GetQueueMessagesAsync(1, TestCancellation); Assert.Equal(fixture.StreamId, Assert.Single(notifications).StreamId); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); Assert.Equal(["next"], ReadBatch(cursor).GetEvents().Select(item => item.Item1)); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); await client.Received(1).GetShardIteratorAsync( Arg.Is(request => request.StreamName == "stream" @@ -75,10 +76,11 @@ public async Task PooledReceiver_ExpiredIteratorResumesAfterLastAdmittedOffset() Assert.Equal("9", fixture.Store.Checkpoint); await receiver.GetQueueMessagesAsync(1, TestCancellation); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(cursor), "10", 0, "first"); AssertBatch(ReadBatch(cursor), "11", 1, "second"); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Equal(["initial", "expired", "renewed"], fixture.ReadRequests.Select(request => request.Iterator)); Assert.Equal(1, fixture.Store.LoadCount); Assert.Empty(fixture.Store.Writes); @@ -119,7 +121,8 @@ public async Task PooledReceiver_CacheAdmissionFailureReplaysWholeBatchWithoutAd Assert.Empty(fixture.Store.Writes); await receiver.GetQueueMessagesAsync(2, TestCancellation); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); if (admitPrefix) { AssertBatch(ReadBatch(cursor), "9", 0, "prefix"); @@ -127,7 +130,7 @@ public async Task PooledReceiver_CacheAdmissionFailureReplaysWholeBatchWithoutAd AssertBatch(ReadBatch(cursor), "10", admitPrefix ? 1 : 0, "first"); AssertBatch(ReadBatch(cursor), "11", admitPrefix ? 2 : 1, "second"); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); await client.Received(admitPrefix ? 1 : 2).GetShardIteratorAsync( Arg.Is(request => request.ShardIteratorType == (admitPrefix ? ShardIteratorType.AFTER_SEQUENCE_NUMBER : ShardIteratorType.TRIM_HORIZON) @@ -168,10 +171,11 @@ public async Task PooledReceiver_RepeatedThrottlingAndTransientFailuresPreserveR Assert.Equal(["iterator", "after-first", "after-first", "after-first", "after-first"], fixture.ReadRequests.Select(request => request.Iterator)); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(cursor), "10", 0, "first"); AssertBatch(ReadBatch(cursor), "11", 1, "second"); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Empty(fixture.Store.Writes); await client.Received(1).GetShardIteratorAsync(Arg.Any(), Arg.Any()); } @@ -203,7 +207,8 @@ await client.Received(2).GetShardIteratorAsync( request.ShardIteratorType == ShardIteratorType.AFTER_SEQUENCE_NUMBER && request.StartingSequenceNumber == "9"), Arg.Any()); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(cursor), "10", 0, "recovered"); } @@ -241,7 +246,8 @@ public async Task PooledReceiver_OwningCallerCancellationAllowsWaitingCallerToRe Assert.Equal(2, calls); Assert.Single(fixture.ReadRequests); Assert.Equal(1, fixture.Store.LoadCount); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(cursor), "10", 0, "recovered"); } @@ -324,10 +330,11 @@ public async Task PooledFactory_ByteBudgetStagesUnreadSuffixUntilDeliveredPrefix Assert.Equal("10", Assert.IsType(firstNotification.SequenceToken).ShardSequence); Assert.True(receiver.IsUnderPressure()); Assert.Equal(0, receiver.GetMaxAddCount()); - using (var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable)) + using (var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor)) { AssertBatch(ReadBatch(cursor), "10", 0, "first"); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); var progress = Assert.IsAssignableFrom(cursor); progress.RecordDeliverySuccess(); receiver.UpdateDeliveryProgress( @@ -341,9 +348,10 @@ public async Task PooledFactory_ByteBudgetStagesUnreadSuffixUntilDeliveredPrefix Assert.Equal("11", Assert.IsType(secondNotification.SequenceToken).ShardSequence); Assert.Single(fixture.ReadRequests); - using var nextCursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var nextCursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(nextCursor), "11", 1, "second"); - Assert.False(nextCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, nextCursor.MoveNextWithResult().Kind); Assert.Equal("10", fixture.Store.Checkpoint); } @@ -475,7 +483,8 @@ public async Task PooledReceiver_MixedTypeNonzeroIndicesPersistSafeWatermarkAndR () => Response("mixed-record", fixture.Record(previousOffset, "prefix")), () => Response("tail", record)); await receiver.GetQueueMessagesAsync(1, TestCancellation); - using (var prefixCursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable)) + using (var prefixCursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor)) { AssertBatch(ReadBatch(prefixCursor), previousOffset, 0, "prefix"); var prefixProgress = Assert.IsAssignableFrom(prefixCursor); @@ -487,7 +496,8 @@ public async Task PooledReceiver_MixedTypeNonzeroIndicesPersistSafeWatermarkAndR Assert.Equal([previousOffset], fixture.Store.Writes); var notification = Assert.Single(await receiver.GetQueueMessagesAsync(1, TestCancellation)); - using var cursor = receiver.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); var batch = ReadBatch(cursor); var progress = Assert.IsAssignableFrom(cursor); @@ -543,9 +553,10 @@ public async Task PooledReceiver_MixedTypeNonzeroIndicesPersistSafeWatermarkAndR var replacementClient = fixture.ReceiverClients.Last(); fixture.SetReads(replacementClient, () => Response("tail", fixture.Record(nextOffset, "next-record"))); await replacement.GetQueueMessagesAsync(1, TestCancellation); - using var restartedCursor = replacement.GetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable); + using var restartedCursor = Assert.IsAssignableFrom( + replacement.TryGetCacheCursorAtPosition(fixture.StreamId, StreamSubscriptionStartPosition.EarliestAvailable).Cursor); AssertBatch(ReadBatch(restartedCursor), nextOffset, 0, "next-record"); - Assert.False(restartedCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, restartedCursor.MoveNextWithResult().Kind); await replacementClient.Received(1).GetShardIteratorAsync( Arg.Is(request => request.ShardIteratorType == ShardIteratorType.AFTER_SEQUENCE_NUMBER @@ -619,7 +630,7 @@ record => Assert.Same(second, record.Record)); private static KinesisBatchContainer ReadBatch(IQueueCacheCursor cursor) { - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); var batch = Assert.IsType(cursor.GetCurrent(out var exception)); Assert.Null(exception); return batch; diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs index 4915bf77822..80e74d43904 100644 --- a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledQueueCacheTests.cs @@ -371,7 +371,10 @@ public void EarliestAvailableCursorCreatedAfterDrainingBlock_WaitsForFirstMatchi Assert.Equal(0, cache.ItemCount); Assert.Null(cache.Oldest); Assert.Null(cache.Newest); - var cursor = cache.GetCursorAtPosition(targetStream, StreamSubscriptionStartPosition.EarliestAvailable); + var result = cache.TryGetCursorAtPosition(targetStream, StreamSubscriptionStartPosition.EarliestAvailable); + Assert.Equal(QueueCacheCursorResultKind.Success, result.Kind); + Assert.NotNull(result.Cursor); + var cursor = result.Cursor; Assert.False(cache.TryGetNextMessage(cursor, out _)); cache.Add( diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index a3122fd3bc7..47ad84ce6b6 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -483,12 +483,13 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? return new ScriptedQueueCursor(messages, streamId, token); } - public IQueueCacheCursor GetCacheCursorAtPosition( + public QueueCacheCursorResult TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) { Assert.Equal(StreamSubscriptionStartPosition.EarliestAvailable, startPosition); - return new ScriptedQueueCursor(messages, streamId, token: null); + return QueueCacheCursorResult.FromCursor( + new ScriptedQueueCursor(messages, streamId, token: null)); } public bool IsUnderPressure() => false; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index ef5f162e167..939212767b2 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -47,14 +47,14 @@ public async Task Receiver_ResumesAfterCheckpointAndPersistsDeliveryProgress() Assert.Equal(streamId, notification.StreamId); Assert.Equal(11, notification.SequenceToken.SequenceNumber); - using var cursor = receiver.GetCacheCursor(streamId, notification.SequenceToken); - Assert.True(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(receiver.TryGetCacheCursor(streamId, notification.SequenceToken).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); var batch = Assert.IsType(cursor.GetCurrent(out var exception)); Assert.Null(exception); Assert.Equal("payload", batch.Payload); Assert.True(adapter.CompareCallCount > 0); - Assert.False(cursor.MoveNext()); - Assert.Same(batch, cursor.GetCurrent(out exception)); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); + Assert.Null(cursor.GetCurrent(out exception)); Assert.Null(exception); receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(11), DateTime.UtcNow); @@ -165,9 +165,9 @@ public async Task Receiver_RestartRedeliversInclusiveBatchWhenCheckpointDidNotAd })); await firstReceiver.Initialize(TimeSpan.FromSeconds(5)); var firstNotification = Assert.Single(await firstReceiver.GetQueueMessagesAsync(10, CancellationToken.None)); - using (var cursor = firstReceiver.GetCacheCursor(streamId, firstNotification.SequenceToken)) + using (var cursor = Assert.IsAssignableFrom(firstReceiver.TryGetCacheCursor(streamId, firstNotification.SequenceToken).Cursor)) { - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); // The inclusive first batch was observed but never confirmed as delivered. } @@ -215,11 +215,11 @@ public async Task Receiver_QuietScanCheckpointRestartsAfterBusyTail() })); await receiver.Initialize(TimeSpan.FromSeconds(5)); var notifications = await receiver.GetQueueMessagesAsync(10, CancellationToken.None); - using var cursor = receiver.GetCacheCursor(quietStream, notifications[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(receiver.TryGetCacheCursor(quietStream, notifications[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); progress.RecordDeliverySuccess(); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); receiver.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UtcNow); @@ -315,12 +315,12 @@ public void Cache_CursorCreatedWhileEmptyReadsFirstLaterRecord() cache.UpdateDeliveryProgress(initial[0].SequenceToken, DateTime.UtcNow); Assert.Equal(0, cache.ItemCount); - using var cursor = cache.GetCacheCursor(streamId, token: null); - Assert.False(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamId, token: null).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); _ = cache.Add([new TestQueueMessage(streamId, 11, "payload")], DateTime.UnixEpoch); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal("payload", Assert.IsType(cursor.GetCurrent(out _)).Payload); } @@ -343,11 +343,11 @@ public void Cache_EarliestAvailableReturnsOldestMatchingRecord() ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursorAtPosition( + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursorAtPosition( streamId, - StreamSubscriptionStartPosition.EarliestAvailable); + StreamSubscriptionStartPosition.EarliestAvailable).Cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal("first", Assert.IsType(cursor.GetCurrent(out _)).Payload); Assert.Equal(1, Assert.IsAssignableFrom(cursor).SafeSequenceToken?.SequenceNumber); } @@ -370,11 +370,11 @@ public void Cache_EarliestAvailableQuietStreamScansUnrelatedRecords() ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursorAtPosition( + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursorAtPosition( quietStreamId, - StreamSubscriptionStartPosition.EarliestAvailable); + StreamSubscriptionStartPosition.EarliestAvailable).Cursor); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Equal(2, Assert.IsAssignableFrom(cursor).SafeSequenceToken?.SequenceNumber); } @@ -534,14 +534,14 @@ public void Cache_AddsRawRecordsInOrderAndDecodesLazily() Assert.Equal(3, adapter.FromQueueMessageCallCount); Assert.Equal(0, adapter.GetBatchContainerCallCount); - using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); - Assert.True(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamA, positions[0].SequenceToken).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal("first", Assert.IsType(cursor.GetCurrent(out _)).Payload); Assert.Equal(1, adapter.GetBatchContainerCallCount); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal("third", Assert.IsType(cursor.GetCurrent(out _)).Payload); Assert.Equal(2, adapter.GetBatchContainerCallCount); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Equal(2, adapter.GetBatchContainerCallCount); } @@ -564,16 +564,16 @@ public void Cache_CursorProgressRequiresDeliveryAndIncludesUnrelatedScans() new TestQueueMessage(streamA, 4, "a-4"), ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamA, positions[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Null(progress.SafeSequenceToken); progress.RecordDeliverySuccess(); Assert.Equal(1, progress.SafeSequenceToken?.SequenceNumber); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); progress.RecordDeliverySuccess(); @@ -598,11 +598,11 @@ public void Cache_BatchedMatchesRemainPendingUntilWholeDeliverySucceeds() new TestQueueMessage(streamA, 3, "a-3"), ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamA, positions[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Null(progress.SafeSequenceToken); progress.RecordDeliverySuccess(); @@ -624,10 +624,12 @@ public void Cache_FutureStartAcrossCapacity_PreservesInclusiveTargetDuringPurge( NullLogger.Instance, maxCacheSize: 3); var target = new EventSequenceTokenV2(7); - var cursor = createCursorBeforeFirstRead ? cache.GetCacheCursor(stream, target) : null; + var cursor = createCursorBeforeFirstRead + ? Assert.IsAssignableFrom(cache.TryGetCacheCursor(stream, target).Cursor) + : null; if (cursor is not null) { - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Null(Assert.IsAssignableFrom(cursor).SafeSequenceToken); } @@ -639,10 +641,10 @@ public void Cache_FutureStartAcrossCapacity_PreservesInclusiveTargetDuringPurge( sequence % 2 == 0 ? otherStream : stream, sequence, $"payload-{sequence}")) .ToList(), DateTime.UnixEpoch); - cursor ??= cache.GetCacheCursor(stream, target); + cursor ??= Assert.IsAssignableFrom(cache.TryGetCacheCursor(stream, target).Cursor); Assert.True(cache.IsUnderPressure()); Assert.Equal(0, cache.GetMaxAddCount()); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); var safeToken = Assert.IsAssignableFrom(cursor).SafeSequenceToken; Assert.NotNull(safeToken); Assert.Equal(first + 2, safeToken.SequenceNumber); @@ -661,9 +663,9 @@ public void Cache_FutureStartAcrossCapacity_PreservesInclusiveTargetDuringPurge( new TestQueueMessage(otherStream, 8, "other"), new TestQueueMessage(stream, 9, "next"), ], DateTime.UnixEpoch); - Assert.True(targetCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, targetCursor.MoveNextWithResult().Kind); Assert.Equal("target", Assert.IsType(targetCursor.GetCurrent(out _)).Payload); - Assert.True(targetCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, targetCursor.MoveNextWithResult().Kind); Assert.Equal("next", Assert.IsType(targetCursor.GetCurrent(out _)).Payload); Assert.Equal(6, progress.SafeSequenceToken?.SequenceNumber); cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); @@ -673,18 +675,18 @@ public void Cache_FutureStartAcrossCapacity_PreservesInclusiveTargetDuringPurge( targetCursor.RecordDeliveryFailure(); Assert.Equal(6, progress.SafeSequenceToken?.SequenceNumber); - Assert.True(targetCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, targetCursor.MoveNextWithResult().Kind); Assert.Equal(target, targetCursor.GetCurrent(out _)!.SequenceToken); progress.RecordDeliverySuccess(); cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); Assert.Equal("7", cache.LastPurgedOffset); Assert.Equal(2, cache.ItemCount); - Assert.True(targetCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, targetCursor.MoveNextWithResult().Kind); Assert.Equal(9, targetCursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); Assert.Equal(8, progress.SafeSequenceToken?.SequenceNumber); progress.RecordDeliverySuccess(); - Assert.False(targetCursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, targetCursor.MoveNextWithResult().Kind); cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); Assert.Equal("9", cache.LastPurgedOffset); Assert.Equal(0, cache.ItemCount); @@ -703,9 +705,9 @@ public void Cache_FutureStartScanWaitsForPendingDeliveryAcknowledgement() NullLogger.Instance, maxCacheSize: 3); cache.Add([new TestQueueMessage(stream, 1, "pending")], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(stream, new EventSequenceTokenV2(1)); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(stream, new EventSequenceTokenV2(1)).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); cache.Add( [ new TestQueueMessage(stream, 2, "before-start-2"), @@ -714,7 +716,7 @@ public void Cache_FutureStartScanWaitsForPendingDeliveryAcknowledgement() cursor.Refresh(new EventSequenceTokenV2(7)); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Null(progress.SafeSequenceToken); cache.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UnixEpoch); Assert.Equal(3, cache.ItemCount); @@ -725,7 +727,7 @@ public void Cache_FutureStartScanWaitsForPendingDeliveryAcknowledgement() Assert.Equal(0, cache.ItemCount); cache.Add([new TestQueueMessage(stream, 7, "target")], DateTime.UnixEpoch); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(7, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); } @@ -749,11 +751,11 @@ public void Cache_DeliveredThroughScansIntermediatePartitionRecordsWithoutRedeli new TestQueueMessage(streamA, 11, "a-11"), ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamA, positions[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); progress.SetDeliveredThrough(new EventSequenceTokenV2(10)); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); Assert.Equal(10, progress.SafeSequenceToken?.SequenceNumber); } @@ -776,20 +778,20 @@ public void Cache_DeliveryFailureRewindsToFirstPendingRecord() new TestQueueMessage(streamA, 3, "a-3"), ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(streamA, positions[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); cursor.RecordDeliveryFailure(); Assert.Null(progress.SafeSequenceToken); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); progress.RecordDeliverySuccess(); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); } @@ -853,12 +855,12 @@ public void Cache_QuietCursorAdvancesAcrossUnrelatedRecords() new TestQueueMessage(busyStream, 4, "busy-4"), ], DateTime.UnixEpoch); - using var cursor = cache.GetCacheCursor(quietStream, positions[0].SequenceToken); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(quietStream, positions[0].SequenceToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); - Assert.True(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); progress.RecordDeliverySuccess(); - Assert.False(cursor.MoveNext()); + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); Assert.Equal(4, progress.SafeSequenceToken?.SequenceNumber); } @@ -1328,9 +1330,18 @@ public bool TryPurgeFromCache(out IList purgedItems) return false; } + [Obsolete("Use TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => throw new NotSupportedException(); + public QueueCacheCursorResult TryGetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => QueueCacheCursorResult.NotSupported; + + public QueueCacheCursorResult TryGetCacheCursorAtPosition( + StreamId streamId, + StreamSubscriptionStartPosition startPosition) + => QueueCacheCursorResult.NotSupported; + public bool IsUnderPressure() => false; } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs index c111db00348..f0839bf5683 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoveryCacheMemoryTests.cs @@ -257,8 +257,8 @@ public void Cache_RepeatedFillDrainReusesBuffersAndPreservesDeliveredCopies() var second = Record(cycle * 2 + 2, 24); _ = cache.Add([first, second], DateTime.UnixEpoch); Assert.Equal(40, cache.SizeInBytes); - using var cursor = cache.GetCacheCursor(first.StreamId, new EventSequenceTokenV2(first.Sequence)); - Assert.True(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(first.StreamId, new EventSequenceTokenV2(first.Sequence)).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); snapshots.Add((first, Assert.IsType(cursor.GetCurrent(out _)).Payload)); cache.UpdateDeliveryProgress(new EventSequenceTokenV2(second.Sequence), DateTime.UnixEpoch); @@ -466,8 +466,8 @@ private static IEnumerable Sequences(IList batches) private static void AssertPayload(RecoverableStreamQueueCache cache, TestRecord record) { - using var cursor = cache.GetCacheCursor(record.StreamId, new EventSequenceTokenV2(record.Sequence)); - Assert.True(cursor.MoveNext()); + using var cursor = Assert.IsAssignableFrom(cache.TryGetCacheCursor(record.StreamId, new EventSequenceTokenV2(record.Sequence)).Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); var batch = Assert.IsType(cursor.GetCurrent(out var exception)); Assert.Null(exception); Assert.Equal(record.StreamId, batch.StreamId); From 28decf0d40db5e3f62b1323f212c0b6554607793 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 6 Sep 2026 10:23:42 -0700 Subject: [PATCH 65/90] test(transactions): cover cancellation before and during table initialization --- ...nAzureTableTransactionStateStorageTests.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/Transactions/Orleans.Transactions.Tests/FaultInjectionAzureTableTransactionStateStorageTests.cs b/test/Transactions/Orleans.Transactions.Tests/FaultInjectionAzureTableTransactionStateStorageTests.cs index 4123ceb04b8..1e926f9f85f 100644 --- a/test/Transactions/Orleans.Transactions.Tests/FaultInjectionAzureTableTransactionStateStorageTests.cs +++ b/test/Transactions/Orleans.Transactions.Tests/FaultInjectionAzureTableTransactionStateStorageTests.cs @@ -145,10 +145,13 @@ public void FactoryCreate_RegisteredNamedOptions_ReturnsWrapperFactoryWithoutSto Assert.Empty(table.SubmittedTransactions); } - [Fact] - public async Task Participate_CanceledStart_ForwardsCancellationTokenWithoutContactingStorage() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Participate_CancellationStopsTableInitialization(bool cancelBeforeStart) { - var table = new RecordingTableClient([]); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var table = new RecordingTableClient([]) { OnCreate = cancellation.Cancel }; var tableService = new RecordingTableServiceClient(table); using var services = CreateServices(options => options.TableServiceClient = tableService); var inner = new AzureTableTransactionalStateStorageFactory( @@ -160,17 +163,20 @@ public async Task Participate_CanceledStart_ForwardsCancellationTokenWithoutCont var factory = new FaultInjectionAzureTableTransactionStateStorageFactory(inner); var lifecycle = new RecordingSiloLifecycle(); factory.Participate(lifecycle); - using var cancellation = new CancellationTokenSource(); - cancellation.Cancel(); + if (cancelBeforeStart) + { + cancellation.Cancel(); + } var exception = await Assert.ThrowsAnyAsync( () => lifecycle.StartAsync(cancellation.Token)); Assert.Equal(cancellation.Token, exception.CancellationToken); Assert.Equal(AzureTableTransactionalStateOptions.DEFAULT_INIT_STAGE, lifecycle.Stage); - Assert.Equal(0, tableService.GetTableClientCallCount); - Assert.Equal(0, table.CreateIfNotExistsCallCount); - Assert.Equal(CancellationToken.None, table.CreateCancellationToken); + var expectedCreateCalls = cancelBeforeStart ? 0 : 1; + Assert.Equal(expectedCreateCalls, tableService.GetTableClientCallCount); + Assert.Equal(expectedCreateCalls, table.CreateIfNotExistsCallCount); + Assert.Equal(cancelBeforeStart ? default : cancellation.Token, table.CreateCancellationToken); Assert.Equal(0, table.QueryCallCount); Assert.Empty(table.SubmittedTransactions); } @@ -297,6 +303,8 @@ private sealed class RecordingTableClient(List events) : TableClient public CancellationToken CreateCancellationToken { get; private set; } + public Action? OnCreate { get; init; } + public List> SubmittedTransactions { get; } = []; public override AsyncPageable QueryAsync( @@ -341,6 +349,7 @@ public override Task> CreateIfNotExistsAsync( { CreateIfNotExistsCallCount++; CreateCancellationToken = cancellationToken; + OnCreate?.Invoke(); cancellationToken.ThrowIfCancellationRequested(); return Task.FromResult( Response.FromValue( From c1a7ec4a3293ee1930382a2d0eb8fe082a1d62e9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 8 Sep 2026 01:37:47 -0700 Subject: [PATCH 66/90] fix(streaming): detach unavailable consumers locally --- .../PersistentStreams/PersistentStreamPullingAgent.cs | 1 + .../PersistentStreamPullingAgentDiagnosticsTests.cs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index c107a370f6e..125374e340a 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -1873,6 +1873,7 @@ private async Task ErrorProtocol( if (exceptionOccured is ClientNotAvailableException) { LogWarningConsumerIsDead(consumerData.StreamConsumer, consumerData.StreamId); + RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); UnregisterUnavailableConsumer(consumerData, cancellationToken).Ignore(); return true; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index a708a88ede4..0ace0391dfe 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -73,6 +73,9 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking Assert.Same(consumer, requested.Consumer); Assert.Null(requested.Exception); Assert.False(outcome.Task.IsCompleted); + Assert.False(streamData.Contains(subscriptionId)); + Assert.Null(data.Cursor); + Assert.DoesNotContain(streamId, await accessor.GetPubSubCache()); var exception = new InvalidOperationException("unregistration storage failure"); if (failUnregistration) From 263ec5e872f5703a7543eddc210d21c88a142fcb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 8 Sep 2026 03:15:43 -0700 Subject: [PATCH 67/90] fix(streaming): close recovery edge cases --- .../Streams/KinesisBatchContainer.cs | 50 +++++++- .../Streams/KinesisPooledAdapterReceiver.cs | 40 +++++-- .../Streams/KinesisRecoverableStream.cs | 3 +- .../AdoNetQueueAdapterFactory.cs | 27 +++-- .../AdoNetQueueAdapterReceiver.cs | 12 +- .../GrainDirectory/ClientDirectory.cs | 45 +++++-- .../Common/PooledCache/PooledQueueCache.cs | 41 ++++++- .../RecoverableStreamReceiver.cs | 39 ++++-- .../Orleans.Streaming.csproj | 1 + .../PersistentStreamPullingAgent.cs | 41 ++++++- .../IQueueCacheBatchContainerFilter.cs | 8 ++ .../AdoNetQueueAdapterFactoryTests.cs | 4 +- .../Streaming/AdoNetRecoverableStreamTests.cs | 30 ++++- .../KinesisPooledRuntimeTests.cs | 112 ++++++++++++++++++ .../Directory/ClientDirectoryTests.cs | 42 +++++++ ...stentStreamPullingAgentDiagnosticsTests.cs | 38 ++++-- .../RecoverableStreamReceiverTests.cs | 19 +++ 17 files changed, 492 insertions(+), 60 deletions(-) create mode 100644 src/Orleans.Streaming/QueueAdapters/IQueueCacheBatchContainerFilter.cs diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 0a658e0683e..85d282ba098 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -11,7 +11,7 @@ namespace Orleans.Streaming.Kinesis { [Serializable] [Orleans.GenerateSerializer] - internal class KinesisBatchContainer : IBatchContainer, IComparable + internal class KinesisBatchContainer : IBatchContainer, IComparable, IQueueCacheBatchContainerFilter { [JsonProperty] [Id(0)] @@ -48,13 +48,14 @@ private KinesisBatchContainer( Serializer serializer, StreamId streamId, string shardSequence, - long sequenceId) + long sequenceId, + int eventIndex = 0) { Serializer = serializer; _rawRecord = rawRecord; _streamId = streamId; _hasStreamId = true; - Token = new KinesisSequenceToken(shardSequence, sequenceId, 0); + Token = new KinesisSequenceToken(shardSequence, sequenceId, eventIndex); } [GeneratedActivatorConstructor] @@ -83,7 +84,7 @@ internal KinesisBatchContainer(Serializer serializer public IEnumerable> GetEvents() { var events = GetPayload().Events; - for (var i = 0; i < events.Count; i++) + for (var i = Token.EventIndex; i < events.Count; i++) { if (events[i] is T item) { @@ -112,6 +113,47 @@ public bool ImportRequestContext() public int CompareTo(KinesisBatchContainer? other) => other is null ? 1 : Token.CompareTo(other.Token); + IBatchContainer IQueueCacheBatchContainerFilter.FilterFrom(StreamSequenceToken inclusiveStartToken) + { + if (inclusiveStartToken is not KinesisSequenceToken token + || !string.Equals(token.ShardSequence, Token.ShardSequence, StringComparison.Ordinal) + || token.EventIndex <= 0) + { + return this; + } + + return new KinesisBatchContainer( + _rawRecord, + Serializer, + StreamId, + Token.ShardSequence, + Token.SequenceNumber, + token.EventIndex); + } + + IBatchContainer? IQueueCacheBatchContainerFilter.FilterAfter(StreamSequenceToken exclusiveStartToken) + { + if (exclusiveStartToken is not KinesisSequenceToken token + || !string.Equals(token.ShardSequence, Token.ShardSequence, StringComparison.Ordinal)) + { + return this; + } + + var firstEventIndex = token.EventIndex + 1; + if (firstEventIndex >= GetPayload().Events.Count) + { + return null; + } + + return new KinesisBatchContainer( + _rawRecord, + Serializer, + StreamId, + Token.ShardSequence, + Token.SequenceNumber, + firstEventIndex); + } + [Serializable] [GenerateSerializer] internal class Body diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index 0dc58fff361..aeb2d3f163a 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -71,10 +71,24 @@ public KinesisPooledAdapterReceiver( maxCacheSize: cacheOptions.CacheSize); } - public async Task Initialize(TimeSpan timeout) + public Task Initialize(TimeSpan timeout) + => InitializeWithCancellation(timeout, CancellationToken.None); + + Task IQueueAdapterReceiver.Initialize(TimeSpan timeout, CancellationToken cancellationToken) + => InitializeWithCancellation(timeout, cancellationToken); + + private async Task InitializeWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) { - using var cancellation = new CancellationTokenSource(timeout); - await EnsureInitialized(cancellation.Token); + cancellationToken.ThrowIfCancellationRequested(); + using var timeoutCancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + using var linkedCancellation = timeoutCancellation is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCancellation.Token); + await EnsureInitialized(linkedCancellation?.Token ?? cancellationToken); } public async Task> GetQueueMessagesAsync(int maxCount) @@ -103,8 +117,15 @@ public Task MessagesDeliveredAsync(IList messages) public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; - public async Task Shutdown(TimeSpan timeout) + public Task Shutdown(TimeSpan timeout) + => ShutdownWithCancellation(timeout, CancellationToken.None); + + Task IQueueAdapterReceiver.Shutdown(TimeSpan timeout, CancellationToken cancellationToken) + => ShutdownWithCancellation(timeout, cancellationToken); + + private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); if (Interlocked.Exchange(ref _shutdown, 1) != 0) { return; @@ -114,10 +135,15 @@ public async Task Shutdown(TimeSpan timeout) { _lifecycleCancellation.Cancel(); var shutdownWatch = Stopwatch.StartNew(); - using var cancellation = timeout == Timeout.InfiniteTimeSpan + using var timeoutCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - var cancellationToken = cancellation?.Token ?? CancellationToken.None; + using var linkedCancellation = timeoutCancellation is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCancellation.Token); + cancellationToken = linkedCancellation?.Token ?? cancellationToken; List? exceptions = null; Task? initializationTask; lock (_lifecycleLock) @@ -155,7 +181,7 @@ public async Task Shutdown(TimeSpan timeout) : timeout > shutdownWatch.Elapsed ? timeout - shutdownWatch.Elapsed : TimeSpan.Zero; - await _inner.Shutdown(remaining); + await ((IQueueAdapterReceiver)_inner).Shutdown(remaining, cancellationToken); } } catch (Exception exception) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs index c11ff68d18c..b7610d0dc2f 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -232,8 +232,7 @@ public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) var offset = 0; var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; - var difference = CompareShardSequences(shardSequence, kinesisToken.ShardSequence); - return difference != 0 ? difference : cachedMessage.EventIndex.CompareTo(kinesisToken.EventIndex); + return CompareShardSequences(shardSequence, kinesisToken.ShardSequence); } private static int CompareShardSequences(string left, string right) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index d115402c49a..5f915a8a9b3 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -40,7 +40,8 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, /// /// Ensures queries are loaded only once while allowing for recovery if the load fails. /// - internal virtual ValueTask GetQueriesAsync() + internal virtual ValueTask GetQueriesAsync( + CancellationToken cancellationToken = default) { // attempt fast path return Volatile.Read(ref _queries) is { } queries ? new(queries) : new(CoreAsync()); @@ -48,7 +49,7 @@ internal virtual ValueTask GetQueriesAsync() // slow path async Task CoreAsync() { - await WaitForInitializationLockAsync(); + await WaitForInitializationLockAsync(cancellationToken); try { // attempt fast path again @@ -60,7 +61,7 @@ async Task CoreAsync() // slow path - the member variable will only be set if the call succeeds var result = await RelationalOrleansQueries .CreateInstance(_streamOptions.Invariant, _streamOptions.ConnectionString, _streamOptions.DataSource) - .WaitAsync(_streamOptions.InitializationTimeout); + .WaitAsync(_streamOptions.InitializationTimeout, cancellationToken); Volatile.Write(ref _queries, result); return result; } @@ -71,16 +72,23 @@ async Task CoreAsync() } } - public async Task CreateAdapter() + public Task CreateAdapter() + => CreateAdapterWithCancellation(CancellationToken.None); + + Task IQueueAdapterFactory.CreateAdapter(CancellationToken cancellationToken) + => CreateAdapterWithCancellation(cancellationToken); + + private async Task CreateAdapterWithCancellation(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); if (Volatile.Read(ref _adapter) is { } adapter) { return adapter; } - var queries = await GetQueriesAsync(); + var queries = await GetQueriesAsync(cancellationToken); - await WaitForInitializationLockAsync(); + await WaitForInitializationLockAsync(cancellationToken); try { if (_adapter is not null) @@ -98,9 +106,12 @@ public async Task CreateAdapter() } } - private async Task WaitForInitializationLockAsync() + private async Task WaitForInitializationLockAsync(CancellationToken cancellationToken = default) { - if (!await _semaphore.WaitAsync(_streamOptions.InitializationTimeout, _lifetime.ApplicationStopping)) + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _lifetime.ApplicationStopping); + if (!await _semaphore.WaitAsync(_streamOptions.InitializationTimeout, linkedCancellation.Token)) { throw new TimeoutException($"Timed out waiting to initialize ADO.NET stream provider '{_name}'."); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 00562c7e214..57d5e22429e 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -63,11 +63,19 @@ public AdoNetQueueAdapterReceiver( public Task Initialize(TimeSpan timeout) => _inner.Initialize(timeout); - public async Task Shutdown(TimeSpan timeout) + Task IQueueAdapterReceiver.Initialize(TimeSpan timeout, CancellationToken cancellationToken) + => ((IQueueAdapterReceiver)_inner).Initialize(timeout, cancellationToken); + + public Task Shutdown(TimeSpan timeout) => ShutdownWithCancellation(timeout, CancellationToken.None); + + Task IQueueAdapterReceiver.Shutdown(TimeSpan timeout, CancellationToken cancellationToken) + => ShutdownWithCancellation(timeout, cancellationToken); + + private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) { try { - await _inner.Shutdown(timeout); + await ((IQueueAdapterReceiver)_inner).Shutdown(timeout, cancellationToken); } finally { diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index 5d54bab7435..80637673d84 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -214,6 +215,8 @@ public void InvalidateCache(GrainId grainId) private async ValueTask?> RefreshInvalidatedRoutes(GrainId grainId) { + var attemptedRefreshes = new Dictionary(); + Exception? refreshFailure = null; while (true) { SiloAddress silo; @@ -232,24 +235,46 @@ public void InvalidateCache(GrainId grainId) } // A cached candidate excluded by TryLocalLookup has a pending owner refresh. - silo = candidates[0].SiloAddress!; + var candidate = candidates.FirstOrDefault(candidate => + _pendingRefreshes.TryGetValue(candidate.SiloAddress!, out var pendingToken) + && (!attemptedRefreshes.TryGetValue(candidate.SiloAddress!, out var attemptedToken) + || !ReferenceEquals(attemptedToken, pendingToken))); + if (candidate is null) + { + if (refreshFailure is not null) + { + ExceptionDispatchInfo.Capture(refreshFailure).Throw(); + } + + return null; + } + + silo = candidate.SiloAddress!; token = _pendingRefreshes[silo]; + attemptedRefreshes[silo] = token; versionVector = _table.ToImmutableDictionary(e => e.Key, e => e.Value.Version); } - var remote = _grainFactory.GetSystemTarget(Constants.ClientDirectoryType, silo); - var delta = await remote.GetClientRoutes(versionVector, _stoppingCts.Token); - lock (_lockObj) + try { - UpdateRoutingTable(delta); - - // Keep the versioned row while refreshing: a same-version response confirms it, - // and a delayed response only completes the invalidation which initiated its read. - if (_pendingRefreshes.TryGetValue(silo, out var currentToken) && ReferenceEquals(currentToken, token)) + var remote = _grainFactory.GetSystemTarget(Constants.ClientDirectoryType, silo); + var delta = await remote.GetClientRoutes(versionVector, _stoppingCts.Token); + lock (_lockObj) { - _pendingRefreshes = _pendingRefreshes.Remove(silo); + UpdateRoutingTable(delta); + + // Keep the versioned row while refreshing: a same-version response confirms it, + // and a delayed response only completes the invalidation which initiated its read. + if (_pendingRefreshes.TryGetValue(silo, out var currentToken) && ReferenceEquals(currentToken, token)) + { + _pendingRefreshes = _pendingRefreshes.Remove(silo); + } } } + catch (Exception exception) when (!_stoppingCts.IsCancellationRequested) + { + refreshFailure = exception; + } } } diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 0fc0f54a6b9..a043a0293c0 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -144,7 +144,10 @@ public QueueCacheCursorResult TryGetCursor(StreamId streamId, StreamSequ return QueueCacheCursorResult.FromCacheMiss(cacheMiss); } - var cursor = new Cursor(streamId); + var cursor = new Cursor(streamId) + { + InclusiveStartToken = sequenceToken, + }; if (SetCursor(cursor, sequenceToken) is { } racedCacheMiss) { return QueueCacheCursorResult.FromCacheMiss(racedCacheMiss); @@ -215,6 +218,7 @@ public void Refresh(object cursorObj, StreamSequenceToken? sequenceToken) } cursor.State = CursorStates.Unset; + cursor.InclusiveStartToken = sequenceToken; if (SetCursor(cursor, sequenceToken) is { } cacheMiss) { throw cacheMiss.ToException(); @@ -585,14 +589,35 @@ public QueueCacheCursorMoveResult TryGetNextMessageWithResult(object cursorObj, if (currentMessage.CompareStreamId(cursor.StreamId)) { if (cursor.DeliveredThroughToken is { } deliveredThrough - && cacheDataAdapter.Compare(ref currentMessage, deliveredThrough) <= 0) + && cacheDataAdapter.Compare(ref currentMessage, deliveredThrough) < 0) { cursor.RecordScanned(currentToken); continue; } - cursor.RecordPending(currentToken); message = cacheDataAdapter.GetBatchContainer(ref currentMessage); + if (cursor.DeliveredThroughToken is { } exclusiveStartToken + && cacheDataAdapter.Compare(ref currentMessage, exclusiveStartToken) == 0) + { + if (message is not IQueueCacheBatchContainerFilter exclusiveFilter + || exclusiveFilter.FilterAfter(exclusiveStartToken) is not { } filtered) + { + cursor.RecordScanned(exclusiveStartToken); + continue; + } + + message = filtered; + } + + if (cursor.InclusiveStartToken is { } inclusiveStartToken) + { + if (message is IQueueCacheBatchContainerFilter filter) + { + message = filter.FilterFrom(inclusiveStartToken); + } + } + + cursor.RecordPending(message.SequenceToken); return QueueCacheCursorMoveResult.Success; } @@ -612,7 +637,11 @@ private QueueCacheMissInfo CreateCacheMissInfo(StreamSequenceToken requestedToke => GetCursor(cursorObj).SafeSequenceToken; internal void SetCursorDeliveredThrough(object cursorObj, StreamSequenceToken token) - => GetCursor(cursorObj).DeliveredThroughToken = token; + { + var cursor = GetCursor(cursorObj); + cursor.DeliveredThroughToken = token; + cursor.InclusiveStartToken = null; + } internal void RecordDeliverySuccess(object cursorObj) => GetCursor(cursorObj).RecordDeliverySuccess(); @@ -628,6 +657,7 @@ internal void RecordDeliveryFailure(object cursorObj) cursor.State = CursorStates.Unset; cursor.CurrentBlock = null; cursor.SequenceToken = retryToken; + cursor.InclusiveStartToken = retryToken; if (SetCursor(cursor, retryToken) is { } cacheMiss) { throw cacheMiss.ToException(); @@ -742,6 +772,7 @@ public Cursor(StreamId streamId) // current sequence token; null while waiting for the first message to arrive public StreamSequenceToken? SequenceToken; + public StreamSequenceToken? InclusiveStartToken; public long BlockGeneration; public StreamSequenceToken? SafeSequenceToken; public StreamSequenceToken? DeliveredThroughToken; @@ -792,6 +823,8 @@ public void RecordDeliverySuccess() pendingSequenceToken = null; pendingStartToken = null; hasPendingDelivery = false; + InclusiveStartToken = null; + DeliveredThroughToken = null; } public StreamSequenceToken? TakePendingStartToken() diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs index 96e656c4c1c..de4928d1394 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -60,13 +60,25 @@ public RecoverableStreamReceiver( } /// - public async Task Initialize(TimeSpan timeout) + public Task Initialize(TimeSpan timeout) + => InitializeWithCancellation(timeout, CancellationToken.None); + + /// + Task IQueueAdapterReceiver.Initialize(TimeSpan timeout, CancellationToken cancellationToken) + => InitializeWithCancellation(timeout, cancellationToken); + + private async Task InitializeWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) { - using var cancellation = timeout == Timeout.InfiniteTimeSpan + cancellationToken.ThrowIfCancellationRequested(); + using var timeoutCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - var cancellationToken = cancellation?.Token ?? CancellationToken.None; - await EnsureInitialized(cancellationToken); + using var linkedCancellation = timeoutCancellation is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCancellation.Token); + await EnsureInitialized(linkedCancellation?.Token ?? cancellationToken); } /// @@ -238,8 +250,16 @@ public Task MessagesDeliveredAsync(IList messages, Cancellation : Task.CompletedTask; /// - public async Task Shutdown(TimeSpan timeout) + public Task Shutdown(TimeSpan timeout) + => ShutdownWithCancellation(timeout, CancellationToken.None); + + /// + Task IQueueAdapterReceiver.Shutdown(TimeSpan timeout, CancellationToken cancellationToken) + => ShutdownWithCancellation(timeout, cancellationToken); + + private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); if (Interlocked.Exchange(ref _shutdown, 1) != 0) { return; @@ -248,10 +268,15 @@ public async Task Shutdown(TimeSpan timeout) Volatile.Write(ref _running, 0); _lifecycleCancellation.Cancel(); ClearPendingMessages(); - using var cancellation = timeout == Timeout.InfiniteTimeSpan + using var timeoutCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - var cancellationToken = cancellation?.Token ?? CancellationToken.None; + using var linkedCancellation = timeoutCancellation is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCancellation.Token); + cancellationToken = linkedCancellation?.Token ?? cancellationToken; List? exceptions = null; Task? initializeTask; lock (_lifecycleLock) diff --git a/src/Orleans.Streaming/Orleans.Streaming.csproj b/src/Orleans.Streaming/Orleans.Streaming.csproj index c2a05f3d348..37bec7c82e8 100644 --- a/src/Orleans.Streaming/Orleans.Streaming.csproj +++ b/src/Orleans.Streaming/Orleans.Streaming.csproj @@ -21,6 +21,7 @@ + diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 125374e340a..0cebc822a16 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -36,6 +36,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly IStreamFailureHandler streamFailureHandler; private readonly StreamInstruments? _streamInstruments; private readonly TimeProvider _timeProvider; + private readonly CancellationTokenSource _shutdownCancellation = new(); internal readonly QueueId QueueId; private int numMessages; @@ -269,6 +270,7 @@ private async Task ShutdownCore(CancellationToken cancellationToken) var drainTask = _workAdmission.CloseAsync(); var asyncTimer = timer; timer = null; + _shutdownCancellation.Cancel(); var localDeliveryProgressTimer = deliveryProgressTimer; deliveryProgressTimer = null; localDeliveryProgressTimer?.Dispose(); @@ -1874,7 +1876,7 @@ private async Task ErrorProtocol( { LogWarningConsumerIsDead(consumerData.StreamConsumer, consumerData.StreamId); RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); - UnregisterUnavailableConsumer(consumerData, cancellationToken).Ignore(); + UnregisterUnavailableConsumer(consumerData).Ignore(); return true; } @@ -1930,16 +1932,49 @@ bool IsCurrent() => operationId == (isDeliveryError ? consumerData.HandshakeGeneration : consumerData.HandshakeRequestId); } - private async Task UnregisterUnavailableConsumer(StreamConsumerData consumerData, CancellationToken cancellationToken) + private async Task UnregisterUnavailableConsumer(StreamConsumerData consumerData) { StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Requested); try { - await pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId, cancellationToken); + await AsyncExecutorWithRetries.ExecuteWithRetries( + async _ => + { + try + { + await pubSub.UnregisterConsumer( + consumerData.SubscriptionId, + consumerData.StreamId, + _shutdownCancellation.Token); + return true; + } + catch (Exception exception) when (!_shutdownCancellation.IsCancellationRequested) + { + StreamingEvents.EmitSubscriptionUnregistration( + streamProviderName, + consumerData, + Silo, + StreamingEvents.SubscriptionUnregistrationStage.Failed, + exception); + LogWarningUnregisterUnavailableConsumer( + consumerData.SubscriptionId, + consumerData.StreamId, + exception); + throw; + } + }, + AsyncExecutorWithRetries.INFINITE_RETRIES, + static (_, _) => true, + Timeout.InfiniteTimeSpan, + deliveryBackoffProvider, + _shutdownCancellation.Token); StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Completed); } + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + { + } catch (Exception exception) { StreamingEvents.EmitSubscriptionUnregistration( diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCacheBatchContainerFilter.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCacheBatchContainerFilter.cs new file mode 100644 index 00000000000..76ea89e52f5 --- /dev/null +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCacheBatchContainerFilter.cs @@ -0,0 +1,8 @@ +namespace Orleans.Streams; + +internal interface IQueueCacheBatchContainerFilter +{ + IBatchContainer FilterFrom(StreamSequenceToken inclusiveStartToken); + + IBatchContainer? FilterAfter(StreamSequenceToken exclusiveStartToken); +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index 08d5bb1efb9..6c020424612 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -111,7 +111,9 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() adapter.CreateReceiver(queueId), factory.GetQueueAdapterCache().CreateQueueCache(queueId)); var firstReceiver = adapter.CreateReceiver(queueId); - await firstReceiver.Shutdown(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await ((IQueueAdapterReceiver)firstReceiver).Shutdown( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); var reassignedReceiver = adapter.CreateReceiver(queueId); Assert.NotSame(firstReceiver, reassignedReceiver); Assert.Same( diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 8cc19b2ff6b..41413295641 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -107,6 +107,29 @@ public async Task TimedOutCreateAdapterDoesNotEnterCriticalSection() Assert.Equal(1, factory.AdapterConstructionCount); } + [Fact] + public async Task CreateAdapterInterfaceOverloadPropagatesCallerCancellation() + { + var lifetime = new FakeHostApplicationLifetime(); + var adapter = Substitute.For(); + var factory = new BlockingAdoNetQueueAdapterFactory( + CreateQueries(new CapturingRelationalStorage()), + adapter, + lifetime); + var first = factory.CreateAdapter(); + await factory.AdapterConstructionStarted.Task; + using var cancellation = new CancellationTokenSource(); + + var canceled = ((IQueueAdapterFactory)factory).CreateAdapter(cancellation.Token); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync(() => canceled); + Assert.True(exception.CancellationToken.IsCancellationRequested); + Assert.Equal(1, factory.AdapterConstructionCount); + factory.CompleteAdapterConstruction(); + Assert.Same(adapter, await first); + } + [Fact] public void ResolveCheckpointUpdate_ReturnsAuthoritativeStateForExpectedVersionConflict() { @@ -695,7 +718,12 @@ private sealed class BlockingAdoNetQueueAdapterFactory( public void CompleteAdapterConstruction() => _adapterConstruction.SetResult(); - internal override ValueTask GetQueriesAsync() => new(queries); + internal override ValueTask GetQueriesAsync( + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(queries); + } internal override async ValueTask CreateAdapterCore(RelationalOrleansQueries value) { diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs index 1e1f2c6dbcf..3d7ae9aa85d 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -579,6 +579,118 @@ async Task Deliver() } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PooledReceiver_NonzeroStartTokenDeliversRequestedRecordSuffix(bool includeNewerRecord) + { + const string offset = "123456789012345678901234567890"; + const string nextOffset = "123456789012345678901234567891"; + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var records = includeNewerRecord + ? new[] { fixture.Record(offset, "first", "second", "third"), fixture.Record(nextOffset, "next") } + : new[] { fixture.Record(offset, "first", "second", "third") }; + fixture.SetReads(client, () => Response("tail", records)); + await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); + + var startToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 1); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursor(fixture.StreamId, startToken).Cursor); + var batch = ReadBatch(cursor); + + Assert.Equal(startToken, batch.SequenceToken); + Assert.Equal(["second", "third"], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal([1, 2], batch.GetEvents().Select(item => item.Item2.EventIndex)); + cursor.RecordDeliveryFailure(); + var retried = ReadBatch(cursor); + Assert.Equal(startToken, retried.SequenceToken); + Assert.Equal(["second", "third"], retried.GetEvents().Select(item => item.Item1)); + Assert.Equal([1, 2], retried.GetEvents().Select(item => item.Item2.EventIndex)); + if (includeNewerRecord) + { + AssertBatch(ReadBatch(cursor), nextOffset, 1, "next"); + } + else + { + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PooledReceiver_NonzeroDeliveryTokenResumesAfterAcknowledgedEvent(bool includeNewerRecord) + { + const string offset = "123456789012345678901234567890"; + const string nextOffset = "123456789012345678901234567891"; + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var records = includeNewerRecord + ? new[] { fixture.Record(offset, "first", "second", "third"), fixture.Record(nextOffset, "next") } + : new[] { fixture.Record(offset, "first", "second", "third") }; + fixture.SetReads(client, () => Response("tail", records)); + await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); + + var deliveredToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 1); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursor(fixture.StreamId, deliveredToken).Cursor); + Assert.IsAssignableFrom(cursor).SetDeliveredThrough(deliveredToken); + var batch = ReadBatch(cursor); + + Assert.Equal(2, batch.SequenceToken.EventIndex); + Assert.Equal(["third"], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal([2], batch.GetEvents().Select(item => item.Item2.EventIndex)); + if (includeNewerRecord) + { + AssertBatch(ReadBatch(cursor), nextOffset, 1, "next"); + } + else + { + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, cursor.MoveNextWithResult().Kind); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PooledReceiver_FinalEventDeliveryTokenPreservesSafePosition(bool includeNewerRecord) + { + const string offset = "123456789012345678901234567890"; + const string nextOffset = "123456789012345678901234567891"; + await using var fixture = new Fixture(); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + var records = includeNewerRecord + ? new[] { fixture.Record(offset, "first", "second", "third"), fixture.Record(nextOffset, "next") } + : new[] { fixture.Record(offset, "first", "second", "third") }; + fixture.SetReads(client, () => Response("tail", records)); + await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); + + var deliveredToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 2); + using var cursor = Assert.IsAssignableFrom( + receiver.TryGetCacheCursor(fixture.StreamId, deliveredToken).Cursor); + var progress = Assert.IsAssignableFrom(cursor); + progress.SetDeliveredThrough(deliveredToken); + var result = cursor.MoveNextWithResult(); + + Assert.Equal(deliveredToken, progress.SafeSequenceToken); + if (includeNewerRecord) + { + Assert.Equal(QueueCacheCursorMoveResultKind.Success, result.Kind); + AssertBatch(Assert.IsType(cursor.GetCurrent(out _)), nextOffset, 1, "next"); + } + else + { + Assert.Equal(QueueCacheCursorMoveResultKind.NoData, result.Kind); + } + } + [Fact] public async Task Source_ReadReturnsOrderedRawRecordMetadataWithoutDecodingOrMutatingWirePayload() { diff --git a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs index 61f4d5617bb..2af3e2d1ed4 100644 --- a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs +++ b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs @@ -526,6 +526,48 @@ public async Task InvalidatedGatewayRefreshFailureRemainsRetryable() Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); } + [Fact] + public async Task InvalidatedGatewayRefreshContinuesPastFailedOwner() + { + var clientId = Client("multi-owner"); + var failedSilo = Silo("127.0.0.1:222@100"); + var healthySilo = Silo("127.0.0.1:333@100"); + var failed = await AddRemoteClient(failedSilo, clientId); + var healthy = await AddRemoteClient(healthySilo, clientId); + var failure = new TimeoutException("Client directory owner did not respond."); + var attempts = 0; + SiloAddress? successfulSilo = null; + Configure(failed, failedSilo); + Configure(healthy, healthySilo); + + _directory.InvalidateCache(clientId); + + var address = Assert.Single(await _directory.Lookup(clientId)); + + Assert.Equal(Gateway.GetClientActivationAddress(clientId, successfulSilo!), address); + _ = failed.Received(1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + _ = healthy.Received(1).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + + void Configure(IRemoteClientDirectory remote, SiloAddress silo) + { + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(_ => + { + if (Interlocked.Increment(ref attempts) == 1) + { + throw failure; + } + + successfulSilo = silo; + return Task.FromResult( + ImmutableDictionary, long)>.Empty); + }); + } + } + [Fact] public async Task InvalidationKeepsLocallyConnectedClientAvailable() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 0ace0391dfe..1be22cf552a 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -35,9 +35,14 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); data.IsRegistered = true; var unregister = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(unregister.Task); + var unregisterAttempts = 0; + pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + Interlocked.Increment(ref unregisterAttempts) == 1 + ? unregister.Task + : Task.CompletedTask); var events = new ConcurrentQueue(); - var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var failedOutcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completedOutcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using var observer = StreamingEvents.AllEvents.Subscribe(value => { if (value is StreamingEvents.MessageDeliveryFailed failed && failed.SubscriptionId == subscriptionId.Guid) @@ -47,9 +52,13 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking else if (value is StreamingEvents.SubscriptionUnregistration registration && registration.SubscriptionId == subscriptionId.Guid) { events.Enqueue(value); - if (registration.Stage != StreamingEvents.SubscriptionUnregistrationStage.Requested) + if (registration.Stage == StreamingEvents.SubscriptionUnregistrationStage.Failed) { - outcome.TrySetResult(registration); + failedOutcome.TrySetResult(registration); + } + else if (registration.Stage == StreamingEvents.SubscriptionUnregistrationStage.Completed) + { + completedOutcome.TrySetResult(registration); } } }); @@ -72,7 +81,8 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking Assert.Equal(siloAddress, requested.SiloAddress); Assert.Same(consumer, requested.Consumer); Assert.Null(requested.Exception); - Assert.False(outcome.Task.IsCompleted); + Assert.False(failedOutcome.Task.IsCompleted); + Assert.False(completedOutcome.Task.IsCompleted); Assert.False(streamData.Contains(subscriptionId)); Assert.Null(data.Cursor); Assert.DoesNotContain(streamId, await accessor.GetPubSubCache()); @@ -87,16 +97,22 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking unregister.SetResult(); } - var completed = await outcome.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal( - failUnregistration ? StreamingEvents.SubscriptionUnregistrationStage.Failed : StreamingEvents.SubscriptionUnregistrationStage.Completed, - completed.Stage); + if (failUnregistration) + { + var failedRegistration = await failedOutcome.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(StreamingEvents.SubscriptionUnregistrationStage.Failed, failedRegistration.Stage); + Assert.Same(exception, failedRegistration.Exception); + } + + var completed = await completedOutcome.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(StreamingEvents.SubscriptionUnregistrationStage.Completed, completed.Stage); Assert.Equal(streamId.StreamId, completed.StreamId); Assert.Equal(streamId.ProviderName, completed.StreamProvider); Assert.Equal(siloAddress, completed.SiloAddress); Assert.Same(consumer, completed.Consumer); - Assert.Same(failUnregistration ? exception : null, completed.Exception); - _ = pubSub.Received(1).UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + Assert.Null(completed.Exception); + _ = pubSub.Received(failUnregistration ? 2 : 1) + .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); } finally { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 939212767b2..543d854a0a0 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -820,6 +820,25 @@ public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetr await receiver.Shutdown(TimeSpan.FromSeconds(5)); } + [Fact] + public async Task Receiver_InterfaceInitializationOverloadPropagatesCallerCancellation() + { + var source = new TestSource([]); + var checkpointer = new BlockingInitializationCheckpointer(); + IQueueAdapterReceiver receiver = CreateReceiver(source, checkpointer); + using var cancellation = new CancellationTokenSource(); + + var initialization = receiver.Initialize(Timeout.InfiniteTimeSpan, cancellation.Token); + await checkpointer.FirstLoadStarted.Task; + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync(() => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.True(checkpointer.FirstLoadCancellation.IsCancellationRequested); + checkpointer.AllowFirstLoadToComplete.TrySetResult(); + await receiver.Shutdown(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + [Fact] public async Task Receiver_IndependentInitializationCancellationIsNotRetried() { From 9f35e47f44fad8c27641519b8b7c9ccb9659899e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 9 Sep 2026 01:20:04 -0700 Subject: [PATCH 68/90] fix(streaming): drain consumer cleanup on shutdown --- .../PersistentStreamPullingAgent.cs | 29 ++++++++++++++- ...stentStreamPullingAgentDiagnosticsTests.cs | 37 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 0cebc822a16..5d17e07dea6 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -37,6 +37,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly StreamInstruments? _streamInstruments; private readonly TimeProvider _timeProvider; private readonly CancellationTokenSource _shutdownCancellation = new(); + private readonly HashSet _pendingUnavailableConsumerUnregistrations = []; internal readonly QueueId QueueId; private int numMessages; @@ -270,7 +271,6 @@ private async Task ShutdownCore(CancellationToken cancellationToken) var drainTask = _workAdmission.CloseAsync(); var asyncTimer = timer; timer = null; - _shutdownCancellation.Cancel(); var localDeliveryProgressTimer = deliveryProgressTimer; deliveryProgressTimer = null; localDeliveryProgressTimer?.Dispose(); @@ -324,6 +324,18 @@ private async Task ShutdownCore(CancellationToken cancellationToken) // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } + try + { + if (_pendingUnavailableConsumerUnregistrations.Count > 0) + { + await Task.WhenAll(_pendingUnavailableConsumerUnregistrations).WaitAsync(cancellationToken); + } + } + finally + { + _shutdownCancellation.Cancel(); + } + var unregisterTasks = new List(); foreach (var tuple in pubSubCache) { @@ -1876,7 +1888,7 @@ private async Task ErrorProtocol( { LogWarningConsumerIsDead(consumerData.StreamConsumer, consumerData.StreamId); RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); - UnregisterUnavailableConsumer(consumerData).Ignore(); + TrackUnavailableConsumerUnregistration(consumerData); return true; } @@ -1932,6 +1944,19 @@ bool IsCurrent() => operationId == (isDeliveryError ? consumerData.HandshakeGeneration : consumerData.HandshakeRequestId); } + private void TrackUnavailableConsumerUnregistration(StreamConsumerData consumerData) + { + var task = UnregisterUnavailableConsumer(consumerData); + _pendingUnavailableConsumerUnregistrations.Add(task); + RemoveWhenComplete(task).Ignore(); + + async Task RemoveWhenComplete(Task pending) + { + await pending.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + _pendingUnavailableConsumerUnregistrations.Remove(pending); + } + } + private async Task UnregisterUnavailableConsumer(StreamConsumerData consumerData) { StreamingEvents.EmitSubscriptionUnregistration( diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 1be22cf552a..7c3bbaee271 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -121,6 +121,43 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking } } + [Fact] + public async Task ShutdownDrainsUnavailableConsumerUnregistrationBeforeCancelingRetries() + { + var streamId = new QualifiedStreamId("provider", StreamId.Create("shutdown-unregister", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var token = new EventSequenceTokenV2(1); + var cache = new ScriptedQueueCache(); + cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); + var (accessor, pubSub, streamData) = await CreateInitializedAgentWithStream( + streamId, token, cache, new StreamPullingAgentOptions()); + var unavailable = Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException(); + var data = streamData.AddConsumer( + subscriptionId, + streamId, + new UnavailableConsumer(unavailable), + filterData: null, + DateTime.UtcNow); + data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); + data.IsRegistered = true; + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + Interlocked.Increment(ref attempts) == 1 ? firstAttempt.Task : Task.CompletedTask); + + await accessor.RunConsumerCursor(data).WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); + var shutdown = accessor.Shutdown(); + + Assert.False(shutdown.IsCompleted); + firstAttempt.SetException(new InvalidOperationException("transient cleanup failure")); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + _ = pubSub.Received(2) + .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + } + private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension { public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, From fbc83cc017770f3a9a23b61338f129a2908096e7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 9 Sep 2026 01:39:22 -0700 Subject: [PATCH 69/90] fix(testing): bound streaming subscription diagnostic reads --- .../StreamingDiagnosticObserverTests.cs | 213 ++++++++++++++++++ .../StreamingDiagnosticObserver.cs | 119 +++++++--- 2 files changed, 295 insertions(+), 37 deletions(-) create mode 100644 test/Orleans.Core.Tests/Diagnostics/StreamingDiagnosticObserverTests.cs diff --git a/test/Orleans.Core.Tests/Diagnostics/StreamingDiagnosticObserverTests.cs b/test/Orleans.Core.Tests/Diagnostics/StreamingDiagnosticObserverTests.cs new file mode 100644 index 00000000000..cbcd9408e91 --- /dev/null +++ b/test/Orleans.Core.Tests/Diagnostics/StreamingDiagnosticObserverTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Reactive.Subjects; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using TestExtensions; +using Xunit; +using StreamingEvents = Orleans.Streaming.Diagnostics.StreamingEvents; + +namespace UnitTests.Diagnostics; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +[TestCategory("BVT")] +public class StreamingDiagnosticObserverTests +{ + private static readonly SiloAddress Silo = SiloAddress.New(IPAddress.Loopback, 12030, 1); + private const string Provider = "provider"; + private const string NoEvents = "No matching lifecycle events were observed."; + + [Fact] + public void Diagnostics_FormatsOnlyLatestSixteenEvents() + { + using var events = new Subject(); + using var observer = new StreamingDiagnosticObserver(DiagnosticObserverSiloScope.For(Silo), events); + var streamId = StreamId.Create("diagnostics", Guid.NewGuid()); + var subscriptionId = Guid.NewGuid(); + var failures = Enumerable.Range(0, 65).Select(index => new CountingException($"failure-{index}")).ToArray(); + for (var index = 0; index < 64; index++) + { + events.OnNext(Failure(streamId, subscriptionId, failures[index], index)); + } + + Assert.All(failures, failure => Assert.Equal(0, failure.FormatCount)); + var first = observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider); + var lines = first.Split(Environment.NewLine); + Assert.Equal(16, lines.Length); + for (var index = 0; index < 16; index++) + { + Assert.EndsWith($": failure-{index + 48}", lines[index]); + } + + Assert.All(failures.Take(48), failure => Assert.Equal(0, failure.FormatCount)); + Assert.All(failures.Skip(48).Take(16), failure => Assert.Equal(1, failure.FormatCount)); + Assert.Equal(first, observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider)); + Assert.All(failures.Skip(48).Take(16), failure => Assert.Equal(2, failure.FormatCount)); + + events.OnNext(Failure(streamId, subscriptionId, failures[64], 64)); + var updated = observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider).Split(Environment.NewLine); + Assert.Equal(16, updated.Length); + Assert.EndsWith(": failure-49", updated[0]); + Assert.EndsWith(": failure-64", updated[^1]); + Assert.Equal(2, failures[48].FormatCount); + Assert.Equal(1, failures[64].FormatCount); + } + + [Fact] + public void Diagnostics_IsolatesProviderStreamSubscriptionAndSilo() + { + using var events = new Subject(); + using var observer = new StreamingDiagnosticObserver(DiagnosticObserverSiloScope.For(Silo), events); + var streamId = StreamId.Create("diagnostics", Guid.NewGuid()); + var otherStream = StreamId.Create("diagnostics", Guid.NewGuid()); + var subscriptionId = Guid.NewGuid(); + var otherSubscription = Guid.NewGuid(); + var consumer = new Consumer(); + events.OnNext(new StreamingEvents.SubscriptionUnregistration( + Provider, streamId, subscriptionId, Silo, consumer, + StreamingEvents.SubscriptionUnregistrationStage.Requested, null)); + for (var index = 0; index < 20; index++) + { + events.OnNext(new StreamingEvents.SubscriptionUnregistration( + "other-provider", streamId, subscriptionId, Silo, consumer, + StreamingEvents.SubscriptionUnregistrationStage.Completed, null)); + events.OnNext(new StreamingEvents.SubscriptionUnregistration( + Provider, otherStream, subscriptionId, Silo, consumer, + StreamingEvents.SubscriptionUnregistrationStage.Completed, null)); + events.OnNext(new StreamingEvents.SubscriptionUnregistration( + Provider, streamId, otherSubscription, Silo, consumer, + StreamingEvents.SubscriptionUnregistrationStage.Completed, null)); + events.OnNext(new StreamingEvents.SubscriptionUnregistration( + Provider, streamId, subscriptionId, SiloAddress.New(IPAddress.Loopback, 12031, 1), consumer, + StreamingEvents.SubscriptionUnregistrationStage.Failed, new InvalidOperationException("other silo"))); + } + + Assert.Equal($"Unregistration Requested on {Silo} for consumer: ", + observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider)); + Assert.Equal(16, observer.GetSubscriptionDiagnostics(streamId, subscriptionId, "other-provider").Split(Environment.NewLine).Length); + Assert.Equal(16, observer.GetSubscriptionDiagnostics(otherStream, subscriptionId, Provider).Split(Environment.NewLine).Length); + Assert.Equal(16, observer.GetSubscriptionDiagnostics(streamId, otherSubscription, Provider).Split(Environment.NewLine).Length); + Assert.Equal(NoEvents, observer.GetSubscriptionDiagnostics(otherStream, otherSubscription, Provider)); + } + + [Fact] + public async Task Diagnostics_EvictionPreservesHistoricalAndFutureLifecycleWaits() + { + using var events = new Subject(); + using var observer = new StreamingDiagnosticObserver(DiagnosticObserverSiloScope.For(Silo), events); + var streamId = StreamId.Create("diagnostics", Guid.NewGuid()); + var subscriptionId = Guid.NewGuid(); + var consumerId = GrainId.Create("consumer", "one"); + var unregistered = new StreamingEvents.SubscriptionUnregistered(Provider, streamId, subscriptionId, Silo); + events.OnNext(new StreamingEvents.SubscriptionAttached(Provider, streamId, subscriptionId, consumerId, Silo)); + events.OnNext(new StreamingEvents.ConsumerCursorDrained(Provider, streamId, subscriptionId, Silo)); + events.OnNext(unregistered); + Assert.Equal( + new[] { $"Attached on {Silo} to {consumerId}", $"Cursor drained on {Silo}", $"Durably unregistered on {Silo}" }, + observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider).Split(Environment.NewLine)); + + for (var index = 0; index < 16; index++) + { + events.OnNext(new StreamingEvents.ConsumerCursorDrained(Provider, streamId, subscriptionId, Silo)); + } + + var diagnostics = observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider).Split(Environment.NewLine); + Assert.Equal(16, diagnostics.Length); + Assert.All(diagnostics, entry => Assert.Equal($"Cursor drained on {Silo}", entry)); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromSeconds(5)); + Assert.Same(unregistered, await observer.WaitForSubscriptionUnregisteredAsync(streamId, subscriptionId, Provider, cancellation.Token)); + var nextSubscription = Guid.NewGuid(); + var future = observer.WaitForSubscriptionUnregisteredAsync(streamId, nextSubscription, Provider, cancellation.Token); + Assert.False(future.IsCompleted); + var next = new StreamingEvents.SubscriptionUnregistered(Provider, streamId, nextSubscription, Silo); + events.OnNext(next); + Assert.Same(next, await future); + } + + [Fact] + public async Task Diagnostics_FormattingDoesNotBlockNewEvents() + { + using var events = new Subject(); + using var observer = new StreamingDiagnosticObserver(DiagnosticObserverSiloScope.For(Silo), events); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromSeconds(5)); + var streamId = StreamId.Create("diagnostics", Guid.NewGuid()); + var subscriptionId = Guid.NewGuid(); + using var release = new ManualResetEventSlim(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var failure = new BlockingException(entered, release, cancellation.Token); + events.OnNext(Failure(streamId, subscriptionId, failure)); + var snapshot = Task.Run(() => observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider), cancellation.Token); + try + { + await entered.Task.WaitAsync(cancellation.Token); + await Task.Run(() => events.OnNext( + new StreamingEvents.ConsumerCursorDrained(Provider, streamId, subscriptionId, Silo)), cancellation.Token) + .WaitAsync(cancellation.Token); + } + finally + { + release.Set(); + } + + Assert.Single((await snapshot.WaitAsync(cancellation.Token)).Split(Environment.NewLine)); + var current = observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider).Split(Environment.NewLine); + Assert.Equal(2, current.Length); + Assert.EndsWith(": blocked-format", current[0]); + Assert.Equal($"Cursor drained on {Silo}", current[1]); + } + + [Fact] + public void Dispose_ReleasesSubscriptionAndDiagnosticHistory() + { + using var events = new Subject(); + var observer = new StreamingDiagnosticObserver(DiagnosticObserverSiloScope.For(Silo), events); + var streamId = StreamId.Create("diagnostics", Guid.NewGuid()); + var subscriptionId = Guid.NewGuid(); + events.OnNext(new StreamingEvents.ConsumerCursorDrained(Provider, streamId, subscriptionId, Silo)); + Assert.True(events.HasObservers); + Assert.Equal($"Cursor drained on {Silo}", observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider)); + + observer.Dispose(); + observer.Dispose(); + events.OnNext(new StreamingEvents.ConsumerCursorDrained(Provider, streamId, subscriptionId, Silo)); + + Assert.False(events.HasObservers); + Assert.Equal(NoEvents, observer.GetSubscriptionDiagnostics(streamId, subscriptionId, Provider)); + } + + private static StreamingEvents.MessageDeliveryFailed Failure( + StreamId streamId, Guid subscriptionId, Exception exception, int sequence = 0) + => new(Provider, streamId, subscriptionId, Silo, new Consumer(), new EventSequenceTokenV2(sequence), exception); + + private sealed class Consumer : IAddressable + { + public override string ToString() => "consumer"; + } + + private sealed class CountingException(string message) : Exception(message) + { + public int FormatCount { get; private set; } + + public override string ToString() + { + FormatCount++; + return Message; + } + } + + private sealed class BlockingException( + TaskCompletionSource entered, + ManualResetEventSlim release, + CancellationToken cancellationToken) : Exception + { + public override string ToString() + { + entered.TrySetResult(); + release.Wait(cancellationToken); + return "blocked-format"; + } + } +} diff --git a/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs b/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs index 3384e0610e7..a5263afc899 100644 --- a/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs +++ b/test/TestInfrastructure/TestExtensions/Diagnostics/StreamingDiagnosticObserver.cs @@ -18,8 +18,12 @@ namespace TestExtensions; /// public sealed class StreamingDiagnosticObserver : IDisposable { + private const int DiagnosticHistoryLimit = 16; private readonly IConnectableObservable _events; private readonly IDisposable _connection; + private readonly object _diagnosticsLock = new(); + private readonly Dictionary<(StreamId StreamId, Guid SubscriptionId, string Provider), Queue> _subscriptionDiagnostics = []; + private bool _disposed; /// /// Creates a new instance of the observer and starts listening for streaming diagnostic events. @@ -31,11 +35,19 @@ public sealed class StreamingDiagnosticObserver : IDisposable public static StreamingDiagnosticObserver Create(SiloAddress siloAddress) => new(DiagnosticObserverSiloScope.For(siloAddress)); private StreamingDiagnosticObserver(DiagnosticObserverSiloScope scope) + : this(scope, StreamingEvents.AllEvents) { - _events = StreamingEvents.AllEvents + } + + internal StreamingDiagnosticObserver( + DiagnosticObserverSiloScope scope, + IObservable events) + { + _events = events .Where(e => e is StreamingEvents.ItemDelivered delivered ? scope.Matches(delivered.SiloAddress, delivered.ClusterId) : scope.Matches(e.SiloAddress)) + .Do(RecordSubscriptionDiagnostic) .Replay(); _connection = _events.Connect(); } @@ -409,48 +421,72 @@ await _events } /// - /// Summarizes the most recent lifecycle events for one subscription. + /// Summarizes up to 16 of the most recent lifecycle events for one subscription. /// public string GetSubscriptionDiagnostics(StreamId streamId, Guid subscriptionId, string streamProvider) { - var recent = new Queue(); - using var subscription = _events.Subscribe(value => + StreamingEvents.StreamingEvent[] recent; + lock (_diagnosticsLock) { - var entry = value switch - { - StreamingEvents.MessageDeliveryFailed failed - when MatchesSubscription(failed.StreamId, failed.SubscriptionId, failed.StreamProvider, streamId, subscriptionId, streamProvider) - => $"Delivery failed on {failed.SiloAddress} to {failed.Consumer} at {failed.SequenceToken}: {failed.Exception}", - StreamingEvents.SubscriptionUnregistration unregister - when MatchesSubscription(unregister.StreamId, unregister.SubscriptionId, unregister.StreamProvider, streamId, subscriptionId, streamProvider) - => $"Unregistration {unregister.Stage} on {unregister.SiloAddress} for {unregister.Consumer}: {unregister.Exception}", - StreamingEvents.SubscriptionUnregistered unregistered - when MatchesSubscription(unregistered.StreamId, unregistered.SubscriptionId, unregistered.StreamProvider, streamId, subscriptionId, streamProvider) - => $"Durably unregistered on {unregistered.SiloAddress}", - StreamingEvents.SubscriptionAttached attached - when MatchesSubscription(attached.StreamId, attached.SubscriptionId, attached.StreamProvider, streamId, subscriptionId, streamProvider) - => $"Attached on {attached.SiloAddress} to {attached.ConsumerGrainId}", - StreamingEvents.ConsumerCursorDrained drained - when MatchesSubscription(drained.StreamId, drained.SubscriptionId, drained.StreamProvider, streamId, subscriptionId, streamProvider) - => $"Cursor drained on {drained.SiloAddress}", - _ => null, - }; - if (entry is not null) + if (!_subscriptionDiagnostics.TryGetValue((streamId, subscriptionId, streamProvider), out var history)) { - lock (recent) - { - if (recent.Count == 16) - { - recent.Dequeue(); - } - - recent.Enqueue(entry); - } + return "No matching lifecycle events were observed."; } - }); - lock (recent) + + recent = history.ToArray(); + } + + return string.Join(Environment.NewLine, recent.Select(static value => value switch { - return recent.Count == 0 ? "No matching lifecycle events were observed." : string.Join(Environment.NewLine, recent); + StreamingEvents.MessageDeliveryFailed failed + => $"Delivery failed on {failed.SiloAddress} to {failed.Consumer} at {failed.SequenceToken}: {failed.Exception}", + StreamingEvents.SubscriptionUnregistration unregister + => $"Unregistration {unregister.Stage} on {unregister.SiloAddress} for {unregister.Consumer}: {unregister.Exception}", + StreamingEvents.SubscriptionUnregistered unregistered + => $"Durably unregistered on {unregistered.SiloAddress}", + StreamingEvents.SubscriptionAttached attached + => $"Attached on {attached.SiloAddress} to {attached.ConsumerGrainId}", + StreamingEvents.ConsumerCursorDrained drained + => $"Cursor drained on {drained.SiloAddress}", + _ => throw new InvalidOperationException("Unexpected subscription diagnostic event."), + })); + } + + private void RecordSubscriptionDiagnostic(StreamingEvents.StreamingEvent value) + { + (StreamId StreamId, Guid SubscriptionId, string Provider)? key = value switch + { + StreamingEvents.MessageDeliveryFailed failed => (failed.StreamId, failed.SubscriptionId, failed.StreamProvider), + StreamingEvents.SubscriptionUnregistration unregister => (unregister.StreamId, unregister.SubscriptionId, unregister.StreamProvider), + StreamingEvents.SubscriptionUnregistered unregistered => (unregistered.StreamId, unregistered.SubscriptionId, unregistered.StreamProvider), + StreamingEvents.SubscriptionAttached attached => (attached.StreamId, attached.SubscriptionId, attached.StreamProvider), + StreamingEvents.ConsumerCursorDrained drained => (drained.StreamId, drained.SubscriptionId, drained.StreamProvider), + _ => null, + }; + if (key is not { } subscriptionKey) + { + return; + } + + lock (_diagnosticsLock) + { + if (_disposed) + { + return; + } + + if (!_subscriptionDiagnostics.TryGetValue(subscriptionKey, out var history)) + { + history = new Queue(DiagnosticHistoryLimit); + _subscriptionDiagnostics.Add(subscriptionKey, history); + } + + if (history.Count == DiagnosticHistoryLimit) + { + history.Dequeue(); + } + + history.Enqueue(value); } } @@ -591,7 +627,16 @@ private static bool MatchesProvider(string eventStreamProvider, string? streamPr } /// - public void Dispose() => _connection.Dispose(); + public void Dispose() + { + lock (_diagnosticsLock) + { + _disposed = true; + _subscriptionDiagnostics.Clear(); + } + + _connection.Dispose(); + } } /// From 99b0f312c4deea6391b4ab97a51ddb30a39b10ed Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 9 Sep 2026 01:55:36 -0700 Subject: [PATCH 70/90] fix(streaming): use numeric Kinesis offsets when slicing batches --- .../Streams/KinesisBatchContainer.cs | 4 +- .../Streams/KinesisRecoverableStream.cs | 2 +- .../KinesisBatchContainerTests.cs | 53 +++++++++++++++++++ .../KinesisPooledRuntimeTests.cs | 30 ++++++----- 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 85d282ba098..f1c15728b34 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -116,7 +116,7 @@ public int CompareTo(KinesisBatchContainer? other) IBatchContainer IQueueCacheBatchContainerFilter.FilterFrom(StreamSequenceToken inclusiveStartToken) { if (inclusiveStartToken is not KinesisSequenceToken token - || !string.Equals(token.ShardSequence, Token.ShardSequence, StringComparison.Ordinal) + || KinesisRecoverableStreamDataAdapter.CompareShardSequences(token.ShardSequence, Token.ShardSequence) != 0 || token.EventIndex <= 0) { return this; @@ -134,7 +134,7 @@ IBatchContainer IQueueCacheBatchContainerFilter.FilterFrom(StreamSequenceToken i IBatchContainer? IQueueCacheBatchContainerFilter.FilterAfter(StreamSequenceToken exclusiveStartToken) { if (exclusiveStartToken is not KinesisSequenceToken token - || !string.Equals(token.ShardSequence, Token.ShardSequence, StringComparison.Ordinal)) + || KinesisRecoverableStreamDataAdapter.CompareShardSequences(token.ShardSequence, Token.ShardSequence) != 0) { return this; } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs index b7610d0dc2f..b446cc876e6 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -235,7 +235,7 @@ public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) return CompareShardSequences(shardSequence, kinesisToken.ShardSequence); } - private static int CompareShardSequences(string left, string right) + internal static int CompareShardSequences(string left, string right) { var leftStart = 0; while (leftStart < left.Length && left[leftStart] == '0') diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index 3222da5bfd0..d321875d73d 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -3,6 +3,7 @@ using Orleans.Runtime; using Orleans.Serialization; using Orleans.Streaming.Kinesis; +using Orleans.Streams; using TestExtensions; using Xunit; using KinesisRecord = Amazon.Kinesis.Model.Record; @@ -126,6 +127,58 @@ public void CompareToOrdersByDurableShardSequenceNotReceiverLocalOrdinal() Assert.True(readSecondButOlder.CompareTo(readFirstButNewer) < 0); } + [Theory] + [InlineData("7", "7")] + [InlineData("7", "0007")] + [InlineData("0007", "7")] + [InlineData("0", "000")] + [InlineData("000", "0")] + [InlineData("170141183460469231731687303715884105727", "000170141183460469231731687303715884105727")] + [InlineData("000170141183460469231731687303715884105727", "170141183460469231731687303715884105727")] + public void BatchFilters_RespectNumericShardSequenceIdentity(string recordOffset, string recoveredOffset) + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, streamId, new object[] { "first", 2, "third", "fourth" }, requestContext: null); + var batch = KinesisBatchContainer.FromCachedRecord(serializer, streamId, payload, recordOffset, sequenceId: 7); + var filter = (IQueueCacheBatchContainerFilter)batch; + var token = new KinesisSequenceToken(recoveredOffset, sequenceNumber: 999, eventIndex: 2); + + var inclusive = Assert.IsType(filter.FilterFrom(token)); + Assert.Equal(["third", "fourth"], inclusive.GetEvents().Select(item => item.Item1)); + Assert.Equal([2, 3], inclusive.GetEvents().Select(item => item.Item2.EventIndex)); + Assert.Empty(inclusive.GetEvents()); + Assert.Equal(streamId, inclusive.StreamId); + Assert.Equal(recordOffset, inclusive.Token.ShardSequence); + Assert.Equal(7, inclusive.Token.SequenceNumber); + + var exclusive = Assert.IsType(filter.FilterAfter(token)); + Assert.Equal(["fourth"], exclusive.GetEvents().Select(item => item.Item1)); + Assert.Equal([3], exclusive.GetEvents().Select(item => item.Item2.EventIndex)); + Assert.Empty(exclusive.GetEvents()); + Assert.Equal(streamId, exclusive.StreamId); + Assert.Equal(recordOffset, exclusive.Token.ShardSequence); + Assert.Equal(7, exclusive.Token.SequenceNumber); + Assert.Null(filter.FilterAfter(new KinesisSequenceToken(recoveredOffset, sequenceNumber: 999, eventIndex: 3))); + Assert.Equal(["first", "third", "fourth"], batch.GetEvents().Select(item => item.Item1)); + } + + [Theory] + [InlineData("7", "8")] + [InlineData("0008", "7")] + [InlineData("170141183460469231731687303715884105727", "170141183460469231731687303715884105728")] + public void BatchFilters_PreserveBatchesFromDifferentRecords(string recordOffset, string recoveredOffset) + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload(serializer, streamId, new[] { "first", "second" }, requestContext: null); + var batch = KinesisBatchContainer.FromCachedRecord(serializer, streamId, payload, recordOffset, sequenceId: 7); + var filter = (IQueueCacheBatchContainerFilter)batch; + var token = new KinesisSequenceToken(recoveredOffset, sequenceNumber: 7, eventIndex: 1); + + Assert.Same(batch, filter.FilterFrom(token)); + Assert.Same(batch, filter.FilterAfter(token)); + } + [Fact] public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering() { diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs index 3d7ae9aa85d..4540c76b29a 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -580,9 +580,11 @@ async Task Deliver() } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task PooledReceiver_NonzeroStartTokenDeliversRequestedRecordSuffix(bool includeNewerRecord) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task PooledReceiver_NonzeroStartTokenDeliversRequestedRecordSuffix(bool includeNewerRecord, bool zeroPaddedToken) { const string offset = "123456789012345678901234567890"; const string nextOffset = "123456789012345678901234567891"; @@ -596,7 +598,7 @@ public async Task PooledReceiver_NonzeroStartTokenDeliversRequestedRecordSuffix( fixture.SetReads(client, () => Response("tail", records)); await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); - var startToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 1); + var startToken = new KinesisSequenceToken(zeroPaddedToken ? $"000{offset}" : offset, sequenceNumber: 0, eventIndex: 1); using var cursor = Assert.IsAssignableFrom( receiver.TryGetCacheCursor(fixture.StreamId, startToken).Cursor); var batch = ReadBatch(cursor); @@ -620,9 +622,11 @@ public async Task PooledReceiver_NonzeroStartTokenDeliversRequestedRecordSuffix( } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task PooledReceiver_NonzeroDeliveryTokenResumesAfterAcknowledgedEvent(bool includeNewerRecord) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task PooledReceiver_NonzeroDeliveryTokenResumesAfterAcknowledgedEvent(bool includeNewerRecord, bool zeroPaddedToken) { const string offset = "123456789012345678901234567890"; const string nextOffset = "123456789012345678901234567891"; @@ -636,7 +640,7 @@ public async Task PooledReceiver_NonzeroDeliveryTokenResumesAfterAcknowledgedEve fixture.SetReads(client, () => Response("tail", records)); await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); - var deliveredToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 1); + var deliveredToken = new KinesisSequenceToken(zeroPaddedToken ? $"000{offset}" : offset, sequenceNumber: 0, eventIndex: 1); using var cursor = Assert.IsAssignableFrom( receiver.TryGetCacheCursor(fixture.StreamId, deliveredToken).Cursor); Assert.IsAssignableFrom(cursor).SetDeliveredThrough(deliveredToken); @@ -656,9 +660,11 @@ public async Task PooledReceiver_NonzeroDeliveryTokenResumesAfterAcknowledgedEve } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task PooledReceiver_FinalEventDeliveryTokenPreservesSafePosition(bool includeNewerRecord) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task PooledReceiver_FinalEventDeliveryTokenPreservesSafePosition(bool includeNewerRecord, bool zeroPaddedToken) { const string offset = "123456789012345678901234567890"; const string nextOffset = "123456789012345678901234567891"; @@ -672,7 +678,7 @@ public async Task PooledReceiver_FinalEventDeliveryTokenPreservesSafePosition(bo fixture.SetReads(client, () => Response("tail", records)); await receiver.GetQueueMessagesAsync(records.Length, TestCancellation); - var deliveredToken = new KinesisSequenceToken(offset, sequenceNumber: 0, eventIndex: 2); + var deliveredToken = new KinesisSequenceToken(zeroPaddedToken ? $"000{offset}" : offset, sequenceNumber: 0, eventIndex: 2); using var cursor = Assert.IsAssignableFrom( receiver.TryGetCacheCursor(fixture.StreamId, deliveredToken).Cursor); var progress = Assert.IsAssignableFrom(cursor); From 903db7d26e4728da7a52c41d5c8dfc80afbae630 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 9 Sep 2026 02:16:27 -0700 Subject: [PATCH 71/90] fix(runtime): isolate client route invalidations --- .../GrainDirectory/ClientDirectory.cs | 25 ++-- .../Directory/ClientDirectoryTests.cs | 114 +++++++++++++++++- 2 files changed, 124 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index 80637673d84..9002b994e7a 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -57,7 +57,8 @@ internal sealed partial class ClientDirectory : SystemTarget, ILocalClientDirect private ImmutableHashSet _localClients = ImmutableHashSet.Empty; private ImmutableDictionary> _currentSnapshot = ImmutableDictionary>.Empty; private ImmutableDictionary ConnectedClients, long Version)> _table = ImmutableDictionary ConnectedClients, long Version)>.Empty; - private volatile ImmutableDictionary _pendingRefreshes = ImmutableDictionary.Empty; + private volatile ImmutableDictionary<(GrainId GrainId, SiloAddress SiloAddress), object> _pendingRefreshes = + ImmutableDictionary<(GrainId, SiloAddress), object>.Empty; // For synchronization with remote silos. private readonly AdmissionGate _publicationGate = new(); @@ -167,9 +168,9 @@ public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out List !pendingRefreshes.ContainsKey(candidate.SiloAddress!)); + clientRoutes = clientRoutes.FindAll(candidate => !pendingRefreshes.ContainsKey((grainId, candidate.SiloAddress!))); break; } } @@ -204,7 +205,7 @@ public void InvalidateCache(GrainId grainId) { if (!route.SiloAddress!.Equals(_localSilo)) { - pending[route.SiloAddress] = token; + pending[(grainId, route.SiloAddress)] = token; } } @@ -236,7 +237,7 @@ public void InvalidateCache(GrainId grainId) // A cached candidate excluded by TryLocalLookup has a pending owner refresh. var candidate = candidates.FirstOrDefault(candidate => - _pendingRefreshes.TryGetValue(candidate.SiloAddress!, out var pendingToken) + _pendingRefreshes.TryGetValue((grainId, candidate.SiloAddress!), out var pendingToken) && (!attemptedRefreshes.TryGetValue(candidate.SiloAddress!, out var attemptedToken) || !ReferenceEquals(attemptedToken, pendingToken))); if (candidate is null) @@ -250,7 +251,7 @@ public void InvalidateCache(GrainId grainId) } silo = candidate.SiloAddress!; - token = _pendingRefreshes[silo]; + token = _pendingRefreshes[(grainId, silo)]; attemptedRefreshes[silo] = token; versionVector = _table.ToImmutableDictionary(e => e.Key, e => e.Value.Version); } @@ -265,9 +266,9 @@ public void InvalidateCache(GrainId grainId) // Keep the versioned row while refreshing: a same-version response confirms it, // and a delayed response only completes the invalidation which initiated its read. - if (_pendingRefreshes.TryGetValue(silo, out var currentToken) && ReferenceEquals(currentToken, token)) + if (_pendingRefreshes.TryGetValue((grainId, silo), out var currentToken) && ReferenceEquals(currentToken, token)) { - _pendingRefreshes = _pendingRefreshes.Remove(silo); + _pendingRefreshes = _pendingRefreshes.Remove((grainId, silo)); } } } @@ -392,8 +393,6 @@ private void UpdateRoutingTable(ImmutableDictionary 0) + { + // Retire invalidations along with the client routes removed from the versioned table. + _pendingRefreshes = _pendingRefreshes.RemoveRange(_pendingRefreshes.Keys.Where(key => + !_table.TryGetValue(key.SiloAddress, out var owner) || !owner.ConnectedClients.Contains(key.GrainId))); + } } } } diff --git a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs index 2af3e2d1ed4..64a9a668012 100644 --- a/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs +++ b/test/Orleans.Core.Tests/Directory/ClientDirectoryTests.cs @@ -374,7 +374,7 @@ public async Task InvalidationRefreshesAllCachedGateways() [InlineData(1, true)] [InlineData(5, false)] [InlineData(5, true)] - public async Task DiscoveryRefreshesNewlyRelevantGateway(int discoveryAttempt, bool refreshFails) + public async Task DiscoveryKeepsUnrelatedClientsAvailable(int discoveryAttempt, bool refreshFails) { var clientId = Client("discovered"); var otherClientId = Client("already-invalidated"); @@ -403,22 +403,126 @@ public async Task DiscoveryRefreshesNewlyRelevantGateway(int discoveryAttempt, b _directory.InvalidateCache(otherClientId); Assert.False(_directory.TryLocalLookup(clientId, out _)); + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); + Assert.True(_directory.TryLocalLookup(clientId, out _)); + Assert.False(_directory.TryLocalLookup(otherClientId, out _)); + _ = remote.Received(discoveryAttempt).GetClientRoutes( + Arg.Any>(), + Arg.Any()); + if (refreshFails) { - Assert.Same(failure, await Assert.ThrowsAsync(() => _directory.Lookup(clientId).AsTask())); - Assert.False(_directory.TryLocalLookup(clientId, out _)); + Assert.Same(failure, await Assert.ThrowsAsync(() => _directory.Lookup(otherClientId).AsTask())); + Assert.False(_directory.TryLocalLookup(otherClientId, out _)); } else { - Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(await _directory.Lookup(clientId))); - Assert.True(_directory.TryLocalLookup(clientId, out _)); + Assert.Equal(Gateway.GetClientActivationAddress(otherClientId, remoteSilo), Assert.Single(await _directory.Lookup(otherClientId))); + Assert.True(_directory.TryLocalLookup(otherClientId, out _)); } + Assert.True(_directory.TryLocalLookup(clientId, out _)); _ = remote.Received(discoveryAttempt + 1).GetClientRoutes( Arg.Any>(), Arg.Any()); } + [Fact] + public async Task InvalidationKeepsUnrelatedRoutesOnSharedGateways() + { + var clientId = Client("invalidated"); + var otherClientId = Client("unrelated"); + var remoteSilos = new[] { Silo("127.0.0.1:222@100"), Silo("127.0.0.1:333@100") }; + var failure = new TimeoutException("Client directory owner did not respond."); + foreach (var silo in remoteSilos) + { + var remote = await AddRemoteClient(silo, clientId); + await _directory.OnUpdateClientRoutes(CreateRoutes(silo, 3, clientId, otherClientId), TestContext.Current.CancellationToken); + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(_ => + Task.FromException, long)>>(failure)); + } + + _directory.InvalidateCache(clientId); + + var expectedRoutes = remoteSilos.Select(silo => Gateway.GetClientActivationAddress(otherClientId, silo)).ToArray(); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + Assert.True(_directory.TryLocalLookup(otherClientId, out var cached)); + Assert.Equal(expectedRoutes, cached.OrderBy(route => route.SiloAddress!.ToString())); + var lookup = _directory.Lookup(otherClientId); + Assert.True(lookup.IsCompletedSuccessfully); + Assert.Equal(expectedRoutes, (await lookup).OrderBy(route => route.SiloAddress!.ToString())); + foreach (var remote in _remoteDirectories.Values) + { + _ = remote.DidNotReceive().GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + + Assert.Same(failure, await Assert.ThrowsAsync(() => _directory.Lookup(clientId).AsTask())); + Assert.True(_directory.TryLocalLookup(otherClientId, out cached)); + Assert.Equal(expectedRoutes, cached.OrderBy(route => route.SiloAddress!.ToString())); + } + + [Fact] + public async Task RefreshForOneClientPreservesAnotherClientsInvalidation() + { + var clientId = Client("first"); + var otherClientId = Client("second"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + await _directory.OnUpdateClientRoutes(CreateRoutes(remoteSilo, 3, clientId, otherClientId), TestContext.Current.CancellationToken); + var firstResponse = new TaskCompletionSource, long)>>( + TaskCreationOptions.RunContinuationsAsynchronously); + var unchanged = ImmutableDictionary, long)>.Empty; + var calls = 0; + remote.GetClientRoutes(default!, Arg.Any()).ReturnsForAnyArgs(_ => + Interlocked.Increment(ref calls) == 1 ? firstResponse.Task : Task.FromResult(unchanged)); + + _directory.InvalidateCache(clientId); + var lookup = _directory.Lookup(clientId).AsTask(); + try + { + Assert.Equal(1, calls); + Assert.False(lookup.IsCompleted); + _directory.InvalidateCache(otherClientId); + firstResponse.SetResult(unchanged); + Assert.Equal( + Gateway.GetClientActivationAddress(clientId, remoteSilo), + Assert.Single(await lookup.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken))); + Assert.Equal(1, calls); + Assert.True(_directory.TryLocalLookup(clientId, out _)); + Assert.False(_directory.TryLocalLookup(otherClientId, out _)); + + Assert.Equal(Gateway.GetClientActivationAddress(otherClientId, remoteSilo), Assert.Single(await _directory.Lookup(otherClientId))); + Assert.Equal(2, calls); + Assert.True(_directory.TryLocalLookup(otherClientId, out _)); + } + finally + { + firstResponse.TrySetResult(unchanged); + } + } + + [Fact] + public async Task OwnerSnapshotRetiresObsoleteClientInvalidation() + { + var clientId = Client("reconnected"); + var otherClientId = Client("unrelated"); + var remoteSilo = Silo("127.0.0.1:222@100"); + var remote = await AddRemoteClient(remoteSilo, clientId); + _directory.InvalidateCache(clientId); + + await _directory.OnUpdateClientRoutes(CreateRoutes(remoteSilo, 3, otherClientId), TestContext.Current.CancellationToken); + Assert.False(_directory.TryLocalLookup(clientId, out _)); + await _directory.OnUpdateClientRoutes(CreateRoutes(remoteSilo, 4, clientId, otherClientId), TestContext.Current.CancellationToken); + + Assert.True(_directory.TryLocalLookup(clientId, out var cached)); + Assert.Equal(Gateway.GetClientActivationAddress(clientId, remoteSilo), Assert.Single(cached)); + _ = remote.DidNotReceive().GetClientRoutes( + Arg.Any>(), + Arg.Any()); + } + [Fact] public async Task InvalidationRefreshesOnlyRequestedClientGateways() { From 4f494d21c266f3e449d3b41c37134abada3b09b4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 01:17:58 -0700 Subject: [PATCH 72/90] fix(streaming): drain in-flight consumer delivery --- .../PersistentStreamPullingAgent.cs | 41 ++++++++++- ...stentStreamPullingAgentDiagnosticsTests.cs | 70 +++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 5d17e07dea6..8e0cc7f97a0 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -37,6 +37,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly StreamInstruments? _streamInstruments; private readonly TimeProvider _timeProvider; private readonly CancellationTokenSource _shutdownCancellation = new(); + private readonly HashSet _activeConsumerCursorTasks = []; private readonly HashSet _pendingUnavailableConsumerUnregistrations = []; internal readonly QueueId QueueId; @@ -146,7 +147,7 @@ IQueueCacheCursor ITestAccessor.GetRecoveryCursor(StreamConsumerData consumerDat => GetRecoveryCursor(consumerData); Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) - => this.RunOrQueueTask(() => RunConsumerCursor(consumerData)); + => this.RunOrQueueTask(() => TrackConsumerCursor(consumerData)); Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); @@ -412,7 +413,7 @@ private async Task AddSubscriber_Impl( data.IsRegistered = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); if (data.State == StreamConsumerDataState.Inactive) - RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + TrackConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so } } finally @@ -1417,7 +1418,7 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq if (consumerData.State == StreamConsumerDataState.Inactive) { // wake up inactive consumers - RunConsumerCursor(consumerData, cancellationToken).Ignore(); + TrackConsumerCursor(consumerData, cancellationToken).Ignore(); } } else @@ -1685,6 +1686,40 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } } + private Task TrackConsumerCursor( + StreamConsumerData consumerData, + CancellationToken cancellationToken = default) + { + var task = RunConsumerCursor(consumerData, cancellationToken); + _activeConsumerCursorTasks.Add(task); + RemoveWhenComplete(task).Ignore(); + return task; + + async Task RemoveWhenComplete(Task pending) + { + await pending.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + _activeConsumerCursorTasks.Remove(pending); + } + } + + private async Task DrainConsumerCursors(CancellationToken cancellationToken) + { + while (_activeConsumerCursorTasks.Count > 0) + { + var active = Task.WhenAll(_activeConsumerCursorTasks); + try + { + await active.WaitAsync(cancellationToken); + } + catch (Exception) when (!cancellationToken.IsCancellationRequested) + { + // RunConsumerCursor logs its failure before propagating it. + } + + cancellationToken.ThrowIfCancellationRequested(); + } + } + private readonly struct ConsumerBatch { public ConsumerBatch(IBatchContainer? batch, StreamSequenceToken? progressToken) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 7c3bbaee271..3ef5b973403 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -158,6 +158,43 @@ await accessor.RunConsumerCursor(data).WaitAsync( .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); } + [Fact] + public async Task ShutdownDrainsCleanupStartedByInFlightDelivery() + { + var streamId = new QualifiedStreamId("provider", StreamId.Create("shutdown-delivery", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var token = new EventSequenceTokenV2(1); + var cache = new ScriptedQueueCache(); + cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); + var (accessor, pubSub, streamData) = await CreateInitializedAgentWithStream( + streamId, token, cache, new StreamPullingAgentOptions()); + var unavailable = Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException(); + var consumer = new BlockingUnavailableConsumer(unavailable); + var data = streamData.AddConsumer(subscriptionId, streamId, consumer, filterData: null, DateTime.UtcNow); + data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); + data.IsRegistered = true; + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + Interlocked.Increment(ref attempts) == 1 ? firstAttempt.Task : Task.CompletedTask); + + var delivery = accessor.RunConsumerCursor(data); + await consumer.DeliveryStarted.Task.WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); + var shutdown = accessor.Shutdown(); + + Assert.False(shutdown.IsCompleted); + consumer.FailDelivery(); + await delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(shutdown.IsCompleted); + firstAttempt.SetException(new InvalidOperationException("transient cleanup failure")); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + _ = pubSub.Received(2) + .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + } + private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension { public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, @@ -179,4 +216,37 @@ private sealed class UnavailableConsumer(ClientNotAvailableException exception) public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(null); } + + private sealed class BlockingUnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension + { + private readonly TaskCompletionSource _delivery = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource DeliveryStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void FailDelivery() => _delivery.SetException(exception); + + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, + StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, + StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) + { + DeliveryStarted.TrySetResult(); + return _delivery.Task; + } + + public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception error, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => Task.FromResult(null); + } } From 122f74a07276d08ebdca85b9622d04186772be4a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 01:45:53 -0700 Subject: [PATCH 73/90] fix(streaming): preserve registration barriers during shutdown --- .../PersistentStreamPullingAgentTests.cs | 73 +++++++++++++++---- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 47ad84ce6b6..a46650fe8b9 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4238,13 +4238,20 @@ private sealed class IsolatedProviderToken(long sequenceNumber) : EventSequenceT [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_WaitsForPendingRegistrationsBeforeDeliveryProgress() + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool registrationFails, bool hasRegisteredStream) { var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var registeredStreamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(_ => registration.Task); + .ReturnsForAnyArgs(call => call.ArgAt(0).Equals(registeredStreamId) + ? Task.FromResult>(new HashSet()) + : registration.Task); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = StreamId.Create("namespace", Guid.NewGuid()); @@ -4270,30 +4277,68 @@ public async Task Shutdown_WaitsForPendingRegistrationsBeforeDeliveryProgress() var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); + if (hasRegisteredStream) + { + await testAccessor.RegisterStream(registeredStreamId, new EventSequenceTokenV2(20), DateTime.UtcNow); + var registeredStream = (await testAccessor.GetPubSubCache())[registeredStreamId]; + var consumer = registeredStream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + registeredStreamId, + streamConsumer: null!, + filterData: null, + now: DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new EventSequenceTokenV2(20); + } + // First tick: pump reads messages and kicks off a cold stream registration. await testAccessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); // Verify the cache has the pending stream registered. var cache = await testAccessor.GetPubSubCache(); - Assert.Single(cache); - var (_, streamData) = cache.Single(); + Assert.Equal(hasRegisteredStream ? 2 : 1, cache.Count); + var streamData = cache[new QualifiedStreamId("provider", streamId)]; Assert.NotNull(streamData.RegistrationTask); Assert.False(streamData.RegistrationTask.IsCompleted, "Registration should still be in progress"); queueCache.ClearDeliveryProgress(); var shutdownTask = testAccessor.Shutdown(); - await testAccessor.GetPubSubCache(); - - Assert.False(shutdownTask.IsCompleted); - Assert.False(receiverShutdownStarted.Task.IsCompleted); - Assert.Empty(queueCache.DeliveryProgressTokens); - Assert.Equal(0, queueCache.DeliveryProgressCallCount); + try + { + // A queued accessor turn observes shutdown after it has started draining registration. + await testAccessor.GetPubSubCache().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(shutdownTask.IsCompleted); + Assert.False(receiverShutdownStarted.Task.IsCompleted); + Assert.Empty(queueCache.DeliveryProgressTokens); + Assert.Equal(0, queueCache.DeliveryProgressCallCount); + + if (registrationFails) + { + registration.SetException(new InvalidOperationException("Producer registration failed during shutdown.")); + } + else + { + registration.SetResult(new HashSet + { + new PubSubSubscriptionState( + GuidId.GetGuidId(Guid.NewGuid()), + new QualifiedStreamId("provider", streamId), + GrainId.Create("consumer", Guid.NewGuid().ToString())), + }); + } + } + finally + { + registration.TrySetResult(new HashSet()); + await shutdownTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } - // Complete registration so shutdown can proceed cleanly. - registration.SetResult(new HashSet()); - await shutdownTask; Assert.True(receiverShutdownStarted.Task.IsCompletedSuccessfully); + Assert.Null(streamData.RegistrationTask); + Assert.False(streamData.StreamRegistered); Assert.Empty(queueCache.DeliveryProgressTokens); + Assert.Equal(0, queueCache.DeliveryProgressCallCount); + await receiver.Received(1).Shutdown(Arg.Any(), Arg.Any()); } [TestSuite("BVT")] From b6e947da6c2a610e0d20b546cd415c86e4242f56 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 13:16:03 -0700 Subject: [PATCH 74/90] docs(streaming): clarify minimum ADO.NET retention intervals --- docs/site/src/content/docs/streaming/stream-providers.md | 2 +- .../Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs | 9 ++++++--- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 2b17eb4c930..44a773fbcf5 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -72,7 +72,7 @@ Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/S Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. Message-ID allocation and insertion commit together, producing gap-free IDs within each partition. The stream partition pipeline resumes strictly after the earliest safe scan/delivery checkpoint, redelivering uncheckpointed records after recovery. Initial acquisition can start immediately before the earliest retained message. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. - retains checkpointed records for one day by default before cleanup. makes older records eligible for hard deletion, including unprocessed records. Retention and cleanup intervals round fractional seconds upward. Cleanup inspects at most records in message-ID order and deletes the eligible prefix ending before the first ineligible record. A younger leading record therefore retains later records even when database timestamps move backward. + retains checkpointed records for one day by default before cleanup. makes older records eligible for hard deletion, including unprocessed records. Retention and cleanup intervals must be at least one second; for valid intervals, fractional seconds round upward to the next whole second. Cleanup inspects at most records in message-ID order and deletes the eligible prefix ending before the first ineligible record. A younger leading record therefore retains later records even when database timestamps move backward. The receiver checks each read batch for consecutive IDs beginning immediately after its admitted offset. When a cleanup response fails, the next read reconciles durable retained bounds with that offset, using the next message ID to detect fully deleted history. An unread gap permanently faults that receiver's source with ; subsequent reads report the same failure. Size retention for the expected processing lag and recovery window, and use the reported partition and missing message IDs to investigate data loss. can fault one failing subscription while preserving the multicast partition record. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index e8356e04e46..3a7df2ef0e7 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -66,7 +66,8 @@ public class AdoNetStreamOptions /// Gets or sets the minimum amount of time that a stream record is retained after it is checkpointed. /// /// - /// Storage routines use whole seconds. Fractional values are rounded upward so configured retention is never shortened. + /// The interval must be at least one second. Storage routines round fractional seconds upward + /// to the next whole second, preserving at least the configured retention. /// public TimeSpan RetentionPeriod { get; set; } = TimeSpan.FromDays(1); @@ -76,14 +77,16 @@ public class AdoNetStreamOptions /// /// Stream records older than this value can be deleted even when they are newer than the checkpoint. /// Such deletions are reported by storage so that the receiver can emit gap diagnostics. - /// Fractional values are rounded upward to whole seconds. + /// When set, the interval must be at least one second. Fractional seconds are rounded upward to the next whole second. /// public TimeSpan? MaximumRetentionPeriod { get; set; } /// /// Gets or sets the interval between cleanup attempts for a partition. /// - /// Fractional values are rounded upward to whole seconds. + /// + /// The interval must be at least one second. Fractional seconds are rounded upward to the next whole second. + /// public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(1); /// diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index 312110269bd..8f593a854aa 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -54,9 +54,11 @@ The provider stores each queue as an immutable, ordered stream partition. Its st - `FaultOnDeliveryFailure`: optionally fault a failing subscription while preserving the shared partition records. - `MaxMessagesPerRead`: bound each ordered storage read. - `CheckpointPersistInterval`: throttle durable checkpoint updates. -- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). Fractional seconds round upward. -- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. Fractional seconds round upward. -- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. Fractional cleanup intervals round upward. +- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). +- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. +- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. + +`RetentionPeriod`, `CleanupInterval`, and `MaximumRetentionPeriod` (when set) must be at least one second. For valid intervals, storage routines round fractional seconds upward to the next whole second. The partitioned stream provider resumes strictly after the durable, ownership-fenced queue checkpoint and can redeliver records after a crash without skipping uncheckpointed data. The checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. Subscription starts attach to the live partition position. Partition state retains the next message identifier, so recovery detects a hard-retention gap even when the purge leaves no records. From bf9cd9ab12b2de68551364aafc2203dc1f750468 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 14:09:41 -0700 Subject: [PATCH 75/90] test(streaming): retain delivery progress coverage after token rebase --- .../LegacySequenceTokenRecoveryTests.cs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/LegacySequenceTokenRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/LegacySequenceTokenRecoveryTests.cs index 4504e97e6a1..ac1b1659256 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/LegacySequenceTokenRecoveryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/LegacySequenceTokenRecoveryTests.cs @@ -218,6 +218,10 @@ public static async Task VerifyRecovery( var handle = CreateHandle(streamId, observer, handshake); Add(100, 0); var cursor = GetCursor(cache, streamId, handle.GetSequenceToken()!.Token); + if (acknowledged) + { + cache.SetCursorDeliveredThrough(cursor, legacy); + } Assert.False(TryGetNextMessage(cache, cursor, out _)); foreach (var sequence in new[] { 101L, 102L }) @@ -237,7 +241,6 @@ public static async Task VerifyRecovery( Add(501, 0); cache.Refresh(cursor, createToken(499, 0)); - SkipAcknowledgedPosition(); Assert.True(TryGetNextMessage(cache, cursor, out var first)); Assert.Equal(500, first.SequenceToken.SequenceNumber); Assert.Equal(acknowledged ? 3 : 2, first.SequenceToken.EventIndex); @@ -249,19 +252,29 @@ public static async Task VerifyRecovery( Assert.Same(legacy, requested!.Token); Assert.Empty(observer.Tokens); cursor = GetCursor(cache, streamId, requested.Token); - SkipAcknowledgedPosition(); + if (acknowledged) + { + cache.SetCursorDeliveredThrough(cursor, legacy); + } + Assert.True(TryGetNextMessage(cache, cursor, out first)); Assert.Equal(acknowledged ? 3 : 2, first.SequenceToken.EventIndex); } - // An active refresh keeps its position. + // A failed delivery retries the same retained event, while an active refresh keeps its position. + cache.RecordDeliveryFailure(cursor); + Assert.True(TryGetNextMessage(cache, cursor, out var retried)); + Assert.Equal(first.SequenceToken, retried.SequenceToken); cache.Refresh(cursor, createToken(700, 0)); cache.Refresh(cursor, createToken(100, 0)); - Assert.Null(await handle.DeliverBatch(first, handshake)); + Assert.Null(await handle.DeliverBatch(retried, handshake)); + cache.RecordDeliverySuccess(cursor); + Assert.Equal(retried.SequenceToken, cache.GetSafeSequenceToken(cursor)); while (TryGetNextMessage(cache, cursor, out var batch)) { Assert.Null(await handle.DeliverBatch(batch, handle.GetSequenceToken())); + cache.RecordDeliverySuccess(cursor); } Assert.Equal( @@ -275,18 +288,6 @@ public static async Task VerifyRecovery( Assert.Equal(500, legacy.SequenceNumber); Assert.Equal(2, legacy.EventIndex); - void SkipAcknowledgedPosition() - { - if (!acknowledged) - { - return; - } - - Assert.True(TryGetNextMessage(cache, cursor, out var duplicate)); - Assert.Equal(0, EventSequenceTokenCompatibility.Compare(legacy, duplicate.SequenceToken)); - Assert.Equal(0, EventSequenceTokenCompatibility.Compare(duplicate.SequenceToken, legacy)); - } - void Add(long sequence, int index) { var now = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); From b9c152b8278c1575f81cc623b9ac7e2efbeb1ecd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 14:20:04 -0700 Subject: [PATCH 76/90] fix(streaming): acquire published ADO.NET adapter before cache access --- .../Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index 5f915a8a9b3..2bf84cf1cbd 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -132,7 +132,7 @@ public async Task GetDeliveryFailureHandler(QueueId queue public IQueueAdapterCache GetQueueAdapterCache() => this; public IQueueCache CreateQueueCache(QueueId queueId) - => ((_adapter as IQueueAdapterCache) ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) + => ((Volatile.Read(ref _adapter) as IQueueAdapterCache) ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) .CreateQueueCache(queueId); public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; From 5128d2ead403c1922f650e6e8994f743f758f4d2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 10 Sep 2026 14:30:55 -0700 Subject: [PATCH 77/90] fix(streaming): deprecate tokenless Event Hubs checkpointer factory --- .../Providers/Streams/EventHub/EventHubCheckpointer.cs | 2 ++ .../Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 2dbb15510a7..03873af6679 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -36,9 +36,11 @@ public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpoi } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) => Create(partition, CancellationToken.None); + /// public Task> Create( string partition, CancellationToken cancellationToken) diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index 4cbf5022adb..a506d441f63 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -391,6 +391,7 @@ public EventHubCheckpointerFactory(string providerName, Configuration.AzureTable public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } From cfa45e87b2460d65f8ca4533b367891ca1ffec4b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 14:21:14 -0700 Subject: [PATCH 78/90] fix(streaming): forward cancellation while loading ADO.NET queries --- .../Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index 2bf84cf1cbd..aaa3888d99b 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -60,7 +60,7 @@ async Task CoreAsync() // slow path - the member variable will only be set if the call succeeds var result = await RelationalOrleansQueries - .CreateInstance(_streamOptions.Invariant, _streamOptions.ConnectionString, _streamOptions.DataSource) + .CreateInstance(_streamOptions.Invariant, _streamOptions.ConnectionString, _streamOptions.DataSource, cancellationToken) .WaitAsync(_streamOptions.InitializationTimeout, cancellationToken); Volatile.Write(ref _queries, result); return result; From 16dd269d684c64f3ded976de712c4ba5e45a7ee6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 01:21:23 -0700 Subject: [PATCH 79/90] fix(streaming): close retained recovery review gaps --- .../AdoNetRecoverableStream.cs | 4 +- .../MySQL-Streaming.sql | 3 +- .../QueueAdapterReceiverRegistry.cs | 99 ++++++++++++++----- .../RecoverableStreamReceiverTests.cs | 44 +++++++++ 4 files changed, 125 insertions(+), 25 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs index a00b0c4d96f..7c3fe10542c 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -242,7 +242,9 @@ private async Task ReconcileRetainedBounds(CancellationToken cancellationToken) $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' is missing while reconciling retained history."); var earliestAvailablePosition = state.EarliestMessageId ?? state.NextMessageId; if (_readOffset < earliestAvailablePosition - 1 - || (state.NextMessageId > _readOffset + 1 && state.TailMessageId <= _readOffset)) + || (state.NextMessageId > _readOffset + 1 + && state.TailMessageId is { } tailMessageId + && tailMessageId <= _readOffset)) { throw RetainFailure( $"has a retention gap after admitted message {_readOffset}: " diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 9535eacaaae..6197af77821 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -501,7 +501,8 @@ BEGIN WHERE Eligible = FALSE; DELETE FROM OrleansStreamCleanupBatch - WHERE MessageId >= _FirstIneligibleMessageId; + WHERE _FirstIneligibleMessageId IS NOT NULL + AND MessageId >= _FirstIneligibleMessageId; SELECT COUNT(*), diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs index 3490c3d3c28..0b4a8dca209 100644 --- a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; using System.Threading; using Orleans.Streams; @@ -16,6 +15,8 @@ public sealed class QueueAdapterReceiverRegistry { private readonly ConcurrentDictionary> _receivers = new(); private readonly Func _factory; + private Action? _onRemoveAttempt; + private Action? _onRemoveLockAcquired; /// /// Initializes a new instance of the class. @@ -29,30 +30,64 @@ public QueueAdapterReceiverRegistry(Func factory) /// Gets the registered receiver instances. /// public IReadOnlyDictionary Receivers - => _receivers - .Where(pair => pair.Value.IsValueCreated) - .ToDictionary(pair => pair.Key, pair => pair.Value.Value); + { + get + { + var result = new Dictionary(); + foreach (var pair in _receivers) + { + if (!pair.Value.IsValueCreated) + { + continue; + } + + lock (pair.Value) + { + if (_receivers.TryGetValue(pair.Key, out var current) + && ReferenceEquals(current, pair.Value) + && pair.Value.IsValueCreated) + { + result[pair.Key] = pair.Value.Value; + } + } + } + + return result; + } + } /// /// Gets or creates the coordinator for a queue. /// public TReceiver GetOrCreate(QueueId queueId) { - var receiver = _receivers.GetOrAdd( - queueId, - static (id, factory) => new( - () => factory(id), - LazyThreadSafetyMode.ExecutionAndPublication), - _factory); - try - { - return receiver.Value; - } - catch + while (true) { - ((ICollection>>)_receivers) - .Remove(new(queueId, receiver)); - throw; + var receiver = _receivers.GetOrAdd( + queueId, + static (id, factory) => new( + () => factory(id), + LazyThreadSafetyMode.ExecutionAndPublication), + _factory); + lock (receiver) + { + if (!_receivers.TryGetValue(queueId, out var current) + || !ReferenceEquals(current, receiver)) + { + continue; + } + + try + { + return receiver.Value; + } + catch + { + ((ICollection>>)_receivers) + .Remove(new(queueId, receiver)); + throw; + } + } } } @@ -61,14 +96,32 @@ public TReceiver GetOrCreate(QueueId queueId) /// public bool Remove(QueueId queueId, TReceiver receiver) { - if (!_receivers.TryGetValue(queueId, out var registered) - || !registered.IsValueCreated - || !ReferenceEquals(registered.Value, receiver)) + if (!_receivers.TryGetValue(queueId, out var registered)) { return false; } - return ((ICollection>>)_receivers) - .Remove(new(queueId, registered)); + _onRemoveAttempt?.Invoke(); + lock (registered) + { + _onRemoveLockAcquired?.Invoke(); + if (!_receivers.TryGetValue(queueId, out var current) + || !ReferenceEquals(current, registered) + || !registered.IsValueCreated + || !ReferenceEquals(registered.Value, receiver)) + { + return false; + } + + return ((ICollection>>)_receivers) + .Remove(new(queueId, registered)); + } + } + + internal sealed class TestAccessor(QueueAdapterReceiverRegistry instance) + { + public Action? OnRemoveAttempt { set => instance._onRemoveAttempt = value; } + + public Action? OnRemoveLockAcquired { set => instance._onRemoveLockAcquired = value; } } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs index 543d854a0a0..2b7a16b66e9 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -1001,6 +1001,50 @@ public void Registry_RemoveAllowsFreshReceiverForReassignedQueue() Assert.Same(second, Assert.Single(registry.Receivers).Value); } + [Fact] + public async Task Registry_RemoveWaitsForInProgressCreation() + { + var queue = QueueId.GetQueueId("queue", 0, 0); + var receiverCreated = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var registry = new QueueAdapterReceiverRegistry(_ => + { + var receiver = new TestCombinedReceiver(); + receiverCreated.SetResult(receiver); + releaseFactory.Task.GetAwaiter().GetResult(); + return receiver; + }); + var removalAttempted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var removalLockAcquired = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var registryAccessor = new QueueAdapterReceiverRegistry.TestAccessor(registry) + { + OnRemoveAttempt = removalAttempted.SetResult, + OnRemoveLockAcquired = removalLockAcquired.SetResult, + }; + var lookup = Task.Run(() => registry.GetOrCreate(queue)); + var receiver = await receiverCreated.Task; + var removal = Task.Run(() => registry.Remove(queue, receiver)); + await removalAttempted.Task; + + try + { + Assert.False(removalLockAcquired.Task.IsCompleted); + } + finally + { + releaseFactory.TrySetResult(); + } + + Assert.Same(receiver, await lookup); + await removalLockAcquired.Task; + Assert.True(await removal); + Assert.Empty(registry.Receivers); + } + private static RecoverableStreamReceiver CreateReceiver( IRecoverableStreamSource source, IStreamQueueCheckpointer checkpointer) From 8c1ee101ab9e09f7ffcd764c248b6c3ceb0cdef2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 09:24:01 -0700 Subject: [PATCH 80/90] test(streaming): align dropped-client cleanup with shutdown draining --- .../PersistentStreamPullingAgentTests.cs | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index a46650fe8b9..77f15a48fe0 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -167,7 +167,7 @@ public async Task ReadFromQueue_DoesNotStartQueueReadAfterShutdownStarts() [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task DroppedClientCleanupWaitsForUnregistration() + public async Task DroppedClientCleanupCompletesBeforeShutdown() { var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var token = new EventSequenceTokenV2(1); @@ -204,23 +204,41 @@ public async Task DroppedClientCleanupWaitsForUnregistration() var consumerData = streamData.AddConsumer( subscriptionId, streamId, - new DroppedClientConsumer(), + new UnavailableConsumer(Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException()), filterData: null, now: DateTime.UtcNow); consumerData.IsRegistered = true; Assert.True(await accessor.DoHandshakeWithConsumer(consumerData, token)); var deliveryTask = accessor.RunConsumerCursor(consumerData); - await unregistrationStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - Assert.Equal(subscriptionId, unregisteredSubscriptionId); - Assert.Equal(streamId, unregisteredStreamId); - Assert.Equal(CancellationToken.None, unregistrationCancellationToken); - Assert.False(deliveryTask.IsCompleted); + Task? shutdownTask = null; + try + { + await unregistrationStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await deliveryTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(subscriptionId, unregisteredSubscriptionId); + Assert.Equal(streamId, unregisteredStreamId); + Assert.True(unregistrationCancellationToken.CanBeCanceled); + Assert.False(unregistrationCancellationToken.IsCancellationRequested); + Assert.False(streamData.Contains(subscriptionId)); + Assert.Null(consumerData.Cursor); + + shutdownTask = accessor.Shutdown(); + await accessor.GetPubSubCache().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(shutdownTask.IsCompleted); + Assert.False(unregistrationCancellationToken.IsCancellationRequested); + } + finally + { + unregistration.TrySetResult(true); + await deliveryTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + shutdownTask ??= accessor.Shutdown(); + await shutdownTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } - unregistration.TrySetResult(true); - await deliveryTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await accessor.Shutdown(); + Assert.True(unregistrationCancellationToken.IsCancellationRequested); + await pubSub.Received(1).UnregisterConsumer(subscriptionId, streamId, unregistrationCancellationToken); } [TestSuite("BVT")] @@ -1380,27 +1398,6 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } - private sealed class DroppedClientConsumer : IStreamConsumerExtension - { - public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - => throw new NotSupportedException(); - - public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - => throw new NotSupportedException(); - - public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) - { - throw (ClientNotAvailableException)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(ClientNotAvailableException)); - } - - public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) - => Task.FromResult(null); - } - private sealed class ImmediateRecordingConsumer : IStreamConsumerExtension { public List DeliveredBatches { get; } = []; From f744bc58a93ccaa386670a085efecd4090d07bc2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 13:29:10 -0700 Subject: [PATCH 81/90] fix(streaming): make producer-initiated retirement callback-safe --- .../Orleans.Streaming.csproj | 1 + src/Orleans.Streaming/OrleansContracts.txt | 1 + .../ClientStreamingProviderRuntime.cs | 8 +- .../Providers/IStreamProviderRuntime.cs | 14 +- .../Providers/SiloStreamProviderRuntime.cs | 8 +- .../PubSub/GrainBasedPubSubRuntime.cs | 9 +- .../PubSub/IPubSubRendezvousGrain.cs | 3 + .../PubSub/ImplicitStreamPubSub.cs | 9 +- .../PubSub/PubSubRendezvousGrain.cs | 26 +- .../PubSub/StreamPubSubImpl.cs | 15 +- ...RendezvousGrainTests.ProducerRetirement.cs | 414 ++++++++++++++++++ .../PubSubRendezvousGrainTests.cs | 2 +- .../StreamConsumerLifecycleTests.cs | 2 +- 13 files changed, 493 insertions(+), 19 deletions(-) create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs diff --git a/src/Orleans.Streaming/Orleans.Streaming.csproj b/src/Orleans.Streaming/Orleans.Streaming.csproj index 37bec7c82e8..098856468ed 100644 --- a/src/Orleans.Streaming/Orleans.Streaming.csproj +++ b/src/Orleans.Streaming/Orleans.Streaming.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Orleans.Streaming/OrleansContracts.txt b/src/Orleans.Streaming/OrleansContracts.txt index 015e76e9f14..a32fcf5617e 100644 --- a/src/Orleans.Streaming/OrleansContracts.txt +++ b/src/Orleans.Streaming/OrleansContracts.txt @@ -43,6 +43,7 @@ interface [GrainInterfaceType("Orleans.Streams.IPubSubRendezvousGrain")] Orleans 5E7E20BC: RegisterConsumer(Orleans.Runtime.GuidId, Orleans.Runtime.QualifiedStreamId, Orleans.Runtime.GrainId, string?, System.Threading.CancellationToken) -> Task B5FFB7F3: RegisterProducer(Orleans.Runtime.QualifiedStreamId, Orleans.Runtime.GrainId, Orleans.Runtime.MembershipVersion, System.Threading.CancellationToken) -> Task> 974334B6: UnregisterConsumer(Orleans.Runtime.GuidId, Orleans.Runtime.QualifiedStreamId, System.Threading.CancellationToken) -> Task + UnregisterConsumerFromProducer: UnregisterConsumerFromProducer(Orleans.Runtime.GuidId, Orleans.Runtime.QualifiedStreamId, Orleans.Runtime.GrainId, System.Threading.CancellationToken) -> Task C017B47D: UnregisterProducer(Orleans.Runtime.QualifiedStreamId, Orleans.Runtime.GrainId, System.Threading.CancellationToken) -> Task 20AA72BF: Validate(System.Threading.CancellationToken) -> Task diff --git a/src/Orleans.Streaming/Providers/ClientStreamingProviderRuntime.cs b/src/Orleans.Streaming/Providers/ClientStreamingProviderRuntime.cs index 674d56f4b0b..be839c87be2 100644 --- a/src/Orleans.Streaming/Providers/ClientStreamingProviderRuntime.cs +++ b/src/Orleans.Streaming/Providers/ClientStreamingProviderRuntime.cs @@ -9,9 +9,9 @@ namespace Orleans.Providers { internal class ClientStreamingProviderRuntime : IStreamProviderRuntime, ILifecycleParticipant { - private readonly IStreamPubSub grainBasedPubSub; - private readonly IStreamPubSub implicitPubSub; - private readonly IStreamPubSub combinedGrainBasedAndImplicitPubSub; + private readonly IStreamPubSubRuntime grainBasedPubSub; + private readonly IStreamPubSubRuntime implicitPubSub; + private readonly IStreamPubSubRuntime combinedGrainBasedAndImplicitPubSub; private StreamDirectory? streamDirectory; private readonly IInternalGrainFactory grainFactory; private readonly ImplicitStreamSubscriberTable implicitSubscriberTable; @@ -71,7 +71,7 @@ public string ExecutingEntityIdentity() return this.clientContext.GetOrSetExtension(newExtensionFunc); } - public IStreamPubSub? PubSub(StreamPubSubType pubSubType) + public IStreamPubSubRuntime? PubSub(StreamPubSubType pubSubType) { switch (pubSubType) { diff --git a/src/Orleans.Streaming/Providers/IStreamProviderRuntime.cs b/src/Orleans.Streaming/Providers/IStreamProviderRuntime.cs index 56b7c1d3579..beca5f4f335 100644 --- a/src/Orleans.Streaming/Providers/IStreamProviderRuntime.cs +++ b/src/Orleans.Streaming/Providers/IStreamProviderRuntime.cs @@ -28,7 +28,7 @@ internal interface IStreamProviderRuntime : IProviderRuntime /// A Pub Sub runtime interface. /// /// - IStreamPubSub? PubSub(StreamPubSubType pubSubType); + IStreamPubSubRuntime? PubSub(StreamPubSubType pubSubType); } /// @@ -44,6 +44,18 @@ Task InitializePullingAgents( CancellationToken cancellationToken); } + internal interface IStreamPubSubRuntime : IStreamPubSub + { + /// + /// Durably removes a subscription already detached by the requesting producer, and notifies the other producers. + /// + Task UnregisterConsumerFromProducer( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId producer, + CancellationToken cancellationToken); + } + /// /// Identifies the publish/subscribe system types which stream providers can use. /// diff --git a/src/Orleans.Streaming/Providers/SiloStreamProviderRuntime.cs b/src/Orleans.Streaming/Providers/SiloStreamProviderRuntime.cs index 924523cff93..288e3ba3e99 100644 --- a/src/Orleans.Streaming/Providers/SiloStreamProviderRuntime.cs +++ b/src/Orleans.Streaming/Providers/SiloStreamProviderRuntime.cs @@ -15,9 +15,9 @@ internal partial class SiloStreamProviderRuntime : ISiloSideStreamProviderRuntim { private readonly IConsistentRingProvider consistentRingProvider; private readonly InsideRuntimeClient runtimeClient; - private readonly IStreamPubSub grainBasedPubSub; - private readonly IStreamPubSub implictPubSub; - private readonly IStreamPubSub combinedGrainBasedAndImplicitPubSub; + private readonly IStreamPubSubRuntime grainBasedPubSub; + private readonly IStreamPubSubRuntime implictPubSub; + private readonly IStreamPubSubRuntime combinedGrainBasedAndImplicitPubSub; private readonly ILoggerFactory loggerFactory; private readonly ILocalSiloDetails siloDetails; private readonly IGrainContextAccessor grainContextAccessor; @@ -49,7 +49,7 @@ public SiloStreamProviderRuntime( this.combinedGrainBasedAndImplicitPubSub = new StreamPubSubImpl(this.grainBasedPubSub, tmp); } - public IStreamPubSub? PubSub(StreamPubSubType pubSubType) + public IStreamPubSubRuntime? PubSub(StreamPubSubType pubSubType) { switch (pubSubType) { diff --git a/src/Orleans.Streaming/PubSub/GrainBasedPubSubRuntime.cs b/src/Orleans.Streaming/PubSub/GrainBasedPubSubRuntime.cs index 71f41d5fcce..401cb3b6876 100644 --- a/src/Orleans.Streaming/PubSub/GrainBasedPubSubRuntime.cs +++ b/src/Orleans.Streaming/PubSub/GrainBasedPubSubRuntime.cs @@ -7,7 +7,7 @@ namespace Orleans.Streams { - internal class GrainBasedPubSubRuntime : IStreamPubSub + internal class GrainBasedPubSubRuntime : IStreamPubSubRuntime { private readonly IGrainFactory grainFactory; private readonly IClusterMembershipService? clusterMembershipService; @@ -73,6 +73,13 @@ public Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId return streamRendezvous.UnregisterConsumer(subscriptionId, streamId, cancellationToken); } + public Task UnregisterConsumerFromProducer( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId producer, + CancellationToken cancellationToken) + => GetRendezvousGrain(streamId).UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellationToken); + public Task ProducerCount(QualifiedStreamId streamId) => ProducerCount(streamId, CancellationToken.None); diff --git a/src/Orleans.Streaming/PubSub/IPubSubRendezvousGrain.cs b/src/Orleans.Streaming/PubSub/IPubSubRendezvousGrain.cs index ae5c5e43ba4..a6ada9e55e6 100644 --- a/src/Orleans.Streaming/PubSub/IPubSubRendezvousGrain.cs +++ b/src/Orleans.Streaming/PubSub/IPubSubRendezvousGrain.cs @@ -24,6 +24,9 @@ Task> RegisterProducer( [Alias("974334B6")] Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken = default); + [Alias("UnregisterConsumerFromProducer")] + Task UnregisterConsumerFromProducer(GuidId subscriptionId, QualifiedStreamId streamId, GrainId producer, CancellationToken cancellationToken = default); + [Alias("29B61035")] Task ProducerCount(QualifiedStreamId streamId, CancellationToken cancellationToken = default); diff --git a/src/Orleans.Streaming/PubSub/ImplicitStreamPubSub.cs b/src/Orleans.Streaming/PubSub/ImplicitStreamPubSub.cs index 8cde7635538..acf1acaec2e 100644 --- a/src/Orleans.Streaming/PubSub/ImplicitStreamPubSub.cs +++ b/src/Orleans.Streaming/PubSub/ImplicitStreamPubSub.cs @@ -8,7 +8,7 @@ namespace Orleans.Streams { - internal class ImplicitStreamPubSub : IStreamPubSub + internal class ImplicitStreamPubSub : IStreamPubSubRuntime { private readonly ImplicitStreamSubscriberTable implicitTable; @@ -76,6 +76,13 @@ public Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId return Task.CompletedTask; } + public Task UnregisterConsumerFromProducer( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId producer, + CancellationToken cancellationToken) + => UnregisterConsumer(subscriptionId, streamId, cancellationToken); + public Task ProducerCount(QualifiedStreamId streamId) => ProducerCount(streamId, CancellationToken.None); diff --git a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs index 0c8cbb4901c..49b290d1ecb 100644 --- a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs +++ b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs @@ -583,7 +583,21 @@ private void RecordRemovedProducers(IEnumerable? producers } } - public async Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken) + public Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken) + => UnregisterConsumerImpl(subscriptionId, streamId, null, cancellationToken); + + public Task UnregisterConsumerFromProducer( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId producer, + CancellationToken cancellationToken) + => UnregisterConsumerImpl(subscriptionId, streamId, producer, cancellationToken); + + private async Task UnregisterConsumerImpl( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId? detachedProducer, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var consumerState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId)); @@ -623,7 +637,8 @@ public async Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId st await NotifyProducersOfRemovedSubscription( subscriptionId, streamId, - numRemoved == 0 ? cancellationToken : CancellationToken.None); + numRemoved == 0 ? cancellationToken : CancellationToken.None, + detachedProducer); } if (_streamInstruments.PubSubConsumersTotal.Enabled) { @@ -770,7 +785,11 @@ public async Task FaultSubscription(GuidId subscriptionId, CancellationToken can } } - private async Task NotifyProducersOfRemovedSubscription(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken) + private async Task NotifyProducersOfRemovedSubscription( + GuidId subscriptionId, + QualifiedStreamId streamId, + CancellationToken cancellationToken, + GrainId? detachedProducer = null) { int numProducersBeforeNotify = State.Producers.Count; if (numProducersBeforeNotify > 0) @@ -780,6 +799,7 @@ private async Task NotifyProducersOfRemovedSubscription(GuidId subscriptionId, Q // Notify producers about unregistered consumer. var producers = State.Producers.ToList(); List tasks = producers + .Where(producer => !detachedProducer.HasValue || producer.Producer != detachedProducer.Value) .Select(producerState => ExecuteProducerTask(producerState, p => p.RemoveSubscriber(subscriptionId, streamId, cancellationToken))) .ToList(); await Task.WhenAll(tasks); diff --git a/src/Orleans.Streaming/PubSub/StreamPubSubImpl.cs b/src/Orleans.Streaming/PubSub/StreamPubSubImpl.cs index ed0d80e142f..57f5fc09cd4 100644 --- a/src/Orleans.Streaming/PubSub/StreamPubSubImpl.cs +++ b/src/Orleans.Streaming/PubSub/StreamPubSubImpl.cs @@ -8,12 +8,12 @@ namespace Orleans.Streams { - internal class StreamPubSubImpl : IStreamPubSub + internal class StreamPubSubImpl : IStreamPubSubRuntime { - private readonly IStreamPubSub explicitPubSub; + private readonly IStreamPubSubRuntime explicitPubSub; private readonly ImplicitStreamPubSub implicitPubSub; - public StreamPubSubImpl(IStreamPubSub explicitPubSub, ImplicitStreamPubSub implicitPubSub) + public StreamPubSubImpl(IStreamPubSubRuntime explicitPubSub, ImplicitStreamPubSub implicitPubSub) { if (explicitPubSub == null) { @@ -68,6 +68,15 @@ public Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId streamId : explicitPubSub.UnregisterConsumer(subscriptionId, streamId, cancellationToken); } + public Task UnregisterConsumerFromProducer( + GuidId subscriptionId, + QualifiedStreamId streamId, + GrainId producer, + CancellationToken cancellationToken) + => implicitPubSub.IsImplicitSubscriber(subscriptionId, streamId) + ? implicitPubSub.UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellationToken) + : explicitPubSub.UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellationToken); + public Task ProducerCount(QualifiedStreamId streamId) => ProducerCount(streamId, CancellationToken.None); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs new file mode 100644 index 00000000000..5d6263291d3 --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs @@ -0,0 +1,414 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Runtime; +using Orleans.Streams; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +public partial class PubSubRendezvousGrainTests +{ + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_PersistsRemovalAndNotifiesOnlyOtherProducer_OrdinaryRemovalNotifiesBoth() + { + await using var scenario = await RetirementScenario.Create(fixture); + var cancellationToken = TestContext.Current.CancellationToken; + var ordinarySubscription = GuidId.GetGuidId(Guid.NewGuid()); + await scenario.PubSub.RegisterConsumer(ordinarySubscription, scenario.StreamId, default, null, cancellationToken); + + await AwaitRetirementPhase(scenario.Retire(cancellationToken), "producer-originated removal"); + + await scenario.AssertDurableConsumers(ordinarySubscription); + Assert.Empty(scenario.Requester.RemovalAttempts); + AssertRemoval(scenario.Other, scenario.SubscriptionId, scenario.StreamId); + + // Validate reloaded durable state above. Neither producer is re-registered here: + // ordinary removal must still reach the exact requester, not merely retain a count. + await AwaitRetirementPhase( + scenario.PubSub.UnregisterConsumer(ordinarySubscription, scenario.StreamId, cancellationToken), + "ordinary removal notifying both retained producers"); + + AssertRemoval(scenario.Requester, ordinarySubscription, scenario.StreamId); + Assert.Equal( + new[] { (scenario.SubscriptionId, scenario.StreamId), (ordinarySubscription, scenario.StreamId) }, + scenario.Other.Removals.ToArray()); + await scenario.AssertDurableConsumers(); + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_WriteFailurePreservesOriginalErrorAndSkipsNotifications_RetryPersistsRemoval() + { + await using var scenario = await RetirementScenario.Create(fixture); + var cancellationToken = TestContext.Current.CancellationToken; + var faultGrain = fixture.GrainFactory.GetGrain(nameof(PubSubRendezvousGrain)); + const string failureMessage = "producer retirement durable write failed"; + using var observer = GrainDiagnosticObserver.Create(fixture.HostedCluster); + var deactivated = observer.WaitForGrainDeactivatedAsync(scenario.PubSub.GetGrainId()); + await faultGrain.AddFaultOnWrite(scenario.PubSub.GetGrainId(), new ApplicationException(failureMessage)); + + var exception = await Assert.ThrowsAsync( + () => AwaitRetirementPhase(scenario.Retire(cancellationToken), "injected retirement write failure")); + + Assert.Equal(failureMessage, Assert.IsType(exception.InnerException).Message); + Assert.Empty(scenario.Requester.RemovalAttempts); + Assert.Empty(scenario.Other.RemovalAttempts); + Assert.Empty(scenario.Other.Removals); + + // Do not inspect the failed activation's mutated in-memory state. Wait for its + // actual deactivation, then prove the next activation reads the original consumer. + await AwaitRetirementPhase(deactivated, $"failed rendezvous {scenario.PubSub.GetGrainId()} deactivation"); + var consumer = Assert.Single(await scenario.PubSub.DiagGetConsumers(scenario.StreamId, cancellationToken)); + Assert.Equal(scenario.SubscriptionId, consumer.SubscriptionId); + Assert.Equal(scenario.StreamId, consumer.Stream); + Assert.Equal(2, await scenario.PubSub.ProducerCount(scenario.StreamId, cancellationToken)); + + await AwaitRetirementPhase(scenario.Retire(cancellationToken), "retirement retry after storage failure"); + + await scenario.AssertDurableConsumers(); + Assert.Empty(scenario.Requester.RemovalAttempts); + AssertRemoval(scenario.Other, scenario.SubscriptionId, scenario.StreamId); + Assert.Single(scenario.Other.RemovalAttempts); + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_AbsentConsumerRetriesFailedAndSuccessfulNotificationsWithoutCallingRequester() + { + await using var scenario = await RetirementScenario.Create(fixture); + var cancellationToken = TestContext.Current.CancellationToken; + const string failureMessage = "other producer removal failed"; + scenario.Other.NextRemovalFailure = new ApplicationException(failureMessage); + using var observer = GrainDiagnosticObserver.Create(fixture.HostedCluster); + var deactivated = observer.WaitForGrainDeactivatedAsync(scenario.PubSub.GetGrainId()); + + var exception = await Assert.ThrowsAsync( + () => AwaitRetirementPhase(scenario.Retire(cancellationToken), "injected producer notification failure")); + + Assert.Equal(failureMessage, exception.Message); + Assert.Empty(scenario.Requester.RemovalAttempts); + Assert.Empty(scenario.Other.Removals); + Assert.Equal((scenario.SubscriptionId, scenario.StreamId), Assert.Single(scenario.Other.RemovalAttempts)); + await AwaitRetirementPhase(deactivated, $"rendezvous {scenario.PubSub.GetGrainId()} notification-failure deactivation"); + await scenario.AssertDurableConsumers(); + + // The durable subscription is already absent, but retry must repair notification. + await AwaitRetirementPhase(scenario.Retire(cancellationToken), "absent-consumer notification retry"); + AssertRemoval(scenario.Other, scenario.SubscriptionId, scenario.StreamId); + await AwaitRetirementPhase(scenario.Retire(cancellationToken), "repeat removal after successful retry"); + + Assert.Empty(scenario.Requester.RemovalAttempts); + Assert.Equal( + Enumerable.Repeat((scenario.SubscriptionId, scenario.StreamId), 3), + scenario.Other.RemovalAttempts.ToArray()); + Assert.Equal( + Enumerable.Repeat((scenario.SubscriptionId, scenario.StreamId), 2), + scenario.Other.Removals.ToArray()); + await scenario.AssertDurableConsumers(); + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_OnDeactivateAsyncCompletesWithoutCallbackCycleAndRetainsStableProducer() + { + await using var scenario = await RetirementScenario.Create(fixture, holdOtherNotification: true); + var cancellationToken = TestContext.Current.CancellationToken; + await AwaitRetirementPhase( + scenario.RequesterReference.RetireOnDeactivation(scenario.SubscriptionId, scenario.StreamId), + "arming producer retirement during deactivation"); + var context = scenario.RequesterContext; + var deactivated = context.Deactivated; + var retirementEntered = scenario.Requester.RetirementEntered.Task; + var otherNotified = scenario.Other.RemovalEntered.Task; + + try + { + context.Deactivate(new DeactivationReason(DeactivationReasonCode.ApplicationRequested, "pubsub retirement regression"), cancellationToken); + await AwaitRetirementPhase(retirementEntered, $"producer {context.GrainId} OnDeactivateAsync entry"); + Assert.False(deactivated.IsCompleted); + Assert.Equal(1, await scenario.PubSub.ConsumerCount(scenario.StreamId, cancellationToken)); + Assert.Equal(2, await scenario.PubSub.ProducerCount(scenario.StreamId, cancellationToken)); + + scenario.Requester.ContinueRetirement.TrySetResult(); + await AwaitRetirementPhase(otherNotified, $"stable producer {scenario.OtherContext.GrainId} notification"); + Assert.Equal((scenario.SubscriptionId, scenario.StreamId), Assert.Single(scenario.Other.RemovalAttempts)); + Assert.Empty(scenario.Requester.RemovalAttempts); + Assert.False(deactivated.IsCompleted); + Assert.False(scenario.Requester.RetirementSucceeded); + + scenario.Other.ContinueRemoval.TrySetResult(); + await AwaitRetirementPhase(deactivated, $"exact producer activation {context.Address} deactivation"); + + // The runtime can finish deactivation even if OnDeactivateAsync throws. + // Require successful retirement as well as completion of this exact context. + Assert.Null(scenario.Requester.RetirementError); + Assert.True(scenario.Requester.RetirementSucceeded); + Assert.Empty(scenario.Requester.RemovalAttempts); + AssertRemoval(scenario.Other, scenario.SubscriptionId, scenario.StreamId); + Assert.False(scenario.OtherContext.Deactivated.IsCompleted); + await scenario.AssertDurableConsumers(); + } + finally + { + scenario.ReleaseBarriers(); + } + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_BuiltinExplicitFacadesForwardExactIdsAndCancellationToken() + { + var grainFactory = Substitute.For(); + var rendezvous = Substitute.For(); + var streamId = new QualifiedStreamId("RetirementProvider", StreamId.Create("RetirementNamespace", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var producer = fixture.GrainFactory.GetGrain(Guid.NewGuid()).GetGrainId(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + grainFactory.GetGrain(streamId.ToString(), null).Returns(rendezvous); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + rendezvous.UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellation.Token).Returns(completion.Task); + var runtime = new GrainBasedPubSubRuntime(grainFactory); + var implicitPubSub = new ImplicitStreamPubSub(fixture.HostedCluster.GetSiloServiceProvider().GetRequiredService()); + var facade = new StreamPubSubImpl(runtime, implicitPubSub); + + try + { + var removal = facade.UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellation.Token); + Assert.Same(completion.Task, removal); + Assert.False(removal.IsCompleted); + completion.SetResult(); + await AwaitRetirementPhase(removal, "built-in explicit retirement forwarding"); + + _ = grainFactory.Received(1).GetGrain(streamId.ToString(), null); + await rendezvous.Received(1).UnregisterConsumerFromProducer(subscriptionId, streamId, producer, cancellation.Token); + await rendezvous.DidNotReceiveWithAnyArgs().UnregisterConsumer(default!, default, Arg.Any()); + } + finally + { + completion.TrySetResult(); + } + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task ProducerRetirement_ImplicitFacadeKeepsValidationAndCancellationWithoutCallingExplicitPubSub() + { + var explicitPubSub = Substitute.For(); + var implicitPubSub = new ImplicitStreamPubSub(fixture.HostedCluster.GetSiloServiceProvider().GetRequiredService()); + var facade = new StreamPubSubImpl(explicitPubSub, implicitPubSub); + var streamId = new QualifiedStreamId("RetirementProvider", StreamId.Create("RetirementNamespace", Guid.NewGuid())); + var producer = fixture.GrainFactory.GetGrain(Guid.NewGuid()).GetGrainId(); + var implicitId = GuidId.GetGuidId(SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid())); + var explicitId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + await facade.UnregisterConsumerFromProducer(implicitId, streamId, producer, cancellation.Token); + var invalid = await Assert.ThrowsAsync( + () => implicitPubSub.UnregisterConsumerFromProducer(explicitId, streamId, producer, cancellation.Token)); + Assert.Equal(streamId.ToString(), invalid.ParamName); + Assert.Contains("does not support explicit subscriptions", invalid.Message); + + cancellation.Cancel(); + var canceled = await Assert.ThrowsAsync( + () => facade.UnregisterConsumerFromProducer(implicitId, streamId, producer, cancellation.Token)); + Assert.Equal(cancellation.Token, canceled.CancellationToken); + Assert.Empty(explicitPubSub.ReceivedCalls()); + } + + private static void AssertRemoval(PubSubRetirementTestProducer producer, GuidId subscriptionId, QualifiedStreamId streamId) + => Assert.Equal((subscriptionId, streamId), Assert.Single(producer.Removals)); + + private static async Task AwaitRetirementPhase(Task task, string phase) + { + try + { + await task.WaitAsync(TimeSpan.FromSeconds(15), TestContext.Current.CancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException($"Pubsub producer retirement timed out waiting for {phase}.", exception); + } + } + + private sealed class RetirementScenario : IAsyncDisposable + { + public QualifiedStreamId StreamId { get; } = new("RetirementProvider", Orleans.Runtime.StreamId.Create("RetirementNamespace", Guid.NewGuid())); + public GuidId SubscriptionId { get; } = GuidId.GetGuidId(Guid.NewGuid()); + public IPubSubRendezvousGrain PubSub { get; } + public IPubSubRetirementTestProducer RequesterReference { get; } + private IPubSubRetirementTestProducer OtherReference { get; } + public IGrainContext RequesterContext { get; private set; } = null!; + public IGrainContext OtherContext { get; private set; } = null!; + public PubSubRetirementTestProducer Requester { get; private set; } = null!; + public PubSubRetirementTestProducer Other { get; private set; } = null!; + + private RetirementScenario(Fixture fixture) + { + PubSub = fixture.GrainFactory.GetGrain(StreamId.ToString()); + RequesterReference = fixture.GrainFactory.GetGrain(Guid.NewGuid()); + OtherReference = fixture.GrainFactory.GetGrain(Guid.NewGuid()); + } + + public static async Task Create(Fixture fixture, bool holdOtherNotification = false) + { + var result = new RetirementScenario(fixture); + using var observer = GrainDiagnosticObserver.Create(fixture.HostedCluster); + var requesterActivated = observer.WaitForGrainActivatedAsync(result.RequesterReference.GetGrainId()); + var otherActivated = observer.WaitForGrainActivatedAsync(result.OtherReference.GetGrainId()); + await AwaitRetirementPhase( + Task.WhenAll(result.RequesterReference.Activate(), result.OtherReference.Activate()), "both producer activations"); + await AwaitRetirementPhase(Task.WhenAll(requesterActivated, otherActivated), "producer activation diagnostics"); + result.RequesterContext = (await requesterActivated).GrainContext; + result.OtherContext = (await otherActivated).GrainContext; + result.Requester = Assert.IsType(result.RequesterContext.GrainInstance); + result.Other = Assert.IsType(result.OtherContext.GrainInstance); + if (holdOtherNotification) + { + result.Other.ContinueRemoval = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + try + { + var cancellationToken = TestContext.Current.CancellationToken; + // Register before producers so setup cannot be confused with removal callbacks. + await result.PubSub.RegisterConsumer(result.SubscriptionId, result.StreamId, default, null, cancellationToken); + var requesterSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.RequesterContext.GrainId, cancellationToken); + var otherSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.OtherContext.GrainId, cancellationToken); + Assert.Equal(result.SubscriptionId, Assert.Single(requesterSubscriptions).SubscriptionId); + Assert.Equal(result.SubscriptionId, Assert.Single(otherSubscriptions).SubscriptionId); + return result; + } + catch + { + await result.DisposeAsync(); + throw; + } + } + + public Task Retire(CancellationToken cancellationToken) + => PubSub.UnregisterConsumerFromProducer(SubscriptionId, StreamId, RequesterContext.GrainId, cancellationToken); + + public async Task AssertDurableConsumers(params GuidId[] subscriptionIds) + { + var cancellationToken = TestContext.Current.CancellationToken; + // Validate reads storage and compares complete producer/consumer sets with memory. + await PubSub.Validate(cancellationToken); + var consumers = await PubSub.DiagGetConsumers(StreamId, cancellationToken); + Assert.Equal(subscriptionIds, consumers.Select(consumer => consumer.SubscriptionId)); + Assert.All(consumers, consumer => Assert.Equal(StreamId, consumer.Stream)); + Assert.Equal(2, await PubSub.ProducerCount(StreamId, cancellationToken)); + } + + public void ReleaseBarriers() + { + foreach (var producer in new[] { Requester, Other }) + { + producer.ContinueRetirement.TrySetResult(); + producer.ContinueRemoval.TrySetResult(); + producer.AbortRetirement.Cancel(); + } + } + + public async ValueTask DisposeAsync() + { + ReleaseBarriers(); + try + { + var cancellationToken = TestContext.Current.CancellationToken; + await AwaitRetirementPhase(PubSub.UnregisterProducer(StreamId, RequesterContext.GrainId, cancellationToken), "requester registration cleanup"); + await AwaitRetirementPhase(PubSub.UnregisterProducer(StreamId, OtherContext.GrainId, cancellationToken), "other producer registration cleanup"); + foreach (var consumer in await PubSub.DiagGetConsumers(StreamId, cancellationToken)) + { + await AwaitRetirementPhase(PubSub.UnregisterConsumer(consumer.SubscriptionId, StreamId, cancellationToken), "remaining consumer cleanup"); + } + await AwaitRetirementPhase(PubSub.UnregisterConsumer(SubscriptionId, StreamId, cancellationToken), "subscription cleanup"); + } + finally + { + var reason = new DeactivationReason(DeactivationReasonCode.ApplicationRequested, "pubsub retirement test cleanup"); + RequesterContext.Deactivate(reason); + OtherContext.Deactivate(reason); + await AwaitRetirementPhase(Task.WhenAll(RequesterContext.Deactivated, OtherContext.Deactivated), "producer activation cleanup"); + Requester.AbortRetirement.Dispose(); + Other.AbortRetirement.Dispose(); + } + } + } +} + +public interface IPubSubRetirementTestProducer : IGrainWithGuidKey +{ + Task Activate(); + Task RetireOnDeactivation(GuidId subscriptionId, QualifiedStreamId streamId); +} + +// Real activation, not a local fake: the extension's AlwaysInterleave methods cannot +// be dispatched back into this activation while OnDeactivateAsync awaits retirement. +public sealed class PubSubRetirementTestProducer : Grain, IPubSubRetirementTestProducer, IStreamProducerExtension +{ + private GuidId? _retiringSubscription; + private QualifiedStreamId _retiringStream; + internal ConcurrentQueue<(GuidId, QualifiedStreamId)> RemovalAttempts { get; } = new(); + internal ConcurrentQueue<(GuidId, QualifiedStreamId)> Removals { get; } = new(); + internal TaskCompletionSource RemovalEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource ContinueRemoval { get; set; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource RetirementEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource ContinueRetirement { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal CancellationTokenSource AbortRetirement { get; } = new(); + internal Exception? NextRemovalFailure; + internal Exception? RetirementError { get; private set; } + internal bool RetirementSucceeded { get; private set; } + + // A replacement activation must not inherit a test barrier if a regressed callback + // is rerouted here after cleanup cancels the original activation's retirement wait. + public PubSubRetirementTestProducer() => ContinueRemoval.SetResult(); + + public Task Activate() => Task.CompletedTask; + + public Task RetireOnDeactivation(GuidId subscriptionId, QualifiedStreamId streamId) + { + _retiringSubscription = subscriptionId; + _retiringStream = streamId; + return Task.CompletedTask; + } + + public Task AddSubscriber(GuidId subscriptionId, QualifiedStreamId streamId, GrainId streamConsumer, string? filterData, CancellationToken cancellationToken) + => Task.CompletedTask; + + public async Task RemoveSubscriber(GuidId subscriptionId, QualifiedStreamId streamId, CancellationToken cancellationToken) + { + RemovalAttempts.Enqueue((subscriptionId, streamId)); + RemovalEntered.TrySetResult(); + await ContinueRemoval.Task.WaitAsync(cancellationToken); + if (Interlocked.Exchange(ref NextRemovalFailure, null) is { } exception) + { + throw exception; + } + + Removals.Enqueue((subscriptionId, streamId)); + } + + public override async Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) + { + if (_retiringSubscription is not { } subscriptionId) + { + return; + } + + RetirementEntered.TrySetResult(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, AbortRetirement.Token); + try + { + await ContinueRetirement.Task.WaitAsync(cancellation.Token); + var pubSub = GrainFactory.GetGrain(_retiringStream.ToString()); + // The local wait is cancelable by test cleanup even if a regressed RPC is + // blocked in a callback cycle, so a failed test does not wedge its fixture. + await pubSub.UnregisterConsumerFromProducer(subscriptionId, _retiringStream, GrainContext.GrainId, cancellation.Token) + .WaitAsync(cancellation.Token); + RetirementSucceeded = true; + } + catch (Exception exception) + { + RetirementError = exception; + throw; + } + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs index 05a67d5f212..6028fb194b0 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs @@ -16,7 +16,7 @@ namespace UnitTests.StreamingTests [TestSuite("BVT")] [TestProvider("None")] [TestArea("Runtime")] - public class PubSubRendezvousGrainTests : OrleansTestingBase, IClassFixture + public partial class PubSubRendezvousGrainTests : OrleansTestingBase, IClassFixture { private readonly Fixture fixture; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/StreamConsumerLifecycleTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/StreamConsumerLifecycleTests.cs index 36ecd5a7665..6c8cc762a79 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/StreamConsumerLifecycleTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/StreamConsumerLifecycleTests.cs @@ -427,7 +427,7 @@ public string ExecutingEntityIdentity() } public StreamDirectory GetStreamDirectory() => throw new NotSupportedException(); - public IStreamPubSub? PubSub(StreamPubSubType pubSubType) => throw new NotSupportedException(); + public IStreamPubSubRuntime? PubSub(StreamPubSubType pubSubType) => throw new NotSupportedException(); public void FailNextObserverRegistration(Exception exception) => nextObserverRegistrationFailure = exception; From 315d80d4208c9caba114b667b58da14c785a04a8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 13:29:11 -0700 Subject: [PATCH 82/90] refactor(streaming): gate processing and retirement shutdown --- .../streams-implementation/index.md | 12 +- .../PersistentStreamPullingAgent.cs | 198 +++++++------- .../PersistentStreamPullingManager.cs | 4 +- ...sistentStreamPullingAgentAdmissionTests.cs | 247 ++++++++++++++++++ ...stentStreamPullingAgentDiagnosticsTests.cs | 62 +++-- ...rsistentStreamPullingAgentRecoveryTests.cs | 2 +- .../PersistentStreamPullingAgentTests.cs | 118 +++++---- .../TaskSchedulerSelectionTests.cs | 2 +- 8 files changed, 456 insertions(+), 189 deletions(-) create mode 100644 test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index dd2bd25d67e..5ddaf32140f 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -75,11 +75,9 @@ Each `PersistentStreamPullingAgent` is a system target with single-threaded Orle The default maximum adapter batch-container batch size is 1 and the empty-poll period is 100 ms. These defaults are runtime behavior, not a universal throughput recommendation. -### Shutdown and queue handoff +### Shutdown and admitted work -When an agent stops, it closes admission for new background work and stops its polling timer. It waits for receiver initialization, the active queue pump, and accepted producer registrations, subscription handshakes, and deliveries to finish. Accepted work completes its token bookkeeping and releases registration pins and batch protection while the cache and receiver remain available. Outstanding calls retain their existing messaging timeouts and retry limits while accepted work drains. - -The agent then reports final delivery progress to the cache, disposes subscription cursors, and shuts down the receiver so provider-specific checkpoint flushing observes the completed progress. Registrations pending when shutdown starts keep the existing checkpoint, since their subscriber positions are still uncertain. Producer unregistration follows receiver cleanup. When the manager reuses an agent for a reassigned queue, initialization waits for that full cleanup and opens admission for the new run. +Each pulling-agent run owns admission for queue reads, producer registrations, subscription handshakes, and consumer delivery. Shutdown closes processing admission, stops polling, cancels processing waits, and drains admitted work and actual receiver initialization. Accepted work completes its token bookkeeping and releases registration pins and batch protection while the cache and receiver remain available. Repeated shutdown calls share the same completion. Initialization waits for the prior run's full cleanup before opening fresh admission and cancellation scopes. Explicit subscription notifications receive an immediate acknowledgement while the agent tracks their asynchronous handshake through completion. This lets the subscribing consumer finish its current call and respond to the handshake. @@ -87,7 +85,11 @@ A completed handshake establishes the subscription's current cursor and replay p A failed re-handshake leaves the subscription's position uncertain even when it was previously registered. The agent retains the stream entry across idle cleanup and keeps the existing checkpoint until a successful handshake reconciles that position. -Subscription removal revokes in-flight handshake and delivery ownership. A terminal pub-sub action issued under valid ownership completes cleanup for that subscription identity, including when a cursor reconciliation overlaps its persistence. +Subscription removal revokes in-flight handshake and delivery ownership. A terminal pub-sub action issued under valid ownership completes cleanup for that subscription identity, including when a cursor reconciliation overlaps its persistence. Once fault persistence starts, it uses the cleanup lifetime so shutdown drains the durable operation. + +An unavailable client is detached locally immediately. Durable subscription retirement has its own admission and cancellation lifetime, allowing a finishing delivery to start cleanup after processing admission closes. Once processing drains, shutdown closes retirement admission and waits for persistence and notification retries to finish. Producer-initiated retirement persists the removal and notifies the other producers; the requesting producer has already detached that subscription. Ordinary consumer unregistration notifies every registered producer. + +The final delivery-progress scan preserves the checkpoint barrier for registrations and handshakes which were pending when shutdown began. Cancellation and failed registration retain the prior safe position. The agent disposes subscription cursors before receiver shutdown flushes the safe checkpoint and releases its resources. Producer unregistration follows receiver cleanup. Operation failures remain observable through their lifecycle outcomes and correlated diagnostics. ## Cache and cursor invariants diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 8e0cc7f97a0..66ab91e023a 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -3,6 +3,7 @@ using System.Diagnostics.Metrics; using System.IO; using System.Linq; +using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -26,7 +27,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly IBackoffProvider deliveryBackoffProvider; private readonly IBackoffProvider queueReaderBackoffProvider; private readonly string streamProviderName; - private readonly IStreamPubSub pubSub; + private readonly IStreamPubSubRuntime pubSub; private readonly IStreamFilter streamFilter; private readonly Dictionary pubSubCache; private readonly StreamPullingAgentOptions options; @@ -36,9 +37,12 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly IStreamFailureHandler streamFailureHandler; private readonly StreamInstruments? _streamInstruments; private readonly TimeProvider _timeProvider; - private readonly CancellationTokenSource _shutdownCancellation = new(); - private readonly HashSet _activeConsumerCursorTasks = []; - private readonly HashSet _pendingUnavailableConsumerUnregistrations = []; + private AdmissionGate _workAdmission = new(); + private CancellationTokenSource? _shutdownCancellation; + private CancellationToken ShutdownToken => _shutdownCancellation?.Token ?? CancellationToken.None; + private CancellationTokenSource _retirementCancellation = new(); + private AdmissionGate _retirementAdmission = new(); + private ExceptionDispatchInfo? _retirementFailure; internal readonly QueueId QueueId; private int numMessages; @@ -49,9 +53,8 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private ITimer? deliveryProgressTimer; private Task? receiverInitTask; - private Task _activePumpTask = Task.CompletedTask; - private AdmissionGate _workAdmission = new(); private Task? _shutdownTask; + private Task _activePumpTask = Task.CompletedTask; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -72,7 +75,7 @@ internal interface ITestAccessor internal PersistentStreamPullingAgent( SystemTargetGrainId id, string strProviderName, - IStreamPubSub streamPubSub, + IStreamPubSubRuntime streamPubSub, IStreamFilter streamFilter, QueueId queueId, StreamPullingAgentOptions options, @@ -130,7 +133,7 @@ Task ITestAccessor.ReadFromQueueWithCancellation( Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now) => this.RunOrQueueTaskResult(() => { - RegisterStream(streamId, firstToken, now, CancellationToken.None); + RegisterStream(streamId, firstToken, now, ShutdownToken); if (pubSubCache.TryGetValue(streamId, out var streamData) && streamData.RegistrationTask is { } registrationTask) { @@ -147,7 +150,7 @@ IQueueCacheCursor ITestAccessor.GetRecoveryCursor(StreamConsumerData consumerDat => GetRecoveryCursor(consumerData); Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) - => this.RunOrQueueTask(() => TrackConsumerCursor(consumerData)); + => this.RunOrQueueTask(() => RunConsumerCursor(consumerData, ShutdownToken)); Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); @@ -179,10 +182,15 @@ public async Task Initialize(CancellationToken cancellationToken) { await shutdownTask.WaitAsync(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - _shutdownTask = null; - _workAdmission = new(); } + _workAdmission = new AdmissionGate(); + _shutdownCancellation = new CancellationTokenSource(); + _retirementCancellation.Dispose(); + _retirementCancellation = new CancellationTokenSource(); + _retirementAdmission = new AdmissionGate(); + _retirementFailure = null; + _shutdownTask = null; LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; @@ -261,15 +269,15 @@ async Task InitializeReceiver() public Task Shutdown(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - return _shutdownTask ??= ShutdownCore(cancellationToken); + return _shutdownTask ??= ShutdownCore(); } - private async Task ShutdownCore(CancellationToken cancellationToken) + private async Task ShutdownCore() { + var workDrained = _workAdmission.CloseAsync(); // Stop pulling from queues that are not in my range anymore. LogInfoShutdown(GetType().Name, new(QueueId)); - var drainTask = _workAdmission.CloseAsync(); var asyncTimer = timer; timer = null; var localDeliveryProgressTimer = deliveryProgressTimer; @@ -281,8 +289,10 @@ private async Task ShutdownCore(CancellationToken cancellationToken) StreamingEvents.EmitPullingAgentStopped(streamProviderName, Silo, QueueId); } - // Pending registrations leave subscriber progress uncertain, even if they exit during the drain. - var hasPendingRegistrations = pubSubCache.Values.Any(static stream => stream.RegistrationTask is { IsCompleted: false }); + // Canceled registrations can remove themselves from the cache before producer cleanup. + var streams = pubSubCache.ToArray(); + var hasPendingRegistrations = streams.Any(static entry => entry.Value.RegistrationTask is not null); + _shutdownCancellation?.Cancel(); Task? localReceiverInitTask = receiverInitTask; if (localReceiverInitTask != null) @@ -291,18 +301,28 @@ private async Task ShutdownCore(CancellationToken cancellationToken) receiverInitTask = null; } - await _activePumpTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); - await drainTask; + await workDrained; + + try + { + await _retirementAdmission.CloseAsync(); + } + finally + { + _retirementCancellation.Cancel(); + _retirementCancellation.Dispose(); + } - // All accepted work has finished progress bookkeeping and released its batch/registration pins. + // Registrations drained during shutdown can exit before discovering subscribers. + // Preserve their checkpoint barrier even after their tasks have completed. if (!hasPendingRegistrations) { NotifyDeliveryProgress(); } - foreach (var streamData in pubSubCache.Values) + foreach (var stream in streams) { - streamData.DisposeAll(logger); + stream.Value.DisposeAll(logger); } this.queueCache = null; @@ -313,7 +333,7 @@ private async Task ShutdownCore(CancellationToken cancellationToken) this.receiver = null; if (localReceiver != null) { - var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(this.options.InitQueueTimeout, cancellationToken)); + var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(this.options.InitQueueTimeout, CancellationToken.None)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07, $"QueueAdapterReceiver {QueueId} failed to Shutdown."); await task; @@ -325,24 +345,12 @@ private async Task ShutdownCore(CancellationToken cancellationToken) // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } - try - { - if (_pendingUnavailableConsumerUnregistrations.Count > 0) - { - await Task.WhenAll(_pendingUnavailableConsumerUnregistrations).WaitAsync(cancellationToken); - } - } - finally - { - _shutdownCancellation.Cancel(); - } - var unregisterTasks = new List(); - foreach (var tuple in pubSubCache) + foreach (var tuple in streams) { var streamId = tuple.Key; LogInfoUnregisterProducer(streamId); - unregisterTasks.Add(pubSub.UnregisterProducer(streamId, GrainId, cancellationToken)); + unregisterTasks.Add(pubSub.UnregisterProducer(streamId, GrainId, CancellationToken.None)); } try @@ -354,6 +362,9 @@ private async Task ShutdownCore(CancellationToken cancellationToken) LogWarningUnregisterProducer(exc); } pubSubCache.Clear(); + _shutdownCancellation?.Dispose(); + _shutdownCancellation = null; + _retirementFailure?.Throw(); } public Task AddSubscriber( @@ -413,7 +424,7 @@ private async Task AddSubscriber_Impl( data.IsRegistered = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); if (data.State == StreamConsumerDataState.Inactive) - TrackConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + RunConsumerCursor(data, ShutdownToken).Ignore(); } } finally @@ -456,14 +467,14 @@ private async Task DoHandshakeWithConsumer( try { requestedHandshakeToken = await AsyncExecutorWithRetries.ExecuteWithRetries( - i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId), + i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId, ShutdownToken).WaitAsync(ShutdownToken), AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException (exception, i) => exception is not ClientNotAvailableException && !IsShutdown && handshakeRequestId == consumerData.HandshakeRequestId, this.options.MaxEventDeliveryTime, deliveryBackoffProvider, - cancellationToken: CancellationToken.None); + cancellationToken: ShutdownToken); if (handshakeRequestId != consumerData.HandshakeRequestId) return false; @@ -568,7 +579,8 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null handshakeRequestId, forceFaultSubscription || effectiveHandshakeToken is StartPositionToken - && exceptionOccured is NotSupportedException); + && exceptionOccured is NotSupportedException, + ShutdownToken); if (handshakeRequestId != consumerData.HandshakeRequestId) return false; var providerFallbackAllowed = providerDefaultRequest @@ -951,6 +963,9 @@ private Task RunQueuePump(QueueId queueId, CancellationToken cancellationToken) private async Task PumpQueue(QueueId queueId, CancellationToken cancellationToken) { + using var admission = _workAdmission.TryEnter(); + if (!admission.Entered) return; + try { Task? localReceiverInitTask = receiverInitTask; @@ -1094,7 +1109,7 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? var partitionStartToken = availableMessages[0].SequenceToken; foreach (var streamData in pubSubCache.Values) { - StartInactiveCursors(streamData, partitionStartToken, CancellationToken.None); + StartInactiveCursors(streamData, partitionStartToken, ShutdownToken); } foreach (var group in availableMessages.GroupBy(container => container.StreamId)) @@ -1109,7 +1124,7 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? { // Run registration in the background so that cold-stream pubsub // calls do not stall message delivery for other streams on the same queue. - RegisterStream(streamId, startToken, now, CancellationToken.None); + RegisterStream(streamId, startToken, now, ShutdownToken); } } @@ -1418,7 +1433,7 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq if (consumerData.State == StreamConsumerDataState.Inactive) { // wake up inactive consumers - TrackConsumerCursor(consumerData, cancellationToken).Ignore(); + RunConsumerCursor(consumerData, cancellationToken).Ignore(); } } else @@ -1651,6 +1666,11 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.Cursor?.RecordDeliveryFailure(); } + if (exc is OperationCanceledException && cancellationToken.IsCancellationRequested) + { + throw; + } + exceptionOccured = exc is ClientNotAvailableException || forceFaultSubscription ? exc : new StreamEventDeliveryFailureException(consumerData.StreamId); @@ -1686,40 +1706,6 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } } - private Task TrackConsumerCursor( - StreamConsumerData consumerData, - CancellationToken cancellationToken = default) - { - var task = RunConsumerCursor(consumerData, cancellationToken); - _activeConsumerCursorTasks.Add(task); - RemoveWhenComplete(task).Ignore(); - return task; - - async Task RemoveWhenComplete(Task pending) - { - await pending.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); - _activeConsumerCursorTasks.Remove(pending); - } - } - - private async Task DrainConsumerCursors(CancellationToken cancellationToken) - { - while (_activeConsumerCursorTasks.Count > 0) - { - var active = Task.WhenAll(_activeConsumerCursorTasks); - try - { - await active.WaitAsync(cancellationToken); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) - { - // RunConsumerCursor logs its failure before propagating it. - } - - cancellationToken.ThrowIfCancellationRequested(); - } - } - private readonly struct ConsumerBatch { public ConsumerBatch(IBatchContainer? batch, StreamSequenceToken? progressToken) @@ -1841,11 +1827,15 @@ private sealed class QueueCacheCursorContractException(string message) : Invalid { try { - StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, handshakeToken, cancellationToken); + StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, handshakeToken, cancellationToken).WaitAsync(cancellationToken); StreamingEvents.EmitMessageDelivered(streamProviderName, consumerData, batch, Silo); return newToken; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception ex) { StreamingEvents.EmitMessageDeliveryFailed(streamProviderName, consumerData, batch.SequenceToken, Silo, ex); @@ -1903,7 +1893,7 @@ private static async Task DeliverErrorToConsumer( RequestContext.Clear(); // clear RequestContext before await! } } - await errorDeliveryTask; + await errorDeliveryTask.WaitAsync(cancellationToken); } private async Task ErrorProtocol( @@ -1923,28 +1913,28 @@ private async Task ErrorProtocol( { LogWarningConsumerIsDead(consumerData.StreamConsumer, consumerData.StreamId); RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); - TrackUnavailableConsumerUnregistration(consumerData); + UnregisterUnavailableConsumer(consumerData).Ignore(); return true; } // notify consumer about the error or that the data is not available. await DeliverErrorToConsumer(consumerData, exceptionOccured, batch, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); if (!IsCurrent()) return false; - + cancellationToken.ThrowIfCancellationRequested(); // record that there was a delivery failure if (isDeliveryError) { await streamFailureHandler.OnDeliveryFailure( - consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } else { await streamFailureHandler.OnSubscriptionFailure( - consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } if (!IsCurrent()) return false; - + cancellationToken.ThrowIfCancellationRequested(); // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if ((forceFaultSubscription || streamFailureHandler.ShouldFaultSubsriptionOnError) && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) @@ -1961,8 +1951,9 @@ await DeliverErrorToConsumer( if (!IsCurrent()) return false; // mark subscription as faulted. + cancellationToken.ThrowIfCancellationRequested(); faultRequested = true; - await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId, cancellationToken); + await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId, _retirementCancellation.Token); } finally { @@ -1979,37 +1970,31 @@ bool IsCurrent() => operationId == (isDeliveryError ? consumerData.HandshakeGeneration : consumerData.HandshakeRequestId); } - private void TrackUnavailableConsumerUnregistration(StreamConsumerData consumerData) - { - var task = UnregisterUnavailableConsumer(consumerData); - _pendingUnavailableConsumerUnregistrations.Add(task); - RemoveWhenComplete(task).Ignore(); - - async Task RemoveWhenComplete(Task pending) - { - await pending.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); - _pendingUnavailableConsumerUnregistrations.Remove(pending); - } - } - private async Task UnregisterUnavailableConsumer(StreamConsumerData consumerData) { + using var admission = _retirementAdmission.TryEnter(); StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Requested); try { + if (!admission.Entered) + { + throw new InvalidOperationException("Consumer retirement admission is closed."); + } + await AsyncExecutorWithRetries.ExecuteWithRetries( async _ => { try { - await pubSub.UnregisterConsumer( + await pubSub.UnregisterConsumerFromProducer( consumerData.SubscriptionId, consumerData.StreamId, - _shutdownCancellation.Token); + GrainId, + _retirementCancellation.Token); return true; } - catch (Exception exception) when (!_shutdownCancellation.IsCancellationRequested) + catch (Exception exception) when (!_retirementCancellation.IsCancellationRequested) { StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, @@ -2028,15 +2013,16 @@ await pubSub.UnregisterConsumer( static (_, _) => true, Timeout.InfiniteTimeSpan, deliveryBackoffProvider, - _shutdownCancellation.Token); + _retirementCancellation.Token); StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Completed); } - catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + catch (OperationCanceledException) when (_retirementCancellation.IsCancellationRequested) { } catch (Exception exception) { + _retirementFailure ??= ExceptionDispatchInfo.Capture(exception); StreamingEvents.EmitSubscriptionUnregistration( streamProviderName, consumerData, Silo, StreamingEvents.SubscriptionUnregistrationStage.Failed, exception); LogWarningUnregisterUnavailableConsumer(consumerData.SubscriptionId, consumerData.StreamId, exception); @@ -2053,9 +2039,13 @@ private static async Task> PubsubRegisterProducer( { try { - var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer, cancellationToken); + var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer, cancellationToken).WaitAsync(cancellationToken); return streamData; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception e) { LogErrorRegisterAsStreamProducer(logger, e); diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingManager.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingManager.cs index 85df99a9f95..3ba47262de8 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingManager.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingManager.cs @@ -23,7 +23,7 @@ internal sealed partial class PersistentStreamPullingManager : SystemTarget, IPe private readonly Dictionary queuesToAgentsMap; private readonly Dictionary deactivatedAgents = new(); private readonly string streamProviderName; - private readonly IStreamPubSub pubSub; + private readonly IStreamPubSubRuntime pubSub; private readonly SystemTargetShared _systemTargetShared; private readonly StreamPullingAgentOptions options; @@ -49,7 +49,7 @@ internal sealed partial class PersistentStreamPullingManager : SystemTarget, IPe internal PersistentStreamPullingManager( SystemTargetGrainId managerId, string strProviderName, - IStreamPubSub streamPubSub, + IStreamPubSubRuntime streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, IStreamFilter streamFilter, diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs new file mode 100644 index 00000000000..ec3aacd4596 --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs @@ -0,0 +1,247 @@ +using NSubstitute; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Runtime.Scheduler; +using Orleans.Streams; +using Xunit; + +namespace UnitTests.StreamingTests; + +public partial class PersistentStreamPullingAgentTests +{ + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Shutdown_ClosesAdmissionAndDrainsPumpBeforeReinitialize(bool failRead) + { + var readStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var read = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var queueId = QueueId.GetQueueId("queue", 0, 0); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => + { + readStarted.TrySetResult(); + return read.Task; + }); + var pubSub = Substitute.For(); + var agent = CreateAgent(pubSub, queueId, receiver); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + var pump = accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + Task? shutdown = null; + try + { + await readStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + shutdown = accessor.Shutdown(); + Assert.Empty(await accessor.GetPubSubCache()); + Assert.False(shutdown.IsCompleted); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + var streamId = new QualifiedStreamId("provider", StreamId.Create("closed", Guid.NewGuid())); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + await agent.RunOrQueueTask(() => agent.AddSubscriber( + GuidId.GetNewGuidId(), streamId, GrainId.Create("consumer", "closed"), null, TestContext.Current.CancellationToken)); + + Assert.False(pump.IsCompleted); + await receiver.Received(1).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); + await receiver.DidNotReceive().Shutdown(Arg.Any(), Arg.Any()); + await pubSub.DidNotReceive().RegisterProducer( + Arg.Any(), Arg.Any(), Arg.Any()); + Assert.Empty(await accessor.GetPubSubCache()); + } + finally + { + if (failRead) + { + read.TrySetException(new InvalidOperationException("Controlled queue read failure.")); + } + else + { + read.TrySetResult([]); + } + + await Task.WhenAll(shutdown ?? accessor.Shutdown(), pump) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await receiver.Received(1).Shutdown(Arg.Any(), Arg.Any()); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + await InitializeAgent(agent); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + await accessor.Shutdown(); + await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); + await receiver.Received(2).Shutdown(Arg.Any(), Arg.Any()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_DrainsReceiverInitializationAfterCallerCancellation() + { + var initialized = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var initializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receiver = Substitute.For(); + receiver.Initialize(Arg.Any(), Arg.Any()).Returns(_ => + { + initializationStarted.TrySetResult(); + return initialized.Task; + }); + var agent = CreateAgent(pubSub: null, QueueId.GetQueueId("queue", 0, 0), receiver); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + using var cancellation = new CancellationTokenSource(); + await agent.RunOrQueueTask(() => agent.Initialize(cancellation.Token)); + Task? shutdown = null; + try + { + await initializationStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + cancellation.Cancel(); + shutdown = accessor.Shutdown(); + Assert.Empty(await accessor.GetPubSubCache()); + Assert.False(initialized.Task.IsCompleted); + Assert.False(shutdown.IsCompleted); + await receiver.DidNotReceive().Shutdown(Arg.Any(), Arg.Any()); + } + finally + { + initialized.TrySetResult(); + await (shutdown ?? accessor.Shutdown()).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await receiver.Received(1).Shutdown(Arg.Any(), Arg.Any()); + await receiver.DidNotReceive().GetQueueMessagesAsync(Arg.Any(), Arg.Any()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_CancelsPendingSubscriberHandshake() + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, Arg.Any()) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0, 0); + var streamId = new QualifiedStreamId("provider", StreamId.Create("handshake", Guid.NewGuid())); + var receiver = Substitute.For(); + var cache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + var streamData = Assert.Single(await accessor.GetPubSubCache()).Value; + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer(getSequenceToken: token => + { + handshakeStarted.TrySetResult(token); + return handshake.Task; + }); + var subscription = GuidId.GetNewGuidId(); + var data = streamData.AddConsumer(subscription, streamId, consumer, null, DateTime.UtcNow); + try + { + await agent.RunOrQueueTask(() => agent.AddSubscriber( + subscription, streamId, GrainId.Create("consumer", "handshake"), null, TestContext.Current.CancellationToken)); + var token = await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(token.CanBeCanceled); + Assert.False(token.IsCancellationRequested); + await accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(token.IsCancellationRequested); + Assert.False(handshake.Task.IsCompleted); + Assert.False(data.IsRegistered); + Assert.Empty(consumer.DeliveredTokens); + Assert.Equal(0, cache.DeliveryProgressCallCount); + Assert.Empty(await accessor.GetPubSubCache()); + await receiver.Received(1).Shutdown(Arg.Any(), Arg.Any()); + } + finally + { + handshake.TrySetResult(null); + await accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_DrainsRetirementWithFreshTokenAfterReinitialize() + { + var streamId = new QualifiedStreamId("provider", StreamId.Create("retirement", Guid.NewGuid())); + var token = new EventSequenceTokenV2(1); + var cache = new ScriptedQueueCache(); + cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, Arg.Any()) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(Arg.Any()).Returns(cache); + var receiver = Substitute.For(); + var agent = CreateAgent(pubSub, QueueId.GetQueueId("queue", 0, 0), receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + var previousToken = default(CancellationToken); + + for (var run = 0; run < 2; run++) + { + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, token, DateTime.UtcNow); + var streamData = Assert.Single(await accessor.GetPubSubCache()).Value; + var subscription = GuidId.GetNewGuidId(); + var data = streamData.AddConsumer( + subscription, streamId, + new UnavailableConsumer(Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException()), + null, DateTime.UtcNow); + data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); + data.IsRegistered = true; + var retirement = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retirementStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + pubSub.UnregisterConsumerFromProducer(subscription, streamId, agent.GrainId, Arg.Any()) + .Returns(call => + { + retirementStarted.TrySetResult(call.ArgAt(3)); + return retirement.Task; + }); + using var callerCancellation = new CancellationTokenSource(); + Task? shutdown = null; + Task? repeatedShutdown = null; + var cleanupToken = default(CancellationToken); + try + { + await accessor.RunConsumerCursor(data).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + cleanupToken = await retirementStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(cleanupToken.CanBeCanceled); + Assert.False(cleanupToken.IsCancellationRequested); + Assert.NotEqual(previousToken, cleanupToken); + shutdown = agent.RunOrQueueTask(() => agent.Shutdown(callerCancellation.Token)); + await accessor.GetPubSubCache(); + callerCancellation.Cancel(); + repeatedShutdown = accessor.Shutdown(); + await accessor.GetPubSubCache(); + + Assert.False(shutdown.IsCompleted); + Assert.False(repeatedShutdown.IsCompleted); + Assert.False(cleanupToken.IsCancellationRequested); + await receiver.Received(run).Shutdown(Arg.Any(), Arg.Any()); + } + finally + { + retirement.TrySetResult(); + await Task.WhenAll(shutdown ?? accessor.Shutdown(), repeatedShutdown ?? Task.CompletedTask) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.True(cleanupToken.IsCancellationRequested); + await receiver.Received(run + 1).Shutdown( + Arg.Any(), Arg.Is(static cancellation => !cancellation.CanBeCanceled)); + previousToken = cleanupToken; + } + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs index 3ef5b973403..03ae61c56a9 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentDiagnosticsTests.cs @@ -36,7 +36,7 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking data.IsRegistered = true; var unregister = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var unregisterAttempts = 0; - pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + pubSub.UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()).Returns(_ => Interlocked.Increment(ref unregisterAttempts) == 1 ? unregister.Task : Task.CompletedTask); @@ -112,7 +112,7 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking Assert.Same(consumer, completed.Consumer); Assert.Null(completed.Exception); _ = pubSub.Received(failUnregistration ? 2 : 1) - .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + .UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()); } finally { @@ -121,7 +121,10 @@ public async Task UnavailableConsumerReportsUnregistrationOutcomeWithoutBlocking } } - [Fact] + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task ShutdownDrainsUnavailableConsumerUnregistrationBeforeCancelingRetries() { var streamId = new QualifiedStreamId("provider", StreamId.Create("shutdown-unregister", Guid.NewGuid())); @@ -142,7 +145,7 @@ public async Task ShutdownDrainsUnavailableConsumerUnregistrationBeforeCanceling data.IsRegistered = true; var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var attempts = 0; - pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + pubSub.UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()).Returns(_ => Interlocked.Increment(ref attempts) == 1 ? firstAttempt.Task : Task.CompletedTask); await accessor.RunConsumerCursor(data).WaitAsync( @@ -155,10 +158,13 @@ await accessor.RunConsumerCursor(data).WaitAsync( await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); _ = pubSub.Received(2) - .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + .UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()); } - [Fact] + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task ShutdownDrainsCleanupStartedByInFlightDelivery() { var streamId = new QualifiedStreamId("provider", StreamId.Create("shutdown-delivery", Guid.NewGuid())); @@ -169,30 +175,44 @@ public async Task ShutdownDrainsCleanupStartedByInFlightDelivery() var (accessor, pubSub, streamData) = await CreateInitializedAgentWithStream( streamId, token, cache, new StreamPullingAgentOptions()); var unavailable = Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException(); - var consumer = new BlockingUnavailableConsumer(unavailable); + var consumer = new UnavailableConsumer(unavailable); var data = streamData.AddConsumer(subscriptionId, streamId, consumer, filterData: null, DateTime.UtcNow); data.Cursor = cache.GetCacheCursor(streamId.StreamId, token); data.IsRegistered = true; var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var attempts = 0; - pubSub.UnregisterConsumer(subscriptionId, streamId, Arg.Any()).Returns(_ => + pubSub.UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()).Returns(_ => Interlocked.Increment(ref attempts) == 1 ? firstAttempt.Task : Task.CompletedTask); - var delivery = accessor.RunConsumerCursor(data); - await consumer.DeliveryStarted.Task.WaitAsync( - TimeSpan.FromSeconds(10), - TestContext.Current.CancellationToken); - var shutdown = accessor.Shutdown(); - - Assert.False(shutdown.IsCompleted); - consumer.FailDelivery(); - await delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.False(shutdown.IsCompleted); - firstAttempt.SetException(new InvalidOperationException("transient cleanup failure")); - await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Task? shutdown = null; + using var observer = StreamingEvents.AllEvents.Subscribe(value => + { + if (value is StreamingEvents.SubscriptionRemoved removed && removed.SubscriptionId == subscriptionId.Guid) + { + // Close work admission before the failing delivery starts durable retirement. + shutdown = accessor.Shutdown(); + } + }); + try + { + await accessor.RunConsumerCursor(data).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.NotNull(shutdown); + await accessor.GetPubSubCache(); + Assert.False(shutdown.IsCompleted); + Assert.Equal(1, attempts); + Assert.False(streamData.Contains(subscriptionId)); + Assert.Null(data.Cursor); + firstAttempt.SetException(new InvalidOperationException("transient cleanup failure")); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + finally + { + firstAttempt.TrySetResult(); + await (shutdown ?? accessor.Shutdown()).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } _ = pubSub.Received(2) - .UnregisterConsumer(subscriptionId, streamId, Arg.Any()); + .UnregisterConsumerFromProducer(subscriptionId, streamId, Arg.Any(), Arg.Any()); } private sealed class UnavailableConsumer(ClientNotAvailableException exception) : IStreamConsumerExtension diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs index 2f97ca6cd46..9dfe97a3adb 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs @@ -64,7 +64,7 @@ public async Task FutureStartToken_BeyondFiniteCapacity_PreservesDeliveryBarrier source, adapter, cache, checkpointer, startFromNow: false); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(queueId).Returns(receiver); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, cancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 77f15a48fe0..9b5d7108fa2 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -54,7 +54,7 @@ public void InitialDeliveryProgressIncludesOnlyAcknowledgedDeliveryToken() public async Task ReadFromQueue_DoesNotWaitForColdStreamRegistration() { var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(_ => registration.Task); @@ -109,7 +109,7 @@ public bool ShouldDeliver(StreamId streamId, object item, string? filterData) [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task ReadFromQueue_ClearsRegistrationTaskWhenColdStreamRegistrationCompletesSynchronously() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -177,16 +177,18 @@ public async Task DroppedClientCleanupCompletesBeforeShutdown() var unregistration = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var unregisteredSubscriptionId = default(GuidId); QualifiedStreamId? unregisteredStreamId = null; + var unregisteredProducer = default(GrainId); var unregistrationCancellationToken = default(CancellationToken); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default!, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - pubSub.UnregisterConsumer(Arg.Any(), Arg.Any(), Arg.Any()) + pubSub.UnregisterConsumerFromProducer(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .ReturnsForAnyArgs(call => { unregisteredSubscriptionId = call.ArgAt(0); unregisteredStreamId = call.ArgAt(1); - unregistrationCancellationToken = call.ArgAt(2); + unregisteredProducer = call.ArgAt(2); + unregistrationCancellationToken = call.ArgAt(3); unregistrationStarted.TrySetResult(true); return unregistration.Task; }); @@ -219,6 +221,7 @@ public async Task DroppedClientCleanupCompletesBeforeShutdown() Assert.Equal(subscriptionId, unregisteredSubscriptionId); Assert.Equal(streamId, unregisteredStreamId); + Assert.Equal(agent.GrainId, unregisteredProducer); Assert.True(unregistrationCancellationToken.CanBeCanceled); Assert.False(unregistrationCancellationToken.IsCancellationRequested); Assert.False(streamData.Contains(subscriptionId)); @@ -238,7 +241,7 @@ public async Task DroppedClientCleanupCompletesBeforeShutdown() } Assert.True(unregistrationCancellationToken.IsCancellationRequested); - await pubSub.Received(1).UnregisterConsumer(subscriptionId, streamId, unregistrationCancellationToken); + await pubSub.Received(1).UnregisterConsumerFromProducer(subscriptionId, streamId, agent.GrainId, unregistrationCancellationToken); } [TestSuite("BVT")] @@ -276,7 +279,7 @@ public async Task ReadFromQueue_CachesDequeuedMessagesBeforeObservingCancellatio var qualifiedStreamId = new QualifiedStreamId("provider", streamId); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(_ => registration.Task); var receiver = Substitute.For(); @@ -333,7 +336,7 @@ public async Task RegisterStream_RemovesCacheEntryWhenProducerRegistrationTermin [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task RegisterStream_DoesNotRegisterProducerAfterShutdownStarts() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var agent = CreateAgent(pubSub, queueId); @@ -354,7 +357,7 @@ public async Task RegisterStream_DoesNotRegisterProducerAfterShutdownStarts() public async Task ReadFromQueue_CleansInactiveStreamsUsingTimeProvider() { var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -387,7 +390,7 @@ public async Task ReadFromQueue_CleansInactiveStreamsUsingTimeProvider() [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task ReadFromQueue_DoesNotAcknowledgeBatchedMessagesDuringConsumerDelivery() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -514,7 +517,7 @@ public QueueCacheCursorResult TryGetCacheCursorAtPosition( } private static PersistentStreamPullingAgent CreateAgent( - IStreamPubSub? pubSub, + IStreamPubSubRuntime? pubSub, QueueId queueId, IQueueAdapterReceiver? receiver = null, IQueueAdapterCache? queueAdapterCache = null, @@ -1352,7 +1355,9 @@ private sealed class RejectingStreamFilter : IStreamFilter public bool ShouldDeliver(StreamId streamId, object item, string? filterData) => false; } - private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = null) : IStreamConsumerExtension + private sealed class RecordingConsumer( + StreamHandshakeToken? requestedToken = null, + Func>? getSequenceToken = null) : IStreamConsumerExtension { private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -1393,7 +1398,7 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke } public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) - => OnHandshake?.Invoke() ?? Task.FromResult(requestedToken); + => getSequenceToken?.Invoke(cancellationToken) ?? OnHandshake?.Invoke() ?? Task.FromResult(requestedToken); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -1570,7 +1575,7 @@ public async Task ExplicitEarliestAvailableOverridesProviderLatest() queueCache.AddToCache([new TestBatchContainer(streamId, retainedToken)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); @@ -1612,7 +1617,7 @@ public async Task LatestLateSubscriberReceivesOnlyFutureMessage() queueCache.AddToCache([new TestBatchContainer(streamId, new EventSequenceTokenV2(1))]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RecordingConsumer(); @@ -1690,7 +1695,7 @@ public async Task LegacyAcknowledgedTokenSkipsDerivedDuplicateAndDeliversPrefetc var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var agent = CreateAgent( - pubSub: Substitute.For(), + pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -1924,7 +1929,7 @@ public async Task InitialImplicitRecoveryAdvancesPastAcknowledgedMessage() var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var agent = CreateAgent( - pubSub: Substitute.For(), + pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -1960,7 +1965,7 @@ public async Task InitialImplicitRecoveryDeliversFirstNewerMessageWhenAcknowledg var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var agent = CreateAgent( - pubSub: Substitute.For(), + pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; @@ -2149,7 +2154,7 @@ public async Task EarliestAvailableWaitsWhenTargetStreamIsNotCached() queueCache.AddToCache([new TestBatchContainer(otherStreamId, new EventSequenceTokenV2(1))]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RecordingConsumer( @@ -2186,7 +2191,7 @@ public async Task EarliestAvailableIsPreservedDuringDeliveryHandshakeRenegotiati queueCache.AddToCache([new TestBatchContainer(streamId, retainedToken)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RenegotiatingEarliestConsumer(); @@ -2217,7 +2222,7 @@ public async Task StartTokenRemainsInclusiveDuringDeliveryHandshakeRenegotiation queueCache.AddToCache([new TestBatchContainer(streamId, startToken)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RenegotiatingStartTokenConsumer(startToken); @@ -2251,7 +2256,7 @@ public async Task DeliveryTokenRenegotiationDeliversFirstNewerMessageWhenAcknowl queueCache.AddToCache([new TestBatchContainer(streamId, attemptedToken)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RenegotiatingDeliveryTokenConsumer(acknowledgedToken); @@ -2287,7 +2292,7 @@ public async Task ImplicitRecoveryStartTokenAdvancesPastAcknowledgedMessage() ]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); + var agent = CreateAgent(pubSub: Substitute.For(), QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); var consumer = new RenegotiatingStartTokenConsumer(acknowledgedToken); @@ -2345,7 +2350,7 @@ await pubSub.Received(1).FaultSubscription( private static async Task<( PersistentStreamPullingAgent.ITestAccessor Accessor, - IStreamPubSub PubSub, + IStreamPubSubRuntime PubSub, StreamConsumerCollection StreamData, StreamConsumerData ConsumerData, UnknownHandshakeConsumer Consumer)> CreateUnknownTokenTest(bool returnDuringInitialHandshake) @@ -2357,7 +2362,7 @@ await pubSub.Received(1).FaultSubscription( queueCache.AddToCache([new TestBatchContainer(streamId, token)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); @@ -2389,7 +2394,7 @@ public async Task UnsupportedCacheFaultsDeliveryHandshakeRenegotiation() queueCache.AddToCache([new TestBatchContainer(streamId, retainedToken)]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); @@ -2427,7 +2432,7 @@ public async Task EarliestAvailableReportsUnsupportedCustomCache() var queueCache = new RecordingQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); @@ -2467,7 +2472,7 @@ public async Task ProviderDefaultEarliestRejectsInvalidCustomCacheResult() var queueCache = new InvalidPositionQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( @@ -2506,7 +2511,7 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() var queueCache = new InvalidMoveQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( @@ -2595,7 +2600,7 @@ private sealed class ImmediateConsumer : IStreamConsumerExtension [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task ReadFromQueue_RefreshesIdleCursorAfterItsTokenMetadataIsPurged() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -2699,7 +2704,7 @@ public Task RunConsumerCursor_RecoversThroughLegacyOnlyCacheWhenEarliestIsUnsupp private static async Task VerifyRunConsumerCursorRecoversThroughLegacyCursor( bool supportsEarliestAvailable) { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>( new HashSet())); @@ -2806,7 +2811,7 @@ private static async Task RunCacheMissRecovery( StreamSequenceToken newestToken, bool supportsEarliestAvailable) { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -2847,7 +2852,7 @@ private static async Task RunCacheMissRecovery( [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task Shutdown_AdvancesAfterConsumerAcceptsRetryFollowingHandshakeToken() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -3498,7 +3503,7 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => private static async Task<( PersistentStreamPullingAgent.ITestAccessor Accessor, - IStreamPubSub PubSub, + IStreamPubSubRuntime PubSub, StreamConsumerCollection StreamData)> CreateInitializedAgentWithStream( QualifiedStreamId streamId, StreamSequenceToken registrationToken, @@ -3508,7 +3513,7 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => IQueueAdapterReceiver? receiver = null, IStreamFailureHandler? failureHandler = null) { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); pubSub.FaultSubscription(Arg.Any(), Arg.Any(), Arg.Any()) @@ -3587,7 +3592,7 @@ public async Task RegisterStream_KeepsCacheEntryWhenSubscriberHandshakeFails() var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var consumerGrainId = GrainId.Create("test", Guid.NewGuid().ToString()); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>( new HashSet @@ -4076,7 +4081,7 @@ await accessor.RegisterStream( [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task Shutdown_PushesEarliestDeliveryProgressTokenToCache() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -4129,7 +4134,7 @@ public async Task Shutdown_PushesEarliestDeliveryProgressTokenToCache() [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task Shutdown_PushesEarliestDeliveryProgressUsingBaseTokenPosition() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -4184,7 +4189,7 @@ public async Task Shutdown_PushesEarliestDeliveryProgressUsingBaseTokenPosition( [InlineData(true)] public async Task Shutdown_PreservesCheckpointForIncompatibleTokens(bool providerTokenFirst) { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -4244,7 +4249,7 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg { var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); var registeredStreamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(call => call.ArgAt(0).Equals(registeredStreamId) ? Task.FromResult>(new HashSet()) @@ -4302,16 +4307,18 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg var shutdownTask = testAccessor.Shutdown(); try { - // A queued accessor turn observes shutdown after it has started draining registration. - await testAccessor.GetPubSubCache().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.False(shutdownTask.IsCompleted); - Assert.False(receiverShutdownStarted.Task.IsCompleted); + await shutdownTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(registration.Task.IsCompleted); + Assert.True(receiverShutdownStarted.Task.IsCompletedSuccessfully); Assert.Empty(queueCache.DeliveryProgressTokens); Assert.Equal(0, queueCache.DeliveryProgressCallCount); + await pubSub.Received(1).UnregisterProducer( + new QualifiedStreamId("provider", streamId), agent.GrainId, Arg.Any()); if (registrationFails) { registration.SetException(new InvalidOperationException("Producer registration failed during shutdown.")); + await Assert.ThrowsAsync(() => registration.Task); } else { @@ -4322,6 +4329,7 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg new QualifiedStreamId("provider", streamId), GrainId.Create("consumer", Guid.NewGuid().ToString())), }); + await registration.Task; } } finally @@ -4344,7 +4352,7 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg [Fact, TestCategory("BVT"), TestCategory("Streaming")] public async Task Shutdown_SkipsDeliveryProgressForUnregisteredConsumer() { - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); @@ -4501,7 +4509,7 @@ public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparis var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); @@ -4565,7 +4573,7 @@ public async Task DeliveryProgress_InclusiveStartRemainsUnsafeBeforeFirstBatch( var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( @@ -4630,7 +4638,7 @@ public async Task Handshake_DeliveryTokenStartsAfterConfirmedRecord() receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult>( [new TestBatchContainer(streamId, new EventSequenceTokenV2(12))])); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); @@ -4675,7 +4683,7 @@ public async Task ReadFromQueue_WakesIdleCursorAtFirstPartitionRecord() var queueCache = new PurgeablePooledQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); @@ -4714,7 +4722,7 @@ public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); @@ -4769,7 +4777,7 @@ .. Enumerable.Range(2, 8) ]); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); @@ -4841,7 +4849,7 @@ public async Task DeliveryProgress_QuietSubscriptionAdvancesAcrossBusyStreamBeyo var queueCache = new ScriptedQueueCache(maxCacheSize: 3); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( @@ -4915,7 +4923,7 @@ await queueCache.DeliveryProgressUpdated.WaitAsync( public async Task DeliveryProgress_PendingRegistrationBlocksPeriodicUpdate() { var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken).ReturnsForAnyArgs(_ => registration.Task); var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); var options = new StreamPullingAgentOptions(); @@ -4966,7 +4974,7 @@ public async Task DeliveryProgress_IncludesFilteredBatches() var queueCache = new ScriptedQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( @@ -5026,7 +5034,7 @@ public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() var queueCache = new ScriptedQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/TaskSchedulerSelectionTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/TaskSchedulerSelectionTests.cs index ad8e439a7d0..0df1adace45 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/TaskSchedulerSelectionTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/TaskSchedulerSelectionTests.cs @@ -126,7 +126,7 @@ public async Task BatchWorker_CancellationRemainsObservableOnCallingScheduler() public async Task StreamSubscriptionManager_QueryProjectionUsesCallingScheduler() { var scheduler = new ManualTaskScheduler(); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); var subscriptions = new List(); var query = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); Func>> getAllSubscriptions = pubSub.GetAllSubscriptions; From 7a298c63bee5a0402415abc520fe2d5c9fdc48cf Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 15:16:51 -0700 Subject: [PATCH 83/90] fix(streaming): integrate drained checkpoints with admission shutdown --- .../PersistentStreamPullingAgent.cs | 39 ++++++++++++++----- .../PersistentStreamPullingAgentTests.cs | 8 ++-- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 66ab91e023a..5aab1b43992 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -291,7 +291,8 @@ private async Task ShutdownCore() // Canceled registrations can remove themselves from the cache before producer cleanup. var streams = pubSubCache.ToArray(); - var hasPendingRegistrations = streams.Any(static entry => entry.Value.RegistrationTask is not null); + var hasPendingSubscriptions = streams.Any(static entry => entry.Value.RegistrationTask is not null + || entry.Value.AllConsumers().Any(static consumer => consumer.PendingHandshakes != 0)); _shutdownCancellation?.Cancel(); Task? localReceiverInitTask = receiverInitTask; @@ -313,9 +314,8 @@ private async Task ShutdownCore() _retirementCancellation.Dispose(); } - // Registrations drained during shutdown can exit before discovering subscribers. - // Preserve their checkpoint barrier even after their tasks have completed. - if (!hasPendingRegistrations) + // Canceled registrations and handshakes retain their pre-shutdown checkpoint barrier. + if (!hasPendingSubscriptions) { NotifyDeliveryProgress(); } @@ -1107,18 +1107,23 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? } var partitionStartToken = availableMessages[0].SequenceToken; - foreach (var streamData in pubSubCache.Values) + var groupedMessages = availableMessages.ToLookup(static container => container.StreamId); + foreach (var (streamId, streamData) in pubSubCache) { - StartInactiveCursors(streamData, partitionStartToken, ShutdownToken); + if (!groupedMessages.Contains(streamId.StreamId)) + { + StartInactiveCursors(streamData, partitionStartToken, ShutdownToken, onlyProgressAware: true); + } } - foreach (var group in availableMessages.GroupBy(container => container.StreamId)) + foreach (var group in groupedMessages) { var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); StreamSequenceToken startToken = group.First().SequenceToken; if (pubSubCache.TryGetValue(streamId, out var streamData)) { streamData.RefreshActivity(now); + StartInactiveCursors(streamData, startToken, ShutdownToken); } else { @@ -1417,7 +1422,11 @@ async Task SubscribeWithIsolation(PubSubSubscriptionState item) } } - private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSequenceToken startToken, CancellationToken cancellationToken) + private void StartInactiveCursors( + StreamConsumerCollection streamData, + StreamSequenceToken startToken, + CancellationToken cancellationToken, + bool onlyProgressAware = false) { foreach (StreamConsumerData consumerData in streamData.AllConsumers()) { @@ -1426,6 +1435,11 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq return; } + if (onlyProgressAware && consumerData.Cursor is not IQueueCacheCursorProgress) + { + continue; + } + // Some consumer might not be fully registered yet if (consumerData.IsRegistered) { @@ -1651,7 +1665,12 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (Exception exc) { - LogErrorDeliveringMessages(consumerData.StreamId, exc); + var isCancellation = exc is OperationCanceledException && cancellationToken.IsCancellationRequested; + if (!isCancellation) + { + LogErrorDeliveringMessages(consumerData.StreamId, exc); + } + if (handshakeGeneration != consumerData.HandshakeGeneration) { continue; @@ -1666,7 +1685,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.Cursor?.RecordDeliveryFailure(); } - if (exc is OperationCanceledException && cancellationToken.IsCancellationRequested) + if (isCancellation) { throw; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 9b5d7108fa2..46407ab2e2f 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -1365,7 +1365,7 @@ private sealed class RecordingConsumer( public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); - public Func>? OnHandshake { get; set; } + public Func>? OnHandshake { get; set; } = getSequenceToken; public Func>? OnDelivery { get; set; } public Func? OnError { get; set; } @@ -1398,7 +1398,7 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke } public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) - => getSequenceToken?.Invoke(cancellationToken) ?? OnHandshake?.Invoke() ?? Task.FromResult(requestedToken); + => OnHandshake?.Invoke(cancellationToken) ?? Task.FromResult(requestedToken); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -3444,7 +3444,7 @@ private static async Task CreateCheckpointScenario(TimeProvi var busyId = new QualifiedStreamId("provider", StreamId.Create("busy", Guid.NewGuid())); var (accessor, _, idleStream) = await CreateInitializedAgentWithStream( idleId, new EventSequenceTokenV2(1), cache, - new StreamPullingAgentOptions(), timeProvider); + new StreamPullingAgentOptions(), timeProvider: timeProvider); await accessor.RegisterStream(busyId, new EventSequenceTokenV2(2), DateTime.UtcNow); var busyStream = (await accessor.GetPubSubCache())[busyId]; @@ -3509,6 +3509,7 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => StreamSequenceToken registrationToken, IQueueCache queueCache, StreamPullingAgentOptions options, + IStreamFilter? filter = null, TimeProvider? timeProvider = null, IQueueAdapterReceiver? receiver = null, IStreamFailureHandler? failureHandler = null) @@ -3527,6 +3528,7 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => queueAdapterCache: queueAdapterCache, timeProvider: timeProvider, options: options, + streamFilter: filter, failureHandler: failureHandler); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); From 452cc10b38947388ea24ef5bdfa353df10940621 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 18:52:38 -0700 Subject: [PATCH 84/90] docs(streaming): align receiver shutdown with retained-log ownership --- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index 8f593a854aa..c96cf6c7f01 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -134,9 +134,9 @@ public class ConsumerGrain : Grain, IConsumerGrain, IAsyncObserver ## Receiver shutdown -During shutdown or queue reassignment, each receiver closes admission and waits for accepted dequeue and confirmation operations to finish, including batch conversion and receipt bookkeeping. It then releases its pending messages using their current receipts so another receiver can dequeue them promptly. +During shutdown or queue reassignment, the pulling agent closes processing admission and drains admitted queue reads, registrations, delivery, and durable subscription retirement. It preserves pending-registration and handshake checkpoint barriers before the receiver flushes durable progress and releases its pooled cache resources. -The shutdown timeout bounds the drain wait and the subsequent release query separately. If the drain times out, the receiver logs a warning and pending messages become eligible for redelivery through their database visibility timeout. +The receiver bounds its asynchronous shutdown work using the supplied timeout. Receiver replacement waits for outstanding partition acquisition to settle. The next receiver acquires a fresh ownership epoch and resumes after the durable checkpoint, with retained history governed by the configured retention policy. ## Documentation For more comprehensive documentation, please refer to: From 597249589ac8eb485ffb0c4dd7581d37bfda624b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 19:17:52 -0700 Subject: [PATCH 85/90] test(streaming): align retirement setup with publisher registration --- .../PubSubRendezvousGrainTests.ProducerRetirement.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs index 5d6263291d3..60a23d1fe1d 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.ProducerRetirement.cs @@ -270,8 +270,8 @@ await AwaitRetirementPhase( var cancellationToken = TestContext.Current.CancellationToken; // Register before producers so setup cannot be confused with removal callbacks. await result.PubSub.RegisterConsumer(result.SubscriptionId, result.StreamId, default, null, cancellationToken); - var requesterSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.RequesterContext.GrainId, cancellationToken); - var otherSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.OtherContext.GrainId, cancellationToken); + var requesterSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.RequesterContext.GrainId, cancellationToken: cancellationToken); + var otherSubscriptions = await result.PubSub.RegisterProducer(result.StreamId, result.OtherContext.GrainId, cancellationToken: cancellationToken); Assert.Equal(result.SubscriptionId, Assert.Single(requesterSubscriptions).SubscriptionId); Assert.Equal(result.SubscriptionId, Assert.Single(otherSubscriptions).SubscriptionId); return result; From 790542a8280d1cf4081e54d3396d8b82600bc96f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 21:34:51 -0700 Subject: [PATCH 86/90] fix(streaming): align detached work and retirement lifetimes --- .../streams-implementation/index.md | 4 +- .../PersistentStreamPullingAgent.cs | 11 ++-- ...sistentStreamPullingAgentAdmissionTests.cs | 15 +++-- .../PersistentStreamPullingAgentTests.cs | 58 ++++++++++--------- 4 files changed, 48 insertions(+), 40 deletions(-) diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 5ddaf32140f..82014a143fc 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -77,7 +77,7 @@ The default maximum adapter batch-container batch size is 1 and the empty-poll p ### Shutdown and admitted work -Each pulling-agent run owns admission for queue reads, producer registrations, subscription handshakes, and consumer delivery. Shutdown closes processing admission, stops polling, cancels processing waits, and drains admitted work and actual receiver initialization. Accepted work completes its token bookkeeping and releases registration pins and batch protection while the cache and receiver remain available. Repeated shutdown calls share the same completion. Initialization waits for the prior run's full cleanup before opening fresh admission and cancellation scopes. +Each pulling-agent run owns admission for queue reads, producer registrations, subscription handshakes, and consumer delivery. Shutdown closes processing admission, stops polling, and drains admitted work and actual receiver initialization. Accepted calls retain their messaging timeouts and retry limits while completing token bookkeeping and releasing registration pins and batch protection with the cache and receiver still available. Repeated shutdown calls share the same completion. Initialization waits for the prior run's full cleanup before opening fresh admission and cancellation scopes. Explicit subscription notifications receive an immediate acknowledgement while the agent tracks their asynchronous handshake through completion. This lets the subscribing consumer finish its current call and respond to the handshake. @@ -89,7 +89,7 @@ Subscription removal revokes in-flight handshake and delivery ownership. A termi An unavailable client is detached locally immediately. Durable subscription retirement has its own admission and cancellation lifetime, allowing a finishing delivery to start cleanup after processing admission closes. Once processing drains, shutdown closes retirement admission and waits for persistence and notification retries to finish. Producer-initiated retirement persists the removal and notifies the other producers; the requesting producer has already detached that subscription. Ordinary consumer unregistration notifies every registered producer. -The final delivery-progress scan preserves the checkpoint barrier for registrations and handshakes which were pending when shutdown began. Cancellation and failed registration retain the prior safe position. The agent disposes subscription cursors before receiver shutdown flushes the safe checkpoint and releases its resources. Producer unregistration follows receiver cleanup. Operation failures remain observable through their lifecycle outcomes and correlated diagnostics. +The final delivery-progress scan preserves the checkpoint barrier for producer registrations which were pending when shutdown began. Accepted handshakes establish their final position during the drain; unresolved handshakes keep the existing checkpoint. The agent disposes subscription cursors before receiver shutdown flushes the safe checkpoint and releases its resources. Producer unregistration follows receiver cleanup. Operation failures remain observable through their lifecycle outcomes and correlated diagnostics. ## Cache and cursor invariants diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 5aab1b43992..4ad3ddf812e 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -289,11 +289,9 @@ private async Task ShutdownCore() StreamingEvents.EmitPullingAgentStopped(streamProviderName, Silo, QueueId); } - // Canceled registrations can remove themselves from the cache before producer cleanup. + // Registrations can remove themselves from the cache before producer cleanup. var streams = pubSubCache.ToArray(); - var hasPendingSubscriptions = streams.Any(static entry => entry.Value.RegistrationTask is not null - || entry.Value.AllConsumers().Any(static consumer => consumer.PendingHandshakes != 0)); - _shutdownCancellation?.Cancel(); + var hasPendingRegistrations = streams.Any(static entry => entry.Value.RegistrationTask is not null); Task? localReceiverInitTask = receiverInitTask; if (localReceiverInitTask != null) @@ -303,6 +301,7 @@ private async Task ShutdownCore() } await workDrained; + _shutdownCancellation?.Cancel(); try { @@ -314,8 +313,8 @@ private async Task ShutdownCore() _retirementCancellation.Dispose(); } - // Canceled registrations and handshakes retain their pre-shutdown checkpoint barrier. - if (!hasPendingSubscriptions) + // Registrations which exit during shutdown retain their checkpoint barrier. + if (!hasPendingRegistrations) { NotifyDeliveryProgress(); } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs index ec3aacd4596..a93548e84b2 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs @@ -120,7 +120,7 @@ public async Task Shutdown_DrainsReceiverInitializationAfterCallerCancellation() [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_CancelsPendingSubscriberHandshake() + public async Task Shutdown_DrainsPendingSubscriberHandshake() { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, Arg.Any()) @@ -152,11 +152,18 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( var token = await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(token.CanBeCanceled); Assert.False(token.IsCancellationRequested); - await accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var shutdown = accessor.Shutdown(); + await accessor.GetPubSubCache(); + Assert.False(shutdown.IsCompleted); + Assert.False(token.IsCancellationRequested); + Assert.False(handshake.Task.IsCompleted); + await receiver.DidNotReceive().Shutdown(Arg.Any(), Arg.Any()); + handshake.SetResult(null); + await shutdown.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(token.IsCancellationRequested); - Assert.False(handshake.Task.IsCompleted); - Assert.False(data.IsRegistered); + Assert.True(data.IsRegistered); + Assert.Equal(0, data.PendingHandshakes); Assert.Empty(consumer.DeliveredTokens); Assert.Equal(0, cache.DeliveryProgressCallCount); Assert.Empty(await accessor.GetPubSubCache()); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 46407ab2e2f..6aa5bab40bd 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -2926,7 +2926,7 @@ public async Task Shutdown_PreservesHandshakeRewindWhenOlderDeliveryCompletes(st var releaseDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var consumer = new RecordingConsumer { - OnHandshake = () => + OnHandshake = _ => { handshakeStarted.TrySetResult(); return releaseHandshake.Task; @@ -3014,7 +3014,7 @@ public async Task Shutdown_WithholdsProgressUntilRegisteredHandshakeUncertaintyI var handshakes = 0; var consumer = new RecordingConsumer { - OnHandshake = () => + OnHandshake = _ => { if (++handshakes == 1) { @@ -3120,7 +3120,7 @@ public async Task Shutdown_UsesLatestRequestedHandshakeWhenResponsesArriveOutOfO var calls = 0; var consumer = new RecordingConsumer { - OnHandshake = () => + OnHandshake = _ => { if (++calls == 1) { @@ -3304,7 +3304,8 @@ public async Task TerminalSubscriptionRemoval_RevokesInFlightHandshakeOwnership( streamId, new EventSequenceTokenV2(50), cache, new StreamPullingAgentOptions(), failureHandler: failureHandler); var terminalStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var terminalResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - pubSub.UnregisterConsumer(Arg.Any(), Arg.Any(), Arg.Any()) + var agent = (PersistentStreamPullingAgent)accessor; + pubSub.UnregisterConsumerFromProducer(Arg.Any(), Arg.Any(), agent.GrainId, Arg.Any()) .Returns(_ => { terminalStarted.TrySetResult(); @@ -3322,10 +3323,10 @@ public async Task TerminalSubscriptionRemoval_RevokesInFlightHandshakeOwnership( { OnDelivery = _ => operation == "unregister" ? Task.FromException( - (ClientNotAvailableException)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(ClientNotAvailableException))) + Tester.ClientConnectionTests.ClientObserverRoutingTests.CreateUnavailableClientException()) : Task.FromResult( StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(0))), - OnHandshake = () => + OnHandshake = _ => { handshakeStarted.TrySetResult(); return handshakeResult.Task; @@ -3337,24 +3338,24 @@ public async Task TerminalSubscriptionRemoval_RevokesInFlightHandshakeOwnership( data.IsRegistered = true; data.LastProcessedToken = new EventSequenceTokenV2(50); data.Cursor = cache.GetCacheCursor(streamId.StreamId, new EventSequenceTokenV2(100)); - var delivery = accessor.RunConsumerCursor(data); + var replacementConsumer = new ImmediateRecordingConsumer(); + var replacement = stream.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, replacementConsumer, null, DateTime.UtcNow); + replacement.IsRegistered = true; + var replacementCursor = cache.GetCacheCursor(streamId.StreamId, new EventSequenceTokenV2(50)); + replacement.Cursor = replacementCursor; + Task delivery = Task.CompletedTask; Task attachment = Task.CompletedTask; try { - await terminalStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - var replacementConsumer = new ImmediateRecordingConsumer(); - var replacement = stream.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, replacementConsumer, null, DateTime.UtcNow); - replacement.IsRegistered = true; - var replacementCursor = cache.GetCacheCursor(streamId.StreamId, new EventSequenceTokenV2(50)); - replacement.Cursor = replacementCursor; attachment = accessor.AddSubscriber(data); await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + delivery = accessor.RunConsumerCursor(data); + await terminalStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); if (notifyBeforeHandshakeCompletes) { - var agent = (PersistentStreamPullingAgent)accessor; await agent.RunOrQueueTask(() => agent.RemoveSubscriber( data.SubscriptionId, streamId, TestContext.Current.CancellationToken)); Assert.Null(data.Cursor); @@ -3386,12 +3387,12 @@ await agent.RunOrQueueTask(() => agent.RemoveSubscriber( Assert.Equal(new long[] { 50, 100 }, replacementConsumer.DeliveredTokens.Select(token => token.SequenceNumber)); await pubSub.Received(operation == "fault" ? 1 : 0).FaultSubscription( streamId, data.SubscriptionId, Arg.Any()); - await pubSub.Received(operation == "unregister" ? 1 : 0).UnregisterConsumer( - data.SubscriptionId, streamId, Arg.Any()); + await pubSub.Received(operation == "unregister" ? 1 : 0).UnregisterConsumerFromProducer( + data.SubscriptionId, streamId, agent.GrainId, Arg.Any()); await pubSub.DidNotReceive().FaultSubscription( streamId, replacement.SubscriptionId, Arg.Any()); - await pubSub.DidNotReceive().UnregisterConsumer( - replacement.SubscriptionId, streamId, Arg.Any()); + await pubSub.DidNotReceive().UnregisterConsumerFromProducer( + replacement.SubscriptionId, streamId, agent.GrainId, Arg.Any()); } finally { @@ -3757,7 +3758,7 @@ public async Task Shutdown_DrainsExplicitHandshakeWhileAcknowledgingAndSkippingL var handshakes = 0; var consumer = new RecordingConsumer { - OnHandshake = () => + OnHandshake = _ => { handshakes++; started.TrySetResult(); @@ -3838,7 +3839,7 @@ public async Task Shutdown_DrainsRegistrationPinBeforeReceiverCleanup(string out cache.AddToCache([new TestBatchContainer(streamId.StreamId, token)]); var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(call => { @@ -3988,7 +3989,7 @@ public async Task Initialize_WaitsForFullShutdownAndReopensAdmissionForTheNextRu var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var token = new EventSequenceTokenV2(1); - var pubSub = Substitute.For(); + var pubSub = Substitute.For(); pubSub.RegisterProducer(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult>(new HashSet())); var cleanupStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -4016,7 +4017,7 @@ async Task Attach(TaskCompletionSource release) var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var consumer = new RecordingConsumer { - OnHandshake = () => + OnHandshake = _ => { started.TrySetResult(); return release.Task; @@ -4309,13 +4310,12 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg var shutdownTask = testAccessor.Shutdown(); try { - await shutdownTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await testAccessor.GetPubSubCache().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(shutdownTask.IsCompleted); Assert.False(registration.Task.IsCompleted); - Assert.True(receiverShutdownStarted.Task.IsCompletedSuccessfully); + Assert.False(receiverShutdownStarted.Task.IsCompleted); Assert.Empty(queueCache.DeliveryProgressTokens); Assert.Equal(0, queueCache.DeliveryProgressCallCount); - await pubSub.Received(1).UnregisterProducer( - new QualifiedStreamId("provider", streamId), agent.GrainId, Arg.Any()); if (registrationFails) { @@ -4346,6 +4346,8 @@ await pubSub.Received(1).UnregisterProducer( Assert.Empty(queueCache.DeliveryProgressTokens); Assert.Equal(0, queueCache.DeliveryProgressCallCount); await receiver.Received(1).Shutdown(Arg.Any(), Arg.Any()); + await pubSub.Received(1).UnregisterProducer( + new QualifiedStreamId("provider", streamId), agent.GrainId, Arg.Any()); } [TestSuite("BVT")] From 10c1602ac5e1ec0d472f4793d208bf73d9e903f3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 21:25:11 -0700 Subject: [PATCH 87/90] fix(streaming): release excluded Latest cache prefixes --- .../streams-implementation/index.md | 2 + .../Common/PooledCache/PooledQueueCache.cs | 5 +- ...rsistentStreamPullingAgentRecoveryTests.cs | 190 ++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 82014a143fc..000de56bec8 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -99,6 +99,8 @@ An decouples queue reads from consumer delive The cache tracks the earliest contiguous partition position which is safe across active subscriptions. A matching record becomes safe after delivery or intentional filtering. A cursor also advances safely across records for other streams when no earlier matching delivery is pending, so a quiet stream does not pin an otherwise busy partition. Purging must not remove an item still needed by any cursor. uses pressure buckets to stop or slow reads as lag grows instead of discarding undelivered events. Its default capacity is 4,096 batch containers. +A `Latest` subscription attached to a populated recoverable cache starts after its current partition tail. That excluded prefix contributes safe scan progress immediately, allowing a byte-bounded cache to reclaim it once the other subscriptions have also advanced. The first included record remains pending until delivery succeeds or filtering accepts it. A `Latest` cursor created while the cache is empty starts with the first subsequently admitted record. + ```mermaid flowchart TB New[New queue batches] --> Cache[Queue cache] diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index a043a0293c0..0754cb47fba 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -282,8 +282,8 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) LinkedListNode newestBlock = messageBlocks.First!; // messageBlocks.Count != 0 (checked above). - // A cursor which waited on an empty cache starts at the oldest message; - // otherwise, a null token starts at the newest message. + // A cursor which waited on an empty cache includes the first admitted message. + // A new Latest cursor excludes the entire existing partition prefix. if (sequenceToken == null) { if (cursor.State == CursorStates.Idle) @@ -300,6 +300,7 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) cursor.CurrentBlock = newestBlock; cursor.Index = newestBlock.Value.NewestMessageIndex; cursor.SequenceToken = newestBlock.Value.GetNewestSequenceToken(cacheDataAdapter); + cursor.RecordScanned(cursor.SequenceToken); } return null; diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs index 9dfe97a3adb..a9138db657e 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentRecoveryTests.cs @@ -17,6 +17,196 @@ namespace UnitTests.StreamingTests; public partial class PersistentStreamPullingAgentTests { + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task LateLatestSubscriber_AtFullByteBudget_ReleasesExcludedTailAndPreservesDeliveryBarriers( + bool explicitLatest, + bool tailMatchesStream) + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeout = TimeSpan.FromSeconds(10); + var timeProvider = new FakeTimeProvider(DateTimeOffset.UnixEpoch); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var stream = StreamId.Create("namespace", Guid.NewGuid()); + var otherStream = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStream = new QualifiedStreamId("provider", stream); + var messages = Enumerable.Range(1, 9) + .Select(sequence => new RecoveryMessage( + sequence is 5 or 8 || sequence == 3 && !tailMatchesStream ? otherStream : stream, + sequence, + $"payload-{sequence}")) + .ToArray(); + var budget = messages.Take(3).Sum(message => + SegmentBuilder.CalculateAppendSize(message.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + + SegmentBuilder.CalculateAppendSize(message.Payload)); + var source = new BoundedRecoverySource(messages); + var adapter = new RecoverableStreamReceiverTests.TestDataAdapter(); + using var cache = new RecoverableStreamQueueCache( + 3, + new ObjectPool(() => new FixedSizeBuffer(4096)), + adapter, + new RecoverableStreamReceiverTests.NoOpEvictionStrategy(), + NullLogger.Instance, + maxCacheSizeBytes: budget); + var checkpoints = new List(); + var checkpointer = Substitute.For>(); + checkpointer.CheckpointExists.Returns(true); + checkpointer.Load(Arg.Any()).Returns(Task.FromResult("0")); + checkpointer.When(value => value.Update( + Arg.Any(), Arg.Any(), Arg.Any())) + .Do(call => checkpoints.Add(long.Parse(call.Arg(), CultureInfo.InvariantCulture))); + var receiver = new RecoverableStreamReceiver(source, adapter, cache, checkpointer, false); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(queueId).Returns(receiver); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, cancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + var inclusiveConsumer = new RecordingConsumer(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(1))); + var firstDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retryDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retryResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var delivered = new List(); + var attempts = 0; + var latestConsumer = new RecoveryConsumer( + explicitLatest ? StreamHandshakeToken.CreateStartPositionToken(StreamSubscriptionStartPosition.Latest) : null, + batch => + { + delivered.Add(batch.SequenceToken.SequenceNumber); + if (batch.SequenceToken.SequenceNumber != 4) + { + return Task.FromResult(null); + } + + if (++attempts == 1) + { + firstDelivery.SetResult(batch); + return firstResult.Task; + } + + retryDelivery.SetResult(batch); + return retryResult.Task; + }); + using var diagnostics = new DiagnosticEventCollector(StreamingEvents.ListenerName); + await InitializeAgent(agent); + try + { + await accessor.RegisterStream(qualifiedStream, new EventSequenceTokenV2(1), DateTime.UnixEpoch); + await accessor.RegisterStream(new QualifiedStreamId("provider", otherStream), new EventSequenceTokenV2(1), DateTime.UnixEpoch); + var streamData = (await accessor.GetPubSubCache())[qualifiedStream]; + var inclusive = streamData.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + qualifiedStream, inclusiveConsumer, null, DateTime.UnixEpoch); + Assert.True(await accessor.DoHandshakeWithConsumer(inclusive, new EventSequenceTokenV2(1))); + inclusive.IsRegistered = true; + await accessor.RunQueuePump(queueId, cancellationToken); + await inclusiveConsumer.Delivered.Task.WaitAsync(timeout, cancellationToken); + await accessor.GetPubSubCache(); + Assert.Equal(budget, cache.SizeInBytes); + Assert.Equal(3, cache.ItemCount); + Assert.Equal(0, receiver.GetMaxAddCount()); + Assert.Equal(1, source.ReadCount); + Assert.Null(inclusive.LastSafePartitionToken); + + var latest = streamData.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + qualifiedStream, latestConsumer, null, DateTime.UnixEpoch); + Assert.True(await accessor.DoHandshakeWithConsumer(latest, cacheToken: null)); + latest.IsRegistered = true; + await accessor.RunConsumerCursor(latest); + Assert.Equal(StreamConsumerDataState.Inactive, latest.State); + Assert.Null(latest.LastProcessedToken); + Assert.Empty(delivered); + await UpdateProgress(); + Assert.Empty(checkpoints); + Assert.Equal(budget, cache.SizeInBytes); + await accessor.RunQueuePump(queueId, cancellationToken); + Assert.Equal(1, source.ReadCount); + + var inclusiveDrained = WaitForDrain(inclusive.SubscriptionId); + inclusiveConsumer.ReleaseDelivery(); + await inclusiveDrained; + await accessor.GetPubSubCache(); + Assert.Equal(tailMatchesStream ? new long[] { 1, 2, 3 } : [1L, 2L], + inclusiveConsumer.DeliveredTokens.Select(token => token.SequenceNumber)); + await UpdateProgress(); + Assert.Equal(3, Assert.Single(checkpoints)); + Assert.Equal(3, latest.LastSafePartitionToken?.SequenceNumber); + Assert.Null(latest.LastProcessedToken); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal(3, receiver.GetMaxAddCount()); + + var latestDrained = WaitForDrain(latest.SubscriptionId); + await accessor.RunQueuePump(queueId, cancellationToken); + var pending = Assert.IsType(await firstDelivery.Task.WaitAsync(timeout, cancellationToken)); + await accessor.GetPubSubCache(); + Assert.Equal("payload-4", pending.Payload); + Assert.Equal(2, source.ReadCount); + Assert.Equal(6, source.AdmittedThrough); + Assert.Equal(budget, cache.SizeInBytes); + Assert.Equal(3, latest.LastSafePartitionToken?.SequenceNumber); + Assert.Null(latest.LastProcessedToken); + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3 }, checkpoints); + await accessor.RunQueuePump(queueId, cancellationToken); + Assert.Equal(2, source.ReadCount); + + firstResult.SetException(new InvalidOperationException("Retry the first included Latest record.")); + Assert.Same(pending, await retryDelivery.Task.WaitAsync(timeout, cancellationToken)); + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 3 }, checkpoints); + Assert.Equal(budget, cache.SizeInBytes); + retryResult.SetResult(null); + await latestDrained; + await UpdateProgress(); + Assert.Equal(new long[] { 3, 3, 3, 6 }, checkpoints); + Assert.Equal(0, cache.SizeInBytes); + + latestDrained = WaitForDrain(latest.SubscriptionId); + await accessor.RunQueuePump(queueId, cancellationToken); + await latestDrained; + await UpdateProgress(); + Assert.Equal(3, source.ReadCount); + Assert.Equal(9, source.AdmittedThrough); + Assert.Equal(new long[] { 4, 4, 6, 7, 9 }, delivered); + Assert.Equal(new long[] { 3, 3, 3, 6, 9 }, checkpoints); + Assert.Equal(0, cache.SizeInBytes); + Assert.Equal("9", cache.LastPurgedOffset); + Assert.Empty(latestConsumer.Errors); + Assert.Empty(inclusiveConsumer.Errors); + } + finally + { + inclusiveConsumer.ReleaseDelivery(); + firstResult.TrySetResult(null); + retryResult.TrySetResult(null); + await accessor.Shutdown(); + } + + Task WaitForDrain(GuidId subscription) + => diagnostics.WaitForEventAsync( + nameof(StreamingEvents.ConsumerCursorDrained), + entry => entry.Payload is StreamingEvents.ConsumerCursorDrained drained + && drained.StreamId == stream && drained.SubscriptionId == subscription.Guid, + timeout, + cancellationToken); + + async Task UpdateProgress() + { + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await accessor.GetPubSubCache(); + } + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] From 77464b818fd0bd0b0a3854f8e6004cf07059e580 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 21:30:07 -0700 Subject: [PATCH 88/90] fix(streaming): drain admitted work before ending processing lifetime --- .../PersistentStreamPullingAgent.cs | 12 ++++++------ .../PersistentStreamPullingAgentAdmissionTests.cs | 1 + .../PersistentStreamPullingAgentTests.cs | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 4ad3ddf812e..1df00e04b4d 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -466,7 +466,7 @@ private async Task DoHandshakeWithConsumer( try { requestedHandshakeToken = await AsyncExecutorWithRetries.ExecuteWithRetries( - i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId, ShutdownToken).WaitAsync(ShutdownToken), + i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId, ShutdownToken), AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException (exception, i) => exception is not ClientNotAvailableException && !IsShutdown @@ -1845,7 +1845,7 @@ private sealed class QueueCacheCursorContractException(string message) : Invalid { try { - StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, handshakeToken, cancellationToken).WaitAsync(cancellationToken); + StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, handshakeToken, cancellationToken); StreamingEvents.EmitMessageDelivered(streamProviderName, consumerData, batch, Silo); return newToken; @@ -1911,7 +1911,7 @@ private static async Task DeliverErrorToConsumer( RequestContext.Clear(); // clear RequestContext before await! } } - await errorDeliveryTask.WaitAsync(cancellationToken); + await errorDeliveryTask; } private async Task ErrorProtocol( @@ -1943,12 +1943,12 @@ private async Task ErrorProtocol( if (isDeliveryError) { await streamFailureHandler.OnDeliveryFailure( - consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } else { await streamFailureHandler.OnSubscriptionFailure( - consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } if (!IsCurrent()) return false; @@ -2057,7 +2057,7 @@ private static async Task> PubsubRegisterProducer( { try { - var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer, cancellationToken).WaitAsync(cancellationToken); + var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer, cancellationToken); return streamData; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs index a93548e84b2..24b28ccc4da 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentAdmissionTests.cs @@ -163,6 +163,7 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( Assert.True(token.IsCancellationRequested); Assert.True(data.IsRegistered); + Assert.True(handshake.Task.IsCompletedSuccessfully); Assert.Equal(0, data.PendingHandshakes); Assert.Empty(consumer.DeliveredTokens); Assert.Equal(0, cache.DeliveryProgressCallCount); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 6aa5bab40bd..5ff156f107c 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -4316,6 +4316,8 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations(bool reg Assert.False(receiverShutdownStarted.Task.IsCompleted); Assert.Empty(queueCache.DeliveryProgressTokens); Assert.Equal(0, queueCache.DeliveryProgressCallCount); + await pubSub.DidNotReceive().UnregisterProducer( + new QualifiedStreamId("provider", streamId), agent.GrainId, Arg.Any()); if (registrationFails) { From 2bb8106ad2ef70fdaef031c727344098a49f0339 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 22:36:24 -0700 Subject: [PATCH 89/90] fix(streaming): release Kinesis resources after shutdown cancellation --- src/AWS/Orleans.Streaming.Kinesis/README.md | 4 + .../Streams/KinesisPooledAdapterReceiver.cs | 53 ++++--- .../KinesisPooledRuntimeTests.cs | 132 ++++++++++++++++++ 3 files changed, 171 insertions(+), 18 deletions(-) diff --git a/src/AWS/Orleans.Streaming.Kinesis/README.md b/src/AWS/Orleans.Streaming.Kinesis/README.md index 08a31bc2c56..1bae321e41f 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/README.md +++ b/src/AWS/Orleans.Streaming.Kinesis/README.md @@ -101,6 +101,10 @@ The DynamoDB checkpointer uses on-demand billing and creates its table by defaul To provide a different checkpoint implementation, use the configurator overload and call `ConfigureCheckpointer` with an `IStreamQueueCheckpointerFactory`. +## Receiver shutdown + +Once shutdown is admitted, the receiver releases its Kinesis client and pooled cache before making the shard available for reassignment. An expired shutdown budget or caller cancellation is reported while final resource cleanup proceeds. A replacement receiver resumes from the last durable checkpoint. + ## Documentation - [Microsoft Orleans documentation](https://dotnet.github.io/orleans/docs/) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs index aeb2d3f163a..72fd216f169 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -151,37 +151,49 @@ private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken initializationTask = _initializationTask; } - if (initializationTask is not null) + try { - try + if (initializationTask is not null) { await initializationTask.WaitAsync(cancellationToken); } - catch (OperationCanceledException) - when (_lifecycleCancellation.IsCancellationRequested - && !cancellationToken.IsCancellationRequested) - { - } - catch (Exception exception) - { - (exceptions ??= []).Add(exception); - } + + cancellationToken.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); } try { if (_inner is null) { - await _source.Shutdown(cancellationToken); + try + { + await _source.Shutdown(cancellationToken); + } + finally + { + _cache.Dispose(); + } } else { - var remaining = timeout == Timeout.InfiniteTimeSpan - ? Timeout.InfiniteTimeSpan - : timeout > shutdownWatch.Elapsed - ? timeout - shutdownWatch.Elapsed - : TimeSpan.Zero; - await ((IQueueAdapterReceiver)_inner).Shutdown(remaining, cancellationToken); + var remaining = cancellationToken.IsCancellationRequested + ? TimeSpan.Zero + : timeout == Timeout.InfiniteTimeSpan + ? Timeout.InfiniteTimeSpan + : timeout > shutdownWatch.Elapsed + ? timeout - shutdownWatch.Elapsed + : TimeSpan.Zero; + // Enter cleanup even when the wait budget has expired; the inner receiver releases resources after timeout. + await ((IQueueAdapterReceiver)_inner).Shutdown(remaining, CancellationToken.None); } } catch (Exception exception) @@ -189,6 +201,11 @@ private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken (exceptions ??= []).Add(exception); } + if (exceptions is null) + { + cancellationToken.ThrowIfCancellationRequested(); + } + if (exceptions is [var singleException]) { ExceptionDispatchInfo.Capture(singleException).Throw(); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs index 4540c76b29a..1978247c655 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisPooledRuntimeTests.cs @@ -467,6 +467,138 @@ public async Task PooledReceiver_CanceledTopologyDiscoveryReleasesMonitorForReas await fixture.DiscoveryClient.Received(3).ListShardsAsync(Arg.Any(), Arg.Any()); } + [Theory] + [InlineData("created")] + [InlineData("initializing")] + [InlineData("initialized")] + [InlineData("flushing")] + public async Task PooledReceiver_ShutdownCancellationReleasesResourcesBeforeRemoval(string state) + { + await using var fixture = new Fixture(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellation); + var client = Substitute.For(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var initialization = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(_ => + { + started.TrySetResult(); + return state == "initializing" + ? initialization.Task + : Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator" }); + }); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(Response("tail", fixture.Record("10", "retained"))); + if (state == "flushing") + { + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + checkpointer.FlushAsync(Arg.Any()).Returns(_ => + { + Assert.False(cancellation.IsCancellationRequested); + cancellation.Cancel(); + return Task.CompletedTask; + }); + fixture.CheckpointerFactory.Configure().Create("shard-1", Arg.Any()) + .Returns(checkpointer); + } + + var disposed = false; + client.When(value => value.Dispose()).Do(_ => disposed = true); + var removalCount = 0; + var disposedAtRemoval = false; + var capacityAtRemoval = -1; + var receiver = new KinesisPooledAdapterReceiver( + client, "stream", "shard-1", fixture.CheckpointerFactory, + new SimpleQueueCacheOptions { CacheSize = 8 }, fixture.Serializer, NullLoggerFactory.Instance, + new KinesisShardTopologyMonitor( + fixture.DiscoveryClient, "stream", ["shard-1"], TimeSpan.FromMinutes(1), + fixture.Time, NullLogger.Instance), + TimeSpan.Zero, fixture.Time, + onShutdown: current => + { + removalCount++; + disposedAtRemoval = disposed; + capacityAtRemoval = current.GetMaxAddCount(); + }, + maxCacheSizeBytes: 1); + Task initialize = Task.CompletedTask; + try + { + if (state == "initializing") + { + initialize = ((IQueueAdapterReceiver)receiver).Initialize(Timeout.InfiniteTimeSpan, TestCancellation); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), TestCancellation); + } + else if (state is "initialized" or "flushing") + { + Assert.Single(await receiver.GetQueueMessagesAsync(1, TestCancellation)); + Assert.True(receiver.IsUnderPressure()); + Assert.Equal(0, receiver.GetMaxAddCount()); + } + + // Cancel the caller after shutdown admission, while the receiver closes its lifecycle. + using var cancelOnShutdown = state == "flushing" + ? default + : receiver.LifecycleCancellationToken.Register(cancellation.Cancel); + var error = await Xunit.Record.ExceptionAsync(() => ((IQueueAdapterReceiver)receiver) + .Shutdown(TimeSpan.FromSeconds(5), cancellation.Token) + .WaitAsync(TimeSpan.FromSeconds(5), TestCancellation)); + + Assert.NotNull(error); + var errors = error is AggregateException aggregate + ? aggregate.Flatten().InnerExceptions.ToArray() + : [error]; + Assert.All(errors, exception => Assert.IsAssignableFrom(exception)); + Assert.True(cancellation.IsCancellationRequested); + Assert.All(errors, exception => Assert.True(((OperationCanceledException)exception).CancellationToken.IsCancellationRequested)); + Assert.True(disposedAtRemoval); + Assert.Equal(8, capacityAtRemoval); + Assert.Equal(1, removalCount); + Assert.False(receiver.IsUnderPressure()); + client.Received(1).Dispose(); + Assert.Empty(await receiver.GetQueueMessagesAsync(1, TestCancellation)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, removalCount); + client.Received(1).Dispose(); + } + finally + { + initialization.TrySetResult(new GetShardIteratorResponse { ShardIterator = "late" }); + await initialize.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + if (!disposed) + { + client.Dispose(); + } + } + } + + [Fact] + public async Task PooledFactory_CancellationBeforeShutdownAdmissionPreservesReceiver() + { + await using var fixture = new Fixture(maxCacheSizeBytes: 1); + await fixture.Factory.CreateAdapter(TestCancellation); + var receiver = fixture.GetReceiver(); + var client = Assert.Single(fixture.ReceiverClients); + fixture.SetReads(client, () => Response("tail", fixture.Record("10", "retained"))); + Assert.Single(await receiver.GetQueueMessagesAsync(1, TestCancellation)); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var error = await Assert.ThrowsAnyAsync( + () => ((IQueueAdapterReceiver)receiver).Shutdown(TimeSpan.FromSeconds(5), cancellation.Token)); + + Assert.Equal(cancellation.Token, error.CancellationToken); + Assert.Same(receiver, fixture.GetReceiver()); + Assert.True(receiver.IsUnderPressure()); + client.DidNotReceive().Dispose(); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.False(receiver.IsUnderPressure()); + client.Received(1).Dispose(); + Assert.NotSame(receiver, fixture.GetReceiver()); + } + [Fact] public async Task PooledReceiver_MixedTypeNonzeroIndicesPersistSafeWatermarkAndRestartAtNextRecord() { From 5b02a6bd7bf84d9162770b51b3c2606bf2be4f69 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 23:20:09 -0700 Subject: [PATCH 90/90] fix(streaming): preserve ADO.NET receiver shutdown ownership --- .../AdoNetQueueAdapterReceiver.cs | 15 +- src/AdoNet/Orleans.Streaming.AdoNet/README.md | 2 +- .../Streaming/AdoNetRecoverableStreamTests.cs | 170 ++++++++++++++++++ 3 files changed, 184 insertions(+), 3 deletions(-) diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 57d5e22429e..ae7a77ea860 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -10,6 +10,8 @@ internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueue private const int BufferSize = 1024 * 1024; private readonly RecoverableStreamReceiver _inner; private readonly AdoNetRecoverableStream _source; + private readonly object _shutdownLock = new(); + private Task? _shutdownTask; private int _shutdownNotified; internal Action? OnShutdown { get; set; } @@ -71,11 +73,20 @@ Task IQueueAdapterReceiver.Initialize(TimeSpan timeout, CancellationToken cancel Task IQueueAdapterReceiver.Shutdown(TimeSpan timeout, CancellationToken cancellationToken) => ShutdownWithCancellation(timeout, cancellationToken); - private async Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) + private Task ShutdownWithCancellation(TimeSpan timeout, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_shutdownLock) + { + return _shutdownTask ??= ShutdownCore(timeout); + } + } + + private async Task ShutdownCore(TimeSpan timeout) { try { - await ((IQueueAdapterReceiver)_inner).Shutdown(timeout, cancellationToken); + await ((IQueueAdapterReceiver)_inner).Shutdown(timeout, CancellationToken.None); } finally { diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index c96cf6c7f01..e92e8b77fea 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -136,7 +136,7 @@ public class ConsumerGrain : Grain, IConsumerGrain, IAsyncObserver During shutdown or queue reassignment, the pulling agent closes processing admission and drains admitted queue reads, registrations, delivery, and durable subscription retirement. It preserves pending-registration and handshake checkpoint barriers before the receiver flushes durable progress and releases its pooled cache resources. -The receiver bounds its asynchronous shutdown work using the supplied timeout. Receiver replacement waits for outstanding partition acquisition to settle. The next receiver acquires a fresh ownership epoch and resumes after the durable checkpoint, with retained history governed by the configured retention policy. +Cancellation before shutdown admission preserves the receiver and its registration. An admitted shutdown uses the first caller's timeout and completes cleanup independently of later caller cancellation; concurrent shutdown calls share that operation. Receiver replacement waits for cleanup and outstanding partition acquisition to settle. The next receiver acquires a fresh ownership epoch and resumes after the durable checkpoint, with retained history governed by the configured retention policy. ## Documentation For more comprehensive documentation, please refer to: diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs index 41413295641..d805a428068 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -2,10 +2,13 @@ using System.Data.Common; using System.Reflection; using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using Orleans.Configuration; using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; using Orleans.Streaming.AdoNet; using Orleans.Streaming.AdoNet.Storage; using Orleans.Streams; @@ -19,6 +22,142 @@ namespace Tester.AdoNet.Streaming; [TestArea("Streaming")] public class AdoNetRecoverableStreamTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Receiver_CanceledShutdownPreservesRegistrationUntilAcceptedCleanup(bool initializeFirst) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var storage = new CapturingRelationalStorage + { + PartitionRecord = PartitionRecord(0, 2, 1, 1), + ReadRecords = [MessageRecord(1)], + }; + var notifications = 0; + var capacityAtRemoval = -1; + var queueId = QueueId.GetQueueId("queue", 0, 0); + var registry = CreateReceiverRegistry(storage, services.GetRequiredService>(), + queueId, receiver => + { + notifications++; + capacityAtRemoval = receiver.GetMaxAddCount(); + }); + var receiver = registry.GetOrCreate(queueId); + AdoNetQueueAdapterReceiver? replacement = null; + try + { + if (initializeFirst) + { + Assert.Single(await receiver.GetQueueMessagesAsync(1, TestContext.Current.CancellationToken)); + } + + var reads = storage.ReadCallCount; + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => ((IQueueAdapterReceiver)receiver).Shutdown(TimeSpan.FromSeconds(5), cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, notifications); + Assert.Same(receiver, registry.GetOrCreate(queueId)); + Assert.Equal(reads, storage.ReadCallCount); + if (!initializeFirst) + { + Assert.Single(await receiver.GetQueueMessagesAsync(1, TestContext.Current.CancellationToken)); + } + + Assert.True(receiver.IsUnderPressure()); + Assert.Equal(0, receiver.GetMaxAddCount()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, notifications); + Assert.Equal(8, capacityAtRemoval); + Assert.False(receiver.IsUnderPressure()); + replacement = registry.GetOrCreate(queueId); + Assert.NotSame(receiver, replacement); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, notifications); + Assert.Same(replacement, registry.GetOrCreate(queueId)); + } + finally + { + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + if (replacement is not null) + { + await replacement.Shutdown(TimeSpan.FromSeconds(5)); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Receiver_ConcurrentShutdownWaitsForAdmittedCheckpointFlush(bool cancelCaller) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var writes = 0; + var storage = new CapturingRelationalStorage + { + PartitionRecord = PartitionRecord(0, 2, 1, 1), + ReadRecords = [MessageRecord(1)], + OnCheckpointWrite = _ => + { + writes++; + started.TrySetResult(); + return release.Task; + }, + }; + var notifications = 0; + var queueId = QueueId.GetQueueId("queue", 0, 0); + var registry = CreateReceiverRegistry(storage, services.GetRequiredService>(), + queueId, _ => notifications++); + var receiver = registry.GetOrCreate(queueId); + AdoNetQueueAdapterReceiver? replacement = null; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + Task? first = null; + try + { + Assert.Single(await receiver.GetQueueMessagesAsync(1, TestContext.Current.CancellationToken)); + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(1), DateTime.UtcNow); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + first = ((IQueueAdapterReceiver)receiver).Shutdown(Timeout.InfiniteTimeSpan, cancellation.Token); + if (cancelCaller) + { + cancellation.Cancel(); + } + + var second = receiver.Shutdown(Timeout.InfiniteTimeSpan); + Assert.Same(first, second); + Assert.False(first.IsCompleted); + Assert.Equal(0, notifications); + Assert.Same(receiver, registry.GetOrCreate(queueId)); + Assert.Equal(1, writes); + release.SetResult(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(1, notifications); + Assert.Equal(1, writes); + Assert.False(receiver.IsUnderPressure()); + replacement = registry.GetOrCreate(queueId); + Assert.NotSame(receiver, replacement); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, notifications); + Assert.Same(replacement, registry.GetOrCreate(queueId)); + } + finally + { + release.TrySetResult(); + await (first ?? receiver.Shutdown(TimeSpan.FromSeconds(5))) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (replacement is not null) + { + await replacement.Shutdown(TimeSpan.FromSeconds(5)); + } + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -663,6 +802,31 @@ private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage return new RelationalOrleansQueries(storage, new DbStoredQueries(queryValues)); } + private static QueueAdapterReceiverRegistry CreateReceiverRegistry( + IRelationalStorage storage, + Serializer serializer, + QueueId queueId, + Action onShutdown) + { + QueueAdapterReceiverRegistry registry = null!; + registry = new(_ => + { + var receiver = new AdoNetQueueAdapterReceiver( + "provider", "queue", + new AdoNetStreamOptions { StartFromNow = false, MaxMessagesPerRead = 8, MaxCacheSizeBytes = 1 }, + new ClusterOptions { ServiceId = "service" }, + new SimpleQueueCacheOptions { CacheSize = 8 }, + CreateQueries(storage), serializer, NullLogger.Instance); + receiver.OnShutdown = current => + { + onShutdown(current); + registry.Remove(queueId, current); + }; + return receiver; + }); + return registry; + } + private static IDataRecord Record(params (string Name, object? Value)[] values) => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); @@ -849,6 +1013,8 @@ private sealed class CapturingRelationalStorage : IRelationalStorage public Action? OnCleanup { get; set; } + public Func? OnCheckpointWrite { get; set; } + public IReadOnlyList? CleanupRecords { get; set; } public int ReadCallCount { get; private set; } @@ -875,6 +1041,10 @@ public async Task> ReadAsync( { OnCleanup?.Invoke(); } + else if (query == nameof(DbStoredQueries.AdvanceStreamCheckpointKey) && OnCheckpointWrite is { } onWrite) + { + await onWrite(cancellationToken); + } var records = query switch {