From 4c197906a9469fb32c3a40f834112b73cb46b901 Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 12:14:44 -0700 Subject: [PATCH 1/9] fix(streaming): advance checkpoints past drained subscriptions - Track accounted read boundaries and explicitly drained consumers. - Preserve replay floors across registration, delivery, and cursor errors. - Honor earlier and unknown replay requests before repositioning cursors. - Cover shutdown progress, pending deliveries, and recovery regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 341 ++++++-- .../QueueStreamDataStructures.cs | 15 +- .../PersistentStreamPullingAgentTests.cs | 826 +++++++++++++++++- 3 files changed, 1110 insertions(+), 72 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index e441c795b19..b2967798000 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -46,6 +46,9 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; + private StreamSequenceToken? _lastReadToken; + private StreamSequenceToken? _retainedReplayToken; + private bool _hasUnknownReplayPosition; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -155,6 +158,9 @@ public Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; + _lastReadToken = null; + _retainedReplayToken = null; + _hasUnknownReplayPosition = false; lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -345,14 +351,23 @@ private async Task AddSubscriber_Impl( StreamingEvents.EmitSubscriptionAdded(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); } - if (await DoHandshakeWithConsumer(data, cacheToken)) + data.IsCaughtUp = false; + data.PendingHandshakes++; + try + { + if (await DoHandshakeWithConsumer(data, cacheToken)) + { + RecordDeliveryProgress(data, GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken)); + data.PendingStartToken = null; + data.IsRegistered = true; + StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); + if (data.State == StreamConsumerDataState.Inactive) + RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + } + } + finally { - data.LastProcessedToken = GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken); - data.PendingStartToken = null; - data.IsRegistered = true; - StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); - if (data.State == StreamConsumerDataState.Inactive) - RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + data.PendingHandshakes--; } } @@ -406,6 +421,7 @@ private async Task DoHandshakeWithConsumer( && startPositionToken.Equals(consumerData.LastToken); if (!preservesCurrentPosition) { + RecordDeliveryProgress(consumerData, null); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetCacheCursorAtStartPosition( consumerData.StreamId, @@ -416,6 +432,8 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { + RecordDeliveryProgress( + consumerData, GetInitialDeliveryProgress(effectiveHandshakeToken, consumerData.LastProcessedToken)); consumerData.SafeDisposeCursor(logger); if (effectiveHandshakeToken is DeliveryToken || effectiveHandshakeToken is StartToken @@ -902,37 +920,84 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // Receivers built against older Orleans versions can still return null. if (multiBatch is null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback. - queueCache?.AddToCache(multiBatch); - numMessages += multiBatch.Count; - if (_streamInstruments?.PersistentStreamReadMessages.Enabled is true) + var accountedFor = false; + try + { + queueCache?.AddToCache(multiBatch); + numMessages += multiBatch.Count; + if (_streamInstruments?.PersistentStreamReadMessages.Enabled is true) + { + tags = StreamInstrumentsTagUtils.InitializeTags(myQueueId, streamProviderName); + _streamInstruments.PersistentStreamReadMessages.Add(multiBatch.Count, tags.Value); + } + + LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); + + foreach (var group in + multiBatch + .Where(m => m is not null) + .GroupBy(container => container.StreamId)) + { + var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); + StreamSequenceToken startToken = group.First().SequenceToken; + if (pubSubCache.TryGetValue(streamId, out var streamData)) + { + streamData.RefreshActivity(now); + StartInactiveCursors(streamData, startToken, CancellationToken.None); + } + else + { + // Run registration in the background so that cold-stream pubsub + // calls do not stall message delivery for other streams on the same queue. + RegisterStream(streamId, startToken, now, CancellationToken.None); + } + } + + accountedFor = true; + } + finally { - tags = StreamInstrumentsTagUtils.InitializeTags(myQueueId, streamProviderName); - _streamInstruments.PersistentStreamReadMessages.Add(multiBatch.Count, tags.Value); + // The receiver has consumed this read even if cache insertion or registration fails. + if (!accountedFor) + { + foreach (var batch in multiBatch) + { + if (batch is not null) + { + RetainReplayPosition(batch.SequenceToken); + } + } + } } - LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); + if (IsShutdown || cancellationToken.IsCancellationRequested) + { + return false; + } - foreach (var group in - multiBatch - .Where(m => m is not null) - .GroupBy(container => container.StreamId)) + // Only publish the read boundary after every stream has been accounted for. + foreach (var batch in multiBatch) { - var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); - StreamSequenceToken startToken = group.First().SequenceToken; - if (pubSubCache.TryGetValue(streamId, out var streamData)) + if (batch?.SequenceToken is not { } token) { - streamData.RefreshActivity(now); - StartInactiveCursors(streamData, startToken, CancellationToken.None); + continue; } - else + + if (_lastReadToken is null) + { + _lastReadToken = token; + } + else if (!TryCompareQueueProgress(token, _lastReadToken, out var comparison)) + { + _hasUnknownReplayPosition = true; + } + else if (comparison > 0) { - // Run registration in the background so that cold-stream pubsub - // calls do not stall message delivery for other streams on the same queue. - RegisterStream(streamId, startToken, now, CancellationToken.None); + _lastReadToken = token; } } - return !IsShutdown && !cancellationToken.IsCancellationRequested; + return true; } private void CleanupPubSubCache(DateTime now) @@ -956,6 +1021,16 @@ private void CleanupPubSubCache(DateTime now) { if (pubSubCache.Remove(streamId, out var streamData)) { + // Cleanup must not erase replay requirements left by interrupted delivery. + if (!TryGetStreamDeliveryProgress(streamData, out var requiredToken)) + { + RetainReplayPosition(null); + } + else if (requiredToken is not null) + { + RetainReplayPosition(requiredToken); + } + streamData.DisposeAll(logger); StreamingEvents.EmitStreamInactive(streamProviderName, streamId.StreamId, options.StreamInactivityPeriod, Silo); } @@ -978,54 +1053,150 @@ private void NotifyDeliveryProgress() private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) { - earliest = null; + earliest = _lastReadToken; + if (_hasUnknownReplayPosition) + { + return false; + } + + if (_retainedReplayToken is { } retained && !IncludeProgress(retained, ref earliest)) + { + return false; + } foreach (var streamConsumers in pubSubCache.Values) { - if (streamConsumers.RegistrationTask is { IsCompleted: false }) + if (!TryGetStreamDeliveryProgress(streamConsumers, out var current) + || current is not null && !IncludeProgress(current, ref earliest)) { return false; } + } + + return true; + } + + private bool TryGetStreamDeliveryProgress(StreamConsumerCollection streamConsumers, out StreamSequenceToken? earliest) + { + earliest = null; + if (!streamConsumers.StreamRegistered + || streamConsumers.RegistrationTask is { IsCompleted: false }) + { + return false; + } - foreach (var consumer in streamConsumers.AllConsumers()) + foreach (var consumer in streamConsumers.AllConsumers()) + { + if (!consumer.IsRegistered || consumer.PendingHandshakes != 0) { - if (!consumer.IsRegistered) - { - return false; - } + return false; + } - var current = consumer.LastProcessedToken; - if (current is null) - { - return false; - } + // A drained subscription's last matching event need not pin other streams. + if (consumer.IsCaughtUp && !consumer.HasDeliveryProgressError) + { + continue; + } - if (earliest is null) - { - earliest = current; - continue; - } + var current = consumer.LastProcessedToken; + if (current is null) + { + return false; + } - try - { - if (IsBefore(current, earliest)) - { - earliest = current; - } - } - catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") + if (earliest is null) + { + earliest = current; + continue; + } + + try + { + if (IsBefore(current, earliest)) { - LogWarningIncompatibleDeliveryProgress( - new(QueueId), consumer.SubscriptionId, consumer.StreamId, current, earliest, exception); - earliest = null; - return false; + earliest = current; } } + catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") + { + LogWarningIncompatibleDeliveryProgress( + new(QueueId), consumer.SubscriptionId, consumer.StreamId, current, earliest, exception); + earliest = null; + return false; + } } return true; } + private bool IncludeProgress(StreamSequenceToken token, ref StreamSequenceToken? earliest) + { + if (earliest is null) + { + earliest = token; + } + else if (!TryCompareQueueProgress(token, earliest, out var comparison)) + { + return false; + } + else if (comparison < 0) + { + earliest = token; + } + + return true; + } + + private bool TryCompareQueueProgress(StreamSequenceToken token, StreamSequenceToken other, out int comparison) + { + try + { + comparison = EventSequenceTokenCompatibility.Compare(token, other); + return true; + } + catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") + { + LogWarningIncompatibleQueueProgress(new(QueueId), token, other, exception); + comparison = default; + return false; + } + } + + private void RetainReplayPosition(StreamSequenceToken? token) + { + if (token is null || !IncludeProgress(token, ref _retainedReplayToken)) + { + _hasUnknownReplayPosition = true; + } + } + + private void RecordDeliveryProgress(StreamConsumerData consumer, StreamSequenceToken? token) + { + // A reset cursor's later progress cannot prove that it did not skip earlier data. + // An earlier or unknown replay request must still lower the retained checkpoint floor. + if (!consumer.HasDeliveryProgressError || token is null) + { + consumer.LastProcessedToken = token; + } + else if (consumer.LastProcessedToken is { } current) + { + if (!TryCompareQueueProgress(token, current, out var comparison)) + { + consumer.LastProcessedToken = null; + } + else if (comparison < 0) + { + consumer.LastProcessedToken = token; + } + } + } + + private static void RecordDeliveryProgressError(StreamConsumerData consumer) + { + consumer.HasDeliveryProgressError = true; + consumer.IsCaughtUp = false; + } + private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => EventSequenceTokenCompatibility.Compare(current, other) < 0; @@ -1106,6 +1277,7 @@ async Task RegisterStreamAsync() void RemoveCanceledRegistration() { + RetainReplayPosition(firstToken); if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) && ReferenceEquals(cachedStreamData, streamData)) { @@ -1120,6 +1292,8 @@ void FailRegistration(Exception exception) LogWarningFailedToRegisterStream(streamId, exception); StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); + RetainReplayPosition(firstToken); + // Only reached when producer registration itself fails. if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) && ReferenceEquals(cachedStreamData, streamData)) @@ -1143,6 +1317,7 @@ async Task SubscribeWithIsolation(PubSubSubscriptionState item) } catch (Exception exception) { + RetainReplayPosition(firstToken); LogWarningFailedToAddSubscription(item.Stream, exception); } } @@ -1158,6 +1333,7 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq } // Some consumer might not be fully registered yet + consumerData.IsCaughtUp = false; if (consumerData.IsRegistered) { consumerData.Cursor?.Refresh(startToken); @@ -1203,6 +1379,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati nextBatch = GetBatchForConsumer(consumerData); if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { + RecordDeliveryProgressError(consumerData); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetCacheMissRecoveryCursor(consumerData); continue; @@ -1215,6 +1392,9 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (!nextBatch.HasProgress) { + consumerData.IsCaughtUp = nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.NoData + && !consumerData.HasDeliveryProgressError + && consumerData.UnconfirmedDeliveryToken is null; // Only emit cursor-drained when we transitioned from delivering to empty, // not on every empty poll. if (deliveredAny) @@ -1224,11 +1404,13 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (QueueCacheCursorContractException) { + RecordDeliveryProgressError(consumerData); consumerData.SafeDisposeCursor(logger); throw; } catch (Exception exc) { + RecordDeliveryProgressError(consumerData); exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetRecoveryCursor(consumerData); @@ -1240,7 +1422,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is null) { - consumerData.LastProcessedToken = nextBatch.ProgressToken; + RecordDeliveryProgress(consumerData, nextBatch.ProgressToken); continue; } } @@ -1262,6 +1444,12 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is not null) { var batch = nextBatch.Batch; + if (consumerData.UnconfirmedDeliveryToken is null + || IsBefore(consumerData.UnconfirmedDeliveryToken, nextBatch.ProgressToken!)) + { + consumerData.UnconfirmedDeliveryToken = nextBatch.ProgressToken; + } + StreamHandshakeToken? newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( i => DeliverBatchToConsumer(consumerData, batch, cancellationToken), AsyncExecutorWithRetries.INFINITE_RETRIES, @@ -1270,6 +1458,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati this.options.MaxEventDeliveryTime, deliveryBackoffProvider, cancellationToken: cancellationToken); + StreamSequenceToken? acknowledgedToken = null; if (newToken is not null) { consumerData.LastToken = newToken; @@ -1277,7 +1466,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati IBatchContainer? pendingBatch = null; if (newToken is StartPositionToken startPositionToken) { - consumerData.LastProcessedToken = null; + RecordDeliveryProgress(consumerData, null); try { newCursor = GetCacheCursorAtStartPosition( @@ -1297,7 +1486,8 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati ?? throw new InvalidOperationException("A start handshake token must contain a stream sequence token."); if (SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { - consumerData.LastProcessedToken = sequenceToken; + acknowledgedToken = sequenceToken; + RecordDeliveryProgress(consumerData, acknowledgedToken); // An implicit recovery token identifies the last event processed by the prior activation. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( @@ -1308,7 +1498,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } else { - consumerData.LastProcessedToken = null; + RecordDeliveryProgress(consumerData, null); newCursor = GetCacheCursorOrThrow(consumerData.StreamId, sequenceToken); } } @@ -1316,7 +1506,8 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati { var sequenceToken = newToken.Token ?? throw new InvalidOperationException("A delivery handshake token must contain a stream sequence token."); - consumerData.LastProcessedToken = sequenceToken; + acknowledgedToken = sequenceToken; + RecordDeliveryProgress(consumerData, acknowledgedToken); // The handshake token points to an already processed event, so advance past it. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( @@ -1337,13 +1528,22 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } else { - // Track progress for the periodic delivery scan. - consumerData.LastProcessedToken = nextBatch.ProgressToken; + acknowledgedToken = nextBatch.ProgressToken; + RecordDeliveryProgress(consumerData, acknowledgedToken); + } + + // A rewind or an explicit start request does not acknowledge the attempted batch. + if (acknowledgedToken is { } processed + && consumerData.UnconfirmedDeliveryToken is { } attempted + && !IsBefore(processed, attempted)) + { + consumerData.UnconfirmedDeliveryToken = null; } } } catch (Exception exc) { + RecordDeliveryProgressError(consumerData); if (batchCursor is not null && nextBatch.Batch is not null) { batchCursor.RecordDeliveryFailure(nextBatch.Batch); @@ -1382,6 +1582,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (Exception exc) { + RecordDeliveryProgressError(consumerData); // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception LogErrorRunConsumerCursor(exc); consumerData.State = StreamConsumerDataState.Inactive; @@ -1447,6 +1648,7 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) int i = 0; var batchContainers = new List(); StreamSequenceToken? progressToken = null; + var sawBatch = pendingBatch is not null; if (pendingBatch is not null) { @@ -1478,6 +1680,7 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) var batchContainer = cursor.GetCurrent(out _) ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); + sawBatch = true; progressToken = batchContainer.SequenceToken; if (!ShouldDeliverBatch(streamId, batchContainer, filterData)) @@ -1489,7 +1692,7 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) if (progressToken is null) { - return new ConsumerBatch(QueueCacheCursorMoveResult.NoData); + return new ConsumerBatch(sawBatch ? QueueCacheCursorMoveResult.Success : QueueCacheCursorMoveResult.NoData); } return i == 0 @@ -1885,6 +2088,16 @@ private partial void LogWarningIncompatibleDeliveryProgress( StreamSequenceToken otherToken, Exception exception); + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Retaining the previous delivery watermark for queue {Queue}: token {Token} is incompatible with {OtherToken}." + )] + private partial void LogWarningIncompatibleQueueProgress( + QueueIdLogRecord queue, + StreamSequenceToken token, + StreamSequenceToken otherToken, + Exception exception); + [LoggerMessage( Level = LogLevel.Warning, EventId = (int)ErrorCode.PersistentStreamPullingAgent_13, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index e4c82054321..07d7d0c8969 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -40,12 +40,22 @@ internal sealed class StreamConsumerData public bool StartPositionIsProviderDefault; /// - /// The sequence token of the last batch processed (delivered or filtered) by this subscription. - /// Used by the pulling agent's periodic scan to compute the delivery-based checkpoint watermark. + /// The last proven-safe delivery position for this subscription. + /// Cannot advance after an unresolved delivery or cursor error, but earlier replay requests can lower it. /// [NonSerialized] public StreamSequenceToken? LastProcessedToken; + // Inactive also describes interrupted delivery, so it does not prove that the cursor drained. + [NonSerialized] + public bool IsCaughtUp; + [NonSerialized] + public int PendingHandshakes; + [NonSerialized] + public bool HasDeliveryProgressError; + [NonSerialized] + public StreamSequenceToken? UnconfirmedDeliveryToken; + public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { SubscriptionId = subscriptionId; @@ -56,6 +66,7 @@ public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, ISt internal void SafeDisposeCursor(ILogger logger) { + IsCaughtUp = false; PendingBatch = null; if (Cursor is { } cursor) { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 60aece238a5..ba1559e7467 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -399,7 +399,8 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterReceiver? receiver = null, IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, - StreamPullingAgentOptions? options = null) + StreamPullingAgentOptions? options = null, + IStreamFilter? streamFilter = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -436,7 +437,7 @@ private static PersistentStreamPullingAgent CreateAgent( SystemTargetGrainId.Create(SystemTargetGrainId.CreateGrainType("persistent-stream-pulling-agent-test"), siloAddress), "provider", pubSub!, - new NoOpStreamFilter(), + streamFilter ?? new NoOpStreamFilter(), queueId, options ?? new StreamPullingAgentOptions(), queueAdapter, @@ -879,6 +880,8 @@ private sealed class PurgeablePooledQueueCache : IQueueCache { private readonly PooledQueueCache cache; + public List DeliveryProgressTokens { get; } = []; + public PurgeablePooledQueueCache(bool retainPurgeMetadata = false) { cache = new( @@ -942,6 +945,7 @@ private QueueCacheCursorResult WrapCursorResult(QueueCacheCur public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { + DeliveryProgressTokens.Add(earliestSubscriptionToken); } public void Purge() @@ -1056,6 +1060,8 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); + public Queue DeliveryResponses { get; } = new(); + public StreamHandshakeToken? HandshakeResponse { get; set; } = requestedToken; public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1069,7 +1075,7 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); await releaseDelivery.Task; - return null; + return DeliveryResponses.TryDequeue(out var response) ? response : null; } public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; @@ -1080,7 +1086,7 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke return Task.CompletedTask; } - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(requestedToken); + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(HandshakeResponse); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -2714,6 +2720,807 @@ public async Task RunQueuePump_ReadsAfterReinitialize() await receiver.Received(1).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false, 200)] + [InlineData(true, false, 0)] + [InlineData(false, true, 1)] + public async Task Shutdown_AdvancesPastOnlyDrainedSubscriptions(bool holdFirstDelivery, bool holdNewDelivery, long expectedCheckpoint) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + IList messages = + [ + new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1)), + new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200)) + ]; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(_ => Task.FromResult(messages)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(idleId, new EventSequenceTokenV2(0), DateTime.UtcNow); + await accessor.RegisterStream(activeId, new EventSequenceTokenV2(0), DateTime.UtcNow); + var streams = await accessor.GetPubSubCache(); + var idle = new RecordingConsumer(); + var active = new RecordingConsumer(); + var pending = new RecordingConsumer(); + active.ReleaseDelivery(); + if (!holdFirstDelivery) + { + idle.ReleaseDelivery(); + } + + var idleData = AddConsumer(streams[idleId], idleId, idle); + var activeData = AddConsumer(streams[activeId], activeId, active); + try + { + await accessor.ReadFromQueue(queueId, receiver, 1000); + await idle.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await active.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(holdFirstDelivery ? 0 : 1, idleData.LastProcessedToken!.SequenceNumber); + Assert.Equal(200, activeData.LastProcessedToken!.SequenceNumber); + Assert.Equal(StreamConsumerDataState.Inactive, activeData.State); + Assert.Equal( + holdFirstDelivery ? StreamConsumerDataState.Active : StreamConsumerDataState.Inactive, + idleData.State); + Assert.Empty(idle.Errors); + Assert.Empty(active.Errors); + if (holdNewDelivery) + { + idleData.StreamConsumer = pending; + messages = + [ + new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(201)), + new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(300)) + ]; + await accessor.ReadFromQueue(queueId, receiver, 1000); + await pending.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, idleData.LastProcessedToken.SequenceNumber); + Assert.Equal(300, activeData.LastProcessedToken.SequenceNumber); + Assert.Equal(StreamConsumerDataState.Active, idleData.State); + } + + await accessor.Shutdown(); + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(expectedCheckpoint, checkpoint.SequenceNumber); + } + finally + { + idle.ReleaseDelivery(); + pending.ReleaseDelivery(); + } + + StreamConsumerData AddConsumer(StreamConsumerCollection collection, QualifiedStreamId id, RecordingConsumer consumer) + { + var data = collection.AddConsumer(GuidId.GetGuidId(Guid.NewGuid()), id, consumer, null, DateTime.UtcNow); + data.IsRegistered = true; + data.LastProcessedToken = new EventSequenceTokenV2(0); + data.Cursor = queueCache.GetCacheCursor(id.StreamId, null); + return data; + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shutdown_DoesNotSkipFailedRegistrationOrInitialAttachment(bool subscriberAttachment, bool unknownPosition) + { + var failedId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var otherId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var subscription = new PubSubSubscriptionState( + GuidId.GetGuidId(Guid.NewGuid()), failedId, GrainId.Create("test", Guid.NewGuid().ToString())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(call => + Task.FromResult>(call.ArgAt(0).Equals(failedId) + ? new HashSet { subscription } + : new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [ + new TestBatchContainer(failedId.StreamId, unknownPosition ? null! : new EventSequenceTokenV2(100)), + new TestBatchContainer(otherId.StreamId, new EventSequenceTokenV2(200)) + ])); + + // Missing pubsub terminates registration. A missing RuntimeClient fails attachment + // before a consumer record exists, after producer registration has succeeded. + var agent = CreateAgent(subscriberAttachment ? pubSub : null, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var streams = await accessor.GetPubSubCache(); + await Task.WhenAll(streams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (subscriberAttachment) + { + Assert.True(streams[failedId].StreamRegistered); + Assert.Equal(0, streams[failedId].Count); + } + else + { + Assert.Empty(await accessor.GetPubSubCache()); + } + + await accessor.Shutdown(); + + if (unknownPosition) + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(100, checkpoint.SequenceNumber); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Shutdown_DoesNotAdvanceToReadCancelledAfterReceiverReturns(bool shutdownDuringRead) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var readStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var finishRead = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var receiver = Substitute.For(); + var readCount = 0; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => + { + if (++readCount == 1) + { + return Task.FromResult>([new TestBatchContainer(streamId, new EventSequenceTokenV2(10))]); + } + + readStarted.TrySetResult(); + return finishRead.Task; + }); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(new("provider", streamId), new EventSequenceTokenV2(10), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var pump = accessor.RunQueuePump(queueId, cancellation.Token); + await readStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Task? shutdown = null; + if (shutdownDuringRead) + { + shutdown = accessor.Shutdown(); + Assert.False(shutdown.IsCompleted); + } + else + { + cancellation.Cancel(); + } + + finishRead.SetResult([new TestBatchContainer(streamId, new EventSequenceTokenV2(100))]); + await pump.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await (shutdown ?? accessor.Shutdown()).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(10, checkpoint.SequenceNumber); + await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("delivery", false, false, 1)] + [InlineData("delivery", false, true, 1)] + [InlineData("delivery", true, false, 1)] + [InlineData("delivery", true, true, 1)] + [InlineData("filter", false, false, 1)] + [InlineData("filter", false, true, 1)] + [InlineData("filter", true, false, 1)] + [InlineData("filter", true, true, 1)] + [InlineData("rewind", false, false, 1)] + [InlineData("rewind", false, true, 1)] + [InlineData("rewind", true, false, 1)] + [InlineData("rewind", true, true, 1)] + [InlineData("reattach", false, false, 1)] + [InlineData("reattach", false, true, 1)] + [InlineData("reattach", true, false, 1)] + [InlineData("reattach", true, true, 1)] + [InlineData("delivery", false, false, 2)] + [InlineData("delivery", false, true, 2)] + [InlineData("delivery", true, false, 2)] + [InlineData("delivery", true, true, 2)] + [InlineData("filter", false, false, 2)] + [InlineData("filter", false, true, 2)] + [InlineData("filter", true, false, 2)] + [InlineData("filter", true, true, 2)] + [InlineData("rewind", false, false, 2)] + [InlineData("rewind", false, true, 2)] + [InlineData("rewind", true, false, 2)] + [InlineData("rewind", true, true, 2)] + [InlineData("reattach", false, false, 2)] + [InlineData("reattach", false, true, 2)] + [InlineData("reattach", true, false, 2)] + [InlineData("reattach", true, true, 2)] + public async Task Shutdown_PreservesReplayRequirementAfterLaterProgress( + string progressKind, bool unknownPosition, bool expireStream, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var timeProvider = new FakeTimeProvider(); + var now = timeProvider.GetUtcNow().UtcDateTime; + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new ScriptedQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + StreamSequenceToken? safeToken = unknownPosition ? null : new EventSequenceTokenV2(10); + var laterToken = new EventSequenceTokenV2(200); + var batch = Substitute.For(); + batch.StreamId.Returns(streamId.StreamId); + batch.SequenceToken.Returns(laterToken); + batch.GetEvents().Returns([Tuple.Create("payload", laterToken)]); + var filter = progressKind == "filter" ? Substitute.For() : null; + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([batch]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, streamFilter: filter); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), now); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var observer = new RecordingConsumer(); + if (progressKind == "rewind") + { + observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(100))); + } + + observer.ReleaseDelivery(); + var failedCursor = Substitute.For(); + failedCursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected cursor failure.")); + var consumer = stream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now); + consumer.IsRegistered = true; + consumer.LastProcessedToken = safeToken; + consumer.Cursor = failedCursor; + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + if (progressKind == "reattach") + { + observer.HandshakeResponse = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(300)); + await agent.RunOrQueueTask(() => agent.AddSubscriber( + consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, + TestContext.Current.CancellationToken)); + } + + failedCursor.Received(1).MoveNextWithResult(); + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(consumer.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Same(safeToken, consumer.LastProcessedToken); + Assert.Equal(0, consumer.PendingHandshakes); + if (progressKind == "filter") + { + Assert.NotNull(filter); + filter.Received(1).ShouldDeliver(streamId.StreamId, "payload", null); + Assert.Null(consumer.LastToken); + Assert.Empty(observer.DeliveredTokens); + } + else + { + var deliveryToken = Assert.IsType(consumer.LastToken); + var sequenceToken = Assert.IsType(deliveryToken.Token); + Assert.Equal(progressKind == "reattach" ? 300 : 200, sequenceToken.SequenceNumber); + Assert.Equal(progressKind == "rewind" ? 2 : 1, observer.DeliveredTokens.Count); + Assert.All(observer.DeliveredTokens, token => Assert.Same(laterToken, token)); + } + + // A delivery-token rewind retains the first newer batch on current providers. + // Accepting its redelivery must still not erase the earlier cursor failure. + Assert.Null(consumer.UnconfirmedDeliveryToken); + + if (expireStream) + { + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + } + + await accessor.Shutdown(); + + if (unknownPosition) + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(10, checkpoint.SequenceNumber); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false, 1)] + [InlineData(false, true, 1)] + [InlineData(true, false, 1)] + [InlineData(true, true, 1)] + [InlineData(false, false, 2)] + [InlineData(false, true, 2)] + [InlineData(true, false, 2)] + [InlineData(true, true, 2)] + public async Task Shutdown_PreservesReplayRequirementAfterRewindToEmptyCursor(bool unknownPosition, bool expireStream, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var timeProvider = new FakeTimeProvider(); + var now = timeProvider.GetUtcNow().UtcDateTime; + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + StreamSequenceToken? safeToken = unknownPosition ? null : new EventSequenceTokenV2(10); + var attemptedToken = new EventSequenceTokenV2(200); + var failedCursor = Substitute.For(); + failedCursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected cursor failure.")); + var recoveryCursor = Substitute.For(); + recoveryCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); + recoveryCursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, attemptedToken)); + var queueCache = Substitute.For(); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, attemptedToken)]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), now); + var observer = new RecordingConsumer(); + observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(100))); + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now); + consumer.IsRegistered = true; + consumer.LastProcessedToken = safeToken; + consumer.Cursor = failedCursor; + // Model a provider whose attempted batch is no longer retained when the consumer rewinds. + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns( + QueueCacheCursorResult.FromCursor(recoveryCursor), + QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + + failedCursor.Received(1).MoveNextWithResult(); + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(consumer.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Same(safeToken, consumer.LastProcessedToken); + Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); + var deliveryToken = Assert.IsType(consumer.LastToken); + var sequenceToken = Assert.IsType(deliveryToken.Token); + Assert.Equal(100, sequenceToken.SequenceNumber); + Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); + if (expireStream) + { + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + } + + await accessor.Shutdown(); + + if (unknownPosition) + { + queueCache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); + } + else + { + queueCache.Received(1).UpdateDeliveryProgress(safeToken, Arg.Any()); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("delivery", false, false, false)] + [InlineData("delivery", false, true, false)] + [InlineData("delivery", true, false, false)] + [InlineData("delivery", true, true, false)] + [InlineData("implicit", false, false, false)] + [InlineData("implicit", false, true, false)] + [InlineData("implicit", true, false, false)] + [InlineData("implicit", true, true, false)] + [InlineData("explicit", false, false, false)] + [InlineData("explicit", false, true, false)] + [InlineData("explicit", true, false, false)] + [InlineData("explicit", true, true, false)] + [InlineData("delivery", false, false, true)] + [InlineData("delivery", false, true, true)] + [InlineData("delivery", true, false, true)] + [InlineData("delivery", true, true, true)] + [InlineData("implicit", false, false, true)] + [InlineData("implicit", false, true, true)] + [InlineData("implicit", true, false, true)] + [InlineData("implicit", true, true, true)] + [InlineData("explicit", false, false, true)] + [InlineData("explicit", false, true, true)] + [InlineData("explicit", true, false, true)] + [InlineData("explicit", true, true, true)] + public async Task Shutdown_PreservesReplayFloorForEarlierRequest( + string tokenKind, bool unknownPosition, bool failCursorAcquisition, bool reattach) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var acknowledgedToken = new EventSequenceTokenV2(100); + var attemptedToken = new EventSequenceTokenV2(200); + var earlierToken = new EventSequenceTokenV2(50); + var requestedToken = tokenKind == "delivery" + ? StreamHandshakeToken.CreateDeliveyToken(earlierToken) + : StreamHandshakeToken.CreateStartToken(earlierToken); + var failedCursor = Substitute.For(); + var cursorError = new InvalidOperationException("Injected cursor failure before the rewind."); + failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); + var recoveryCursor = Substitute.For(); + recoveryCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); + recoveryCursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, attemptedToken)); + var queueCache = Substitute.For(); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + var progress = new List(); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => progress.Add(call.ArgAt(0))); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, attemptedToken)])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, acknowledgedToken, DateTime.UtcNow); + var observer = new RecordingConsumer(); + if (!reattach) + { + observer.DeliveryResponses.Enqueue(requestedToken); + } + + observer.ReleaseDelivery(); + var subscriptionId = SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); + if (tokenKind == "implicit") + { + subscriptionId = SubscriptionMarker.MarkAsImplictSubscriptionId(subscriptionId); + } + + Assert.Equal(tokenKind == "implicit", SubscriptionMarker.IsImplicitSubscription(subscriptionId)); + + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(subscriptionId), streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + // Seed the preceding acknowledgement; the unresolved error is raised by the cursor below. + consumer.LastProcessedToken = unknownPosition ? null : acknowledgedToken; + consumer.LastToken = unknownPosition ? null : StreamHandshakeToken.CreateDeliveyToken(acknowledgedToken); + consumer.Cursor = failedCursor; + var recoveryIssued = false; + var earlierCursorRequested = false; + var acquisitionError = new InvalidOperationException("Injected cursor acquisition failure at the earlier position."); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + if (!recoveryIssued) + { + recoveryIssued = true; + return QueueCacheCursorResult.FromCursor(recoveryCursor); + } + + if (call.ArgAt(1)?.SequenceNumber == earlierToken.SequenceNumber) + { + earlierCursorRequested = true; + if (failCursorAcquisition) + { + throw acquisitionError; + } + } + + return QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor()); + }); + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + if (reattach) + { + observer.HandshakeResponse = requestedToken; + await agent.RunOrQueueTask(() => agent.AddSubscriber( + consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, + TestContext.Current.CancellationToken)); + } + + Assert.True(earlierCursorRequested); + failedCursor.Received(1).MoveNextWithResult(); + Assert.Contains(cursorError, observer.Errors); + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(consumer.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); + await accessor.Shutdown(); + + if (unknownPosition || tokenKind == "explicit" || reattach && tokenKind == "implicit") + { + Assert.Null(consumer.LastProcessedToken); + Assert.Empty(progress); + } + else + { + Assert.Equal(earlierToken, consumer.LastProcessedToken); + Assert.Equal(earlierToken, Assert.Single(progress)); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shutdown_PreservesReplayFloorAfterAccountingFailure(bool failCacheAdd, bool includeUnreadTail) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var failedId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var tailId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var backingCache = new PurgeablePooledQueueCache(); + var queueCache = Substitute.For(); + var accountingError = new InvalidOperationException("Injected synchronous fetched-batch accounting failure."); + queueCache.When(cache => cache.AddToCache(Arg.Any>())).Do(call => + { + var batches = call.Arg>(); + if (failCacheAdd && batches.Any(batch => batch.StreamId.Equals(failedId.StreamId))) + { + throw accountingError; + } + + backingCache.AddToCache(batches); + }); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + var streamId = call.ArgAt(0); + if (!failCacheAdd && streamId.Equals(failedId.StreamId)) + { + throw accountingError; + } + + return backingCache.TryGetCacheCursor(streamId, call.ArgAt(1)); + }); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => backingCache.UpdateDeliveryProgress(call.ArgAt(0), call.Arg())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + var failedRead = new List + { + new TestBatchContainer(failedId.StreamId, new EventSequenceTokenV2(100)), + }; + if (includeUnreadTail) + { + failedRead.Add(new TestBatchContainer(tailId.StreamId, new EventSequenceTokenV2(150))); + } + + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1))]), + Task.FromResult>(failedRead), + Task.FromResult>([new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(idleId, new EventSequenceTokenV2(0), DateTime.UtcNow); + await accessor.RegisterStream(activeId, new EventSequenceTokenV2(0), DateTime.UtcNow); + var streams = await accessor.GetPubSubCache(); + var idleObserver = new RecordingConsumer(); + var activeObserver = new RecordingConsumer(); + var idleConsumer = AddConsumer(idleId, idleObserver); + var activeConsumer = AddConsumer(activeId, activeObserver); + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(idleConsumer.IsCaughtUp); + Assert.Equal(1, Assert.Single(idleObserver.DeliveredTokens).SequenceNumber); + Assert.Same(accountingError, await Assert.ThrowsAsync( + () => accessor.ReadFromQueue(queueId, receiver, 1000))); + streams = await accessor.GetPubSubCache(); + Assert.False(streams.ContainsKey(failedId)); + Assert.False(streams.ContainsKey(tailId)); + await pubSub.DidNotReceive().RegisterProducer(failedId, Arg.Any(), Arg.Any()); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(activeConsumer.IsCaughtUp); + Assert.Equal(200, Assert.Single(activeObserver.DeliveredTokens).SequenceNumber); + await accessor.Shutdown(); + + var checkpoint = Assert.Single(backingCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.True(checkpoint.SequenceNumber <= 100, + $"Fetched position 100 was never accounted for, but shutdown advanced to {checkpoint.SequenceNumber}."); + + StreamConsumerData AddConsumer(QualifiedStreamId streamId, RecordingConsumer observer) + { + observer.ReleaseDelivery(); + var consumer = streams[streamId].AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new EventSequenceTokenV2(0); + consumer.Cursor = backingCache.GetCacheCursor(streamId.StreamId, null); + return consumer; + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Shutdown_DoesNotReusePreviousReceiverReadBoundaryAfterReinitialize(bool readAfterReinitialize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]), + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(10))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + await accessor.Shutdown(); + + await InitializeAgent(agent); + if (readAfterReinitialize) + { + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + + await accessor.Shutdown(); + + Assert.Equal( + new long?[] { 200, readAfterReinitialize ? 10 : null }, + queueCache.DeliveryProgressTokens.Select(token => token?.SequenceNumber)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_DoesNotTreatFinishedRegistrationTaskAsRegisteredProducer() + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + // A registration which exits during shutdown can finish without learning its subscribers. + stream.StreamRegistered = false; + stream.RegistrationTask = Task.CompletedTask; + + await accessor.Shutdown(); + + Assert.Empty(queueCache.DeliveryProgressTokens); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, 1, 100)] + [InlineData(true, 1, 42)] + [InlineData(false, 2, 100)] + [InlineData(true, 2, 42)] + public async Task Shutdown_DistinguishesEmptyCursorFromBatchWithMissingPosition(bool missingPosition, int batchSize, long expectedCheckpoint) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var cursor = Substitute.For(); + cursor.MoveNextWithResult().Returns( + missingPosition ? QueueCacheCursorMoveResult.Success : QueueCacheCursorMoveResult.NoData, + QueueCacheCursorMoveResult.NoData); + cursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, null!)); + var queueCache = Substitute.For(); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(cursor)); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); + var observer = new RecordingConsumer(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.Cursor = cursor; + consumer.IsCaughtUp = true; + consumer.LastProcessedToken = missingPosition ? new EventSequenceTokenV2(42) : null; + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Equal(!missingPosition, consumer.IsCaughtUp); + Assert.Empty(observer.DeliveredTokens); + await accessor.Shutdown(); + + queueCache.Received(1).UpdateDeliveryProgress( + Arg.Is(token => token.SequenceNumber == expectedCheckpoint), Arg.Any()); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -2936,8 +3743,11 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations() [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_SkipsDeliveryProgressForUnregisteredConsumer() + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public async Task Shutdown_SkipsDeliveryProgressForUnregisteredOrReattachingConsumer(int pendingHandshakes) { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) @@ -2979,6 +3789,10 @@ public async Task Shutdown_SkipsDeliveryProgressForUnregisteredConsumer() filterData: null, now: DateTime.UtcNow); unregisteredConsumer.PendingStartToken = new EventSequenceTokenV2(50); + unregisteredConsumer.IsRegistered = pendingHandshakes > 0; + unregisteredConsumer.PendingHandshakes = pendingHandshakes; + unregisteredConsumer.IsCaughtUp = pendingHandshakes > 0; + unregisteredConsumer.LastProcessedToken = pendingHandshakes > 0 ? new EventSequenceTokenV2(50) : null; await testAccessor.Shutdown(); From b447d50cafe45905fd6d8fcc96e257dcbe39d012 Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 15:48:35 -0700 Subject: [PATCH 2/9] fix(streaming): recover safe checkpoint progress after failures - Preserve replay constraints through handshake and cursor fallbacks. - Resume progress only after acknowledged, gap-free recovery. - Retry pending registrations and consumers without losing state. - Reject stale delivery callbacks and unproven inclusive replays. - Cover recovery, cleanup, batching, and checkpoint safety regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 504 ++++++-- .../QueueStreamDataStructures.cs | 16 + .../StreamConsumerCollection.cs | 21 +- .../PersistentStreamPullingAgentTests.cs | 1015 ++++++++++++++++- 4 files changed, 1399 insertions(+), 157 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index b2967798000..1cd455e9315 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -29,6 +29,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly IStreamPubSub pubSub; private readonly IStreamFilter streamFilter; private readonly Dictionary pubSubCache; + private readonly HashSet pendingConsumerRecoveries = new(); private readonly StreamPullingAgentOptions options; private readonly ILogger logger; private readonly IQueueAdapterCache queueAdapterCache; @@ -47,7 +48,6 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; private StreamSequenceToken? _lastReadToken; - private StreamSequenceToken? _retainedReplayToken; private bool _hasUnknownReplayPosition; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -159,8 +159,7 @@ public Task Initialize(CancellationToken cancellationToken) _activePumpTask = Task.CompletedTask; _lastReadToken = null; - _retainedReplayToken = null; - _hasUnknownReplayPosition = false; + pendingConsumerRecoveries.Clear(); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -307,6 +306,7 @@ await Task.WhenAll(inFlightRegistrations) LogWarningUnregisterProducer(exc); } pubSubCache.Clear(); + pendingConsumerRecoveries.Clear(); } public Task AddSubscriber( @@ -352,13 +352,17 @@ private async Task AddSubscriber_Impl( } data.IsCaughtUp = false; + data.PendingStartToken ??= cacheToken; data.PendingHandshakes++; try { if (await DoHandshakeWithConsumer(data, cacheToken)) { RecordDeliveryProgress(data, GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken)); - data.PendingStartToken = null; + if (!data.HasDeliveryProgressError && data.LastProcessedToken is not null) + { + data.PendingStartToken = null; + } data.IsRegistered = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); if (data.State == StreamConsumerDataState.Inactive) @@ -421,6 +425,7 @@ private async Task DoHandshakeWithConsumer( && startPositionToken.Equals(consumerData.LastToken); if (!preservesCurrentPosition) { + RecordReplayRequest(consumerData, startPositionToken); RecordDeliveryProgress(consumerData, null); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetCacheCursorAtStartPosition( @@ -432,6 +437,7 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { + RecordReplayRequest(consumerData, effectiveHandshakeToken); RecordDeliveryProgress( consumerData, GetInitialDeliveryProgress(effectiveHandshakeToken, consumerData.LastProcessedToken)); consumerData.SafeDisposeCursor(logger); @@ -441,7 +447,7 @@ private async Task DoHandshakeWithConsumer( { // Delivery tokens and implicit recovery tokens identify already processed events. consumerData.Cursor = GetAdvancedCursorOrFallback( - consumerData.StreamId, + consumerData, requestedToken, cacheToken, out consumerData.PendingBatch); @@ -449,6 +455,11 @@ private async Task DoHandshakeWithConsumer( else { var result = queueCache.TryGetCacheCursor(consumerData.StreamId, requestedToken); + if (result.Kind == QueueCacheCursorResultKind.CacheMiss) + { + RecordDeliveryProgressError(consumerData, effectiveHandshakeToken); + } + consumerData.Cursor = result.Kind switch { QueueCacheCursorResultKind.Success => result.Cursor!, @@ -457,6 +468,10 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null QueueCacheCursorResultKind.CacheMiss => throw result.CacheMiss!.Value.ToException(), _ => throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."), }; + if (result.Kind == QueueCacheCursorResultKind.Success) + { + TrackDeliveryRecoveryCursor(consumerData, consumerData.Cursor, effectiveHandshakeToken); + } } } else if (effectiveHandshakeToken is not null) @@ -473,6 +488,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception exception) { + RecordDeliveryProgressError(consumerData, effectiveHandshakeToken); exceptionOccured = exception; } if (exceptionOccured != null) @@ -514,6 +530,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception) { + RecordDeliveryProgressError(consumerData); consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, null); } } @@ -546,16 +563,7 @@ private static QueueCacheCursorMoveResult AdvanceCursorPastToken( throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - var batch = cursor.GetCurrent(out var exception); - if (exception is not null) - { - throw exception; - } - - if (batch is null) - { - throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); - } + var batch = GetCurrentBatchOrThrow(cursor); var comparison = EventSequenceTokenCompatibility.Compare(batch.SequenceToken, token); if (comparison >= 0) @@ -590,6 +598,30 @@ private IQueueCacheCursor GetCacheCursorAtStartPosition( private IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData) { + if (consumerData.HasDeliveryProgressError) + { + if (consumerData.DeliveryRecoveryToken is { Token: { } requiredToken } recoveryToken) + { + var acquisition = queueCache!.TryGetCacheCursor(consumerData.StreamId, requiredToken); + if (acquisition.Kind == QueueCacheCursorResultKind.Success) + { + var recoveryCursor = acquisition.Cursor!; + if (recoveryToken is StartToken + || TryAdvanceRecoveryCursor(recoveryCursor, requiredToken, out consumerData.PendingBatch)) + { + TrackDeliveryRecoveryCursor(consumerData, recoveryCursor, recoveryToken); + return recoveryCursor; + } + } + else if (acquisition.Kind != QueueCacheCursorResultKind.CacheMiss) + { + throw new InvalidOperationException($"Unexpected cursor result: {acquisition.Kind}."); + } + } + + return GetCacheCursorOrThrow(consumerData.StreamId, null); + } + if (consumerData.LastProcessedToken is { } lastProcessedToken) { var result = queueCache!.TryGetCacheCursor(consumerData.StreamId, lastProcessedToken); @@ -659,15 +691,17 @@ private IQueueCacheCursor GetCacheCursorOrThrow(StreamId streamId, StreamSequenc } private IQueueCacheCursor GetAdvancedCursorOrFallback( - StreamId streamId, + StreamConsumerData consumerData, StreamSequenceToken token, StreamSequenceToken? fallbackToken, out IBatchContainer? pendingBatch) { pendingBatch = null; + var streamId = consumerData.StreamId; var acquisitionResult = queueCache!.TryGetCacheCursor(streamId, token); if (acquisitionResult.Kind == QueueCacheCursorResultKind.CacheMiss) { + RecordDeliveryProgressError(consumerData, StreamHandshakeToken.CreateDeliveyToken(token)); return fallbackToken is not null ? GetCacheCursorOrThrow(streamId, fallbackToken) : throw acquisitionResult.CacheMiss!.Value.ToException(); @@ -685,6 +719,7 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( var moveResult = AdvanceCursorPastToken(cursor, token, out pendingBatch); if (moveResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { + RecordDeliveryProgressError(consumerData, StreamHandshakeToken.CreateDeliveyToken(token)); return fallbackToken is not null ? GetCacheCursorOrThrow(streamId, fallbackToken) : throw moveResult.CacheMiss!.Value.ToException(); @@ -696,6 +731,7 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( } retainCursor = true; + TrackDeliveryRecoveryCursor(consumerData, cursor, StreamHandshakeToken.CreateDeliveyToken(token)); return cursor; } finally @@ -774,8 +810,11 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea LogDebugRemovedConsumer(subscriptionId, streamId); } - if (streamData.Count == 0) + if (streamData.Count == 0 && streamData.RegistrationTask is null) + { pubSubCache.Remove(streamId); + streamData.DisposeAll(logger); + } } private Task RunQueuePump(QueueId queueId, CancellationToken cancellationToken) @@ -807,6 +846,8 @@ private async Task PumpQueue(QueueId queueId, CancellationToken cancellationToke if (IsShutdown || cancellationToken.IsCancellationRequested) return; // timer was already removed, last tick + RetryPendingWork(cancellationToken); + // loop through the queue until it is empty. while (!IsShutdown && !cancellationToken.IsCancellationRequested) // shutdown sets IsShutdown and cancels the timer token. { @@ -862,8 +903,10 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? return false; } + RetryPendingWork(cancellationToken); + // Pause all queue reads so a cold stream's first batch stays pinned until registration completes. - if (pubSubCache.Values.Any(static stream => stream.RegistrationTask is { IsCompleted: false })) + if (pubSubCache.Values.Any(static stream => stream.RegistrationTask is not null)) { return false; } @@ -960,13 +1003,8 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // The receiver has consumed this read even if cache insertion or registration fails. if (!accountedFor) { - foreach (var batch in multiBatch) - { - if (batch is not null) - { - RetainReplayPosition(batch.SequenceToken); - } - } + _hasUnknownReplayPosition = true; + LogErrorQueueRecoveryRequired(new(QueueId), "A fetched read could not be fully admitted to the cache and stream registry."); } } @@ -978,11 +1016,18 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // Only publish the read boundary after every stream has been accounted for. foreach (var batch in multiBatch) { - if (batch?.SequenceToken is not { } token) + if (batch is null) { continue; } + if (batch.SequenceToken is not { } token) + { + _hasUnknownReplayPosition = true; + LogErrorQueueRecoveryRequired(new(myQueueId), $"A fetched batch for stream {batch.StreamId} has no sequence token."); + continue; + } + if (_lastReadToken is null) { _lastReadToken = token; @@ -990,6 +1035,7 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? else if (!TryCompareQueueProgress(token, _lastReadToken, out var comparison)) { _hasUnknownReplayPosition = true; + LogErrorQueueRecoveryRequired(new(myQueueId), "Fetched batches have incompatible sequence tokens."); } else if (comparison > 0) { @@ -1005,7 +1051,10 @@ private void CleanupPubSubCache(DateTime now) List? inactiveStreams = null; foreach (var tuple in pubSubCache) { - if (tuple.Value.IsInactive(now, options.StreamInactivityPeriod)) + if (tuple.Value.IsInactive(now, options.StreamInactivityPeriod) + && tuple.Value.RegistrationTask is null + && (queueCache is null + || TryGetStreamDeliveryProgress(tuple.Value, out var requiredToken) && requiredToken is null)) { inactiveStreams ??= new(); inactiveStreams.Add(tuple.Key); @@ -1021,16 +1070,6 @@ private void CleanupPubSubCache(DateTime now) { if (pubSubCache.Remove(streamId, out var streamData)) { - // Cleanup must not erase replay requirements left by interrupted delivery. - if (!TryGetStreamDeliveryProgress(streamData, out var requiredToken)) - { - RetainReplayPosition(null); - } - else if (requiredToken is not null) - { - RetainReplayPosition(requiredToken); - } - streamData.DisposeAll(logger); StreamingEvents.EmitStreamInactive(streamProviderName, streamId.StreamId, options.StreamInactivityPeriod, Silo); } @@ -1059,11 +1098,6 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return false; } - if (_retainedReplayToken is { } retained && !IncludeProgress(retained, ref earliest)) - { - return false; - } - foreach (var streamConsumers in pubSubCache.Values) { if (!TryGetStreamDeliveryProgress(streamConsumers, out var current) @@ -1080,7 +1114,7 @@ private bool TryGetStreamDeliveryProgress(StreamConsumerCollection streamConsume { earliest = null; if (!streamConsumers.StreamRegistered - || streamConsumers.RegistrationTask is { IsCompleted: false }) + || streamConsumers.RegistrationTask is not null) { return false; } @@ -1162,14 +1196,6 @@ private bool TryCompareQueueProgress(StreamSequenceToken token, StreamSequenceTo } } - private void RetainReplayPosition(StreamSequenceToken? token) - { - if (token is null || !IncludeProgress(token, ref _retainedReplayToken)) - { - _hasUnknownReplayPosition = true; - } - } - private void RecordDeliveryProgress(StreamConsumerData consumer, StreamSequenceToken? token) { // A reset cursor's later progress cannot prove that it did not skip earlier data. @@ -1177,6 +1203,10 @@ private void RecordDeliveryProgress(StreamConsumerData consumer, StreamSequenceT if (!consumer.HasDeliveryProgressError || token is null) { consumer.LastProcessedToken = token; + if (token is not null) + { + consumer.PendingStartToken = null; + } } else if (consumer.LastProcessedToken is { } current) { @@ -1191,10 +1221,199 @@ private void RecordDeliveryProgress(StreamConsumerData consumer, StreamSequenceT } } - private static void RecordDeliveryProgressError(StreamConsumerData consumer) + private void RecordReplayRequest(StreamConsumerData consumer, StreamHandshakeToken request) + { + if (consumer.HasDeliveryProgressError) + { + IncludeDeliveryRecoveryRequest(consumer, NormalizeRecoveryToken(consumer, request)); + consumer.DeliveryRecoveryCursor = null; + consumer.HasObservedRecoveryStart = false; + } + + if (request is StartToken) + { + consumer.PendingStartToken = request.Token; + } + else if (request is StartPositionToken) + { + consumer.PendingStartToken = null; + } + } + + private static StreamHandshakeToken? NormalizeRecoveryToken(StreamConsumerData consumer, StreamHandshakeToken? token) + => token is StartToken && SubscriptionMarker.IsImplicitSubscription(consumer.SubscriptionId.Guid) + ? StreamHandshakeToken.CreateDeliveyToken(token.Token!) + : token is StartToken or DeliveryToken ? token : null; + + private void IncludeDeliveryRecoveryRequest(StreamConsumerData consumer, StreamHandshakeToken? request) { + if (consumer.DeliveryRecoveryToken?.Token is not { } current + || request?.Token is not { } requested + || !TryCompareQueueProgress(requested, current, out var comparison)) + { + consumer.DeliveryRecoveryToken = null; + return; + } + + if (comparison < 0 || comparison == 0 && request is StartToken) + { + consumer.DeliveryRecoveryToken = request; + } + } + + private void TrackDeliveryRecoveryCursor( + StreamConsumerData consumer, + IQueueCacheCursor cursor, + StreamHandshakeToken? request) + { + if (consumer.HasDeliveryProgressError + && consumer.DeliveryRecoveryToken is { Token: { } required } recovery + && request?.Token is { } requested + && TryCompareQueueProgress(requested, required, out var comparison) + && (comparison < 0 || comparison == 0 && (request is StartToken || recovery is DeliveryToken))) + { + consumer.DeliveryRecoveryCursor = cursor; + consumer.HasObservedRecoveryStart = false; + } + } + + private void ObserveDeliveryRecoveryStart(StreamConsumerData consumer, IBatchContainer batch) + { + if (!consumer.HasDeliveryProgressError) + { + if (consumer.LastProcessedToken is null && batch.SequenceToken is { } first + && (consumer.PendingStartToken is null + || TryCompareQueueProgress(first, consumer.PendingStartToken, out var startComparison) && startComparison < 0)) + { + consumer.PendingStartToken = first; + } + } + else if (ReferenceEquals(consumer.DeliveryRecoveryCursor, consumer.Cursor) + && consumer.DeliveryRecoveryToken is StartToken { Token: { } required } + && batch.SequenceToken is { } current + && TryCompareQueueProgress(current, required, out var anchorComparison) && anchorComparison == 0) + { + // Pooled caches can accept a purged token as an exclusive boundary. Inclusive recovery must see it. + consumer.HasObservedRecoveryStart = true; + } + } + + private void RecordAcknowledgedDeliveryProgress( + StreamConsumerData consumer, + StreamSequenceToken? token, + IQueueCacheCursor deliveryCursor, + long cursorVersion) + { + if (consumer.CursorVersion != cursorVersion || !ReferenceEquals(consumer.Cursor, deliveryCursor)) + { + return; + } + + // Opening or draining an empty recovery cursor does not prove that its missing data was replayed. + if (consumer.HasDeliveryProgressError + && ReferenceEquals(consumer.DeliveryRecoveryCursor, deliveryCursor) + && consumer.DeliveryRecoveryToken is { Token: { } required } recovery + && token is not null + && TryCompareQueueProgress(token, required, out var comparison) + && (recovery is DeliveryToken && comparison > 0 + || recovery is StartToken && comparison >= 0 && consumer.HasObservedRecoveryStart)) + { + consumer.HasDeliveryProgressError = false; + consumer.DeliveryRecoveryToken = null; + consumer.DeliveryRecoveryCursor = null; + consumer.HasObservedRecoveryStart = false; + pendingConsumerRecoveries.Remove(consumer); + } + + RecordDeliveryProgress(consumer, token); + if (token is not null + && consumer.UnconfirmedDeliveryToken is { } attempted + && !IsBefore(token, attempted)) + { + consumer.UnconfirmedDeliveryToken = null; + } + } + + private void RecordDeliveryProgressError( + StreamConsumerData consumer, + StreamHandshakeToken? replayRequest = null, + StreamSequenceToken? failedToken = null) + { + var request = NormalizeRecoveryToken(consumer, replayRequest); + if (!consumer.HasDeliveryProgressError) + { + var first = consumer.PendingStartToken; + if (failedToken is not null) + { + if (first is null) + { + first = failedToken; + } + else if (!TryCompareQueueProgress(failedToken, first, out var comparison)) + { + first = null; + } + else if (comparison < 0) + { + first = failedToken; + } + } + + consumer.DeliveryRecoveryToken = replayRequest is not null + ? request + : consumer.LastProcessedToken is { } processed + ? StreamHandshakeToken.CreateDeliveyToken(processed) + : first is { } start + ? StreamHandshakeToken.CreateStartToken(start) + : null; + } + else if (replayRequest is not null) + { + IncludeDeliveryRecoveryRequest(consumer, request); + } + + if (!consumer.HasDeliveryProgressError) + { + LogWarningDeliveryRecoveryRequired(new(QueueId), consumer.SubscriptionId, consumer.StreamId); + } + consumer.HasDeliveryProgressError = true; consumer.IsCaughtUp = false; + consumer.DeliveryRecoveryCursor = null; + consumer.HasObservedRecoveryStart = false; + pendingConsumerRecoveries.Add(consumer); + } + + private void RetryPendingConsumers() + { + foreach (var consumer in pendingConsumerRecoveries.ToArray()) + { + if (!pubSubCache.TryGetValue(consumer.StreamId, out var stream) + || !stream.TryGetConsumer(consumer.SubscriptionId, out var current) + || !ReferenceEquals(current, consumer) + || !consumer.HasDeliveryProgressError + || consumer.DeliveryRecoveryToken is null) + { + pendingConsumerRecoveries.Remove(consumer); + } + else if (consumer.IsRegistered && consumer.PendingHandshakes == 0 + && consumer.State == StreamConsumerDataState.Inactive) + { + RunConsumerCursor(consumer).Ignore(); + } + } + } + + private void RetryPendingWork(CancellationToken cancellationToken) + { + RetryPendingConsumers(); + foreach (var pending in pubSubCache + .Where(static pair => pair.Value.RegistrationTask is { IsCompleted: true }) + .ToArray()) + { + RegisterStream(pending.Key, pending.Value.RegistrationStartToken, + _timeProvider.GetUtcNow().UtcDateTime, cancellationToken); + } } private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) @@ -1202,7 +1421,7 @@ private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken ot private void RegisterStream( QualifiedStreamId streamId, - StreamSequenceToken firstToken, + StreamSequenceToken? firstToken, DateTime now, CancellationToken cancellationToken) { @@ -1211,20 +1430,23 @@ private void RegisterStream( return; } - var streamData = new StreamConsumerCollection(now); + if (!pubSubCache.TryGetValue(streamId, out var streamData)) + { + streamData = new StreamConsumerCollection(now) { RegistrationStartToken = firstToken }; + pubSubCache.Add(streamId, streamData); + } + else if (streamData.RegistrationTask is null || !streamData.RegistrationTask.IsCompleted) + { + return; + } - // Create a fake cursor to point into a cache. - // That way we will not purge the event from the cache, until we talk to pub sub. - // This will help ensure the "casual consistency" between pre-existing subscripton (of a potentially new already subscribed consumer) - // and later production. - var pinCursor = queueCache is null ? null : GetCacheCursorOrThrow(streamId, firstToken); streamData.RegistrationTask = RegisterStreamAsync(); - pubSubCache.Add(streamId, streamData); async Task RegisterStreamAsync() { await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); + var completed = false; try { if (IsShutdown || cancellationToken.IsCancellationRequested) @@ -1232,6 +1454,12 @@ async Task RegisterStreamAsync() return; } + // Retain the original boundary and pin across retries, including failures before producer registration. + if (queueCache is not null) + { + streamData.RegistrationCursor ??= GetCacheCursorOrThrow(streamId, streamData.RegistrationStartToken); + } + var subscribers = await RegisterAsStreamProducer(streamId, cancellationToken); if (IsShutdown || cancellationToken.IsCancellationRequested) @@ -1246,79 +1474,70 @@ async Task RegisterStreamAsync() LogDebugGotBackSubscribers(subscribers.Count, streamId); - // Await all initial subscriber handshakes before releasing the pin cursor so - // that the first batch cannot be purged from the cache before each subscriber - // has established its own cursor. + completed = true; if (subscribers.Count > 0) { - var addSubscriptionTasks = new List(subscribers.Count); + var addSubscriptionTasks = new List>(subscribers.Count); foreach (PubSubSubscriptionState item in subscribers) { addSubscriptionTasks.Add(SubscribeWithIsolation(item)); } - await Task.WhenAll(addSubscriptionTasks); + completed = (await Task.WhenAll(addSubscriptionTasks)).All(static attached => attached); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - RemoveCanceledRegistration(); + completed = false; + // The retained stream entry remains pending for the next uncanceled pump. } catch (Exception exception) { - FailRegistration(exception); + completed = false; + LogWarningFailedToRegisterStream(streamId, exception); + StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); } finally { - streamData.RegistrationTask = null; - pinCursor?.Dispose(); - } - } - - void RemoveCanceledRegistration() - { - RetainReplayPosition(firstToken); - if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) - && ReferenceEquals(cachedStreamData, streamData)) - { - pubSubCache.Remove(streamId); + if (completed) + { + streamData.RegistrationTask = null; + streamData.RegistrationStartToken = null; + streamData.DisposeRegistrationCursor(logger); + } + else + { + streamData.RegistrationTask = Task.CompletedTask; + if (IsShutdown + || !pubSubCache.TryGetValue(streamId, out var current) || !ReferenceEquals(current, streamData)) + { + streamData.DisposeRegistrationCursor(logger); + } + } } - - streamData.DisposeAll(logger); } - void FailRegistration(Exception exception) + async Task SubscribeWithIsolation(PubSubSubscriptionState item) { - LogWarningFailedToRegisterStream(streamId, exception); - StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); - - RetainReplayPosition(firstToken); - - // Only reached when producer registration itself fails. - if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) - && ReferenceEquals(cachedStreamData, streamData)) + if (IsShutdown) { - pubSubCache.Remove(streamId); + return false; } - streamData.DisposeAll(logger); - } - - async Task SubscribeWithIsolation(PubSubSubscriptionState item) - { - if (IsShutdown) + if (streamData.TryGetConsumer(item.SubscriptionId, out var existing) && existing.IsRegistered) { - return; + return existing.PendingHandshakes == 0; } try { - await AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, item.FilterData, firstToken); + await AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, item.FilterData, streamData.RegistrationStartToken); + return !streamData.TryGetConsumer(item.SubscriptionId, out var consumer) || consumer.IsRegistered; } catch (Exception exception) { - RetainReplayPosition(firstToken); LogWarningFailedToAddSubscription(item.Stream, exception); + return false; } } } @@ -1364,9 +1583,19 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.Cursor is null) return; consumerData.State = StreamConsumerDataState.Active; + if (consumerData.HasDeliveryProgressError && consumerData.DeliveryRecoveryToken is not null + && consumerData.PendingHandshakes == 0 + && !ReferenceEquals(consumerData.Cursor, consumerData.DeliveryRecoveryCursor)) + { + consumerData.SafeDisposeCursor(logger); + consumerData.Cursor = GetRecoveryCursor(consumerData); + } + var deliveredAny = false; while (!IsShutdown && !cancellationToken.IsCancellationRequested && consumerData.Cursor is not null) { + var deliveryCursor = consumerData.Cursor; + var cursorVersion = consumerData.CursorVersion; var batchCursor = options.BatchContainerBatchSize > 1 ? consumerData.Cursor as IQueueCacheCursorBatchDelivery : null; @@ -1374,6 +1603,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati ConsumerBatch nextBatch = default; Exception? exceptionOccured = null; var forceFaultSubscription = false; + var deliveryPending = false; try { nextBatch = GetBatchForConsumer(consumerData); @@ -1422,7 +1652,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is null) { - RecordDeliveryProgress(consumerData, nextBatch.ProgressToken); + RecordAcknowledgedDeliveryProgress(consumerData, nextBatch.ProgressToken, deliveryCursor, cursorVersion); continue; } } @@ -1450,17 +1680,26 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.UnconfirmedDeliveryToken = nextBatch.ProgressToken; } + deliveryPending = true; StreamHandshakeToken? newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( - i => DeliverBatchToConsumer(consumerData, batch, cancellationToken), + i => DeliverBatchToConsumer(consumerData, batch, cursorVersion, cancellationToken), AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException - (exception, i) => exception is not ClientNotAvailableException && !IsShutdown, + (exception, i) => exception is not ClientNotAvailableException && !IsShutdown + && consumerData.CursorVersion == cursorVersion, this.options.MaxEventDeliveryTime, deliveryBackoffProvider, cancellationToken: cancellationToken); + deliveryPending = false; + if (consumerData.CursorVersion != cursorVersion) + { + continue; + } + StreamSequenceToken? acknowledgedToken = null; if (newToken is not null) { + RecordReplayRequest(consumerData, newToken); consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; @@ -1491,7 +1730,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati // An implicit recovery token identifies the last event processed by the prior activation. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( - consumerData.StreamId, + consumerData, sequenceToken, batch.SequenceToken, out pendingBatch); @@ -1500,6 +1739,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati { RecordDeliveryProgress(consumerData, null); newCursor = GetCacheCursorOrThrow(consumerData.StreamId, sequenceToken); + TrackDeliveryRecoveryCursor(consumerData, newCursor, newToken); } } else if (newToken is DeliveryToken) @@ -1511,7 +1751,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati // The handshake token points to an already processed event, so advance past it. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( - consumerData.StreamId, + consumerData, sequenceToken, batch.SequenceToken, out pendingBatch); @@ -1529,7 +1769,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati else { acknowledgedToken = nextBatch.ProgressToken; - RecordDeliveryProgress(consumerData, acknowledgedToken); + RecordAcknowledgedDeliveryProgress(consumerData, acknowledgedToken, deliveryCursor, cursorVersion); } // A rewind or an explicit start request does not acknowledge the attempted batch. @@ -1543,7 +1783,12 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (Exception exc) { - RecordDeliveryProgressError(consumerData); + if (deliveryPending && consumerData.CursorVersion != cursorVersion) + { + continue; + } + + RecordDeliveryProgressError(consumerData, failedToken: nextBatch.Batch?.SequenceToken); if (batchCursor is not null && nextBatch.Batch is not null) { batchCursor.RecordDeliveryFailure(nextBatch.Batch); @@ -1572,6 +1817,11 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati forceFaultSubscription, cancellationToken); if (faultedSubscription) return; + if (nextBatch.Batch is not null) + { + // Retry through the normal pump rather than skipping the failed delivery or spinning here. + break; + } } } consumerData.State = StreamConsumerDataState.Inactive; @@ -1635,10 +1885,10 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - batchContainer = cursor.GetCurrent(out _) - ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); + batchContainer = GetCurrentBatchOrThrow(cursor); } + ObserveDeliveryRecoveryStart(consumerData, batchContainer); return ShouldDeliverBatch(streamId, batchContainer, filterData) ? new ConsumerBatch(batchContainer, batchContainer.SequenceToken) : new ConsumerBatch(null, batchContainer.SequenceToken); @@ -1652,6 +1902,7 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) if (pendingBatch is not null) { + ObserveDeliveryRecoveryStart(consumerData, pendingBatch); progressToken = pendingBatch.SequenceToken; if (ShouldDeliverBatch(streamId, pendingBatch, filterData)) { @@ -1678,8 +1929,8 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - var batchContainer = cursor.GetCurrent(out _) - ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); + var batchContainer = GetCurrentBatchOrThrow(cursor); + ObserveDeliveryRecoveryStart(consumerData, batchContainer); sawBatch = true; progressToken = batchContainer.SequenceToken; @@ -1703,18 +1954,33 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) return default; } + private static IBatchContainer GetCurrentBatchOrThrow(IQueueCacheCursor cursor) + { + var batch = cursor.GetCurrent(out var exception); + if (exception is not null) + { + throw exception; + } + + return batch ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); + } + private sealed class QueueCacheCursorContractException(string message) : InvalidOperationException(message); private async Task DeliverBatchToConsumer( StreamConsumerData consumerData, IBatchContainer batch, + long cursorVersion, CancellationToken cancellationToken) { try { StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, cancellationToken); - consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token - consumerData.StartPositionIsProviderDefault = false; + if (consumerData.CursorVersion == cursorVersion) + { + consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token + consumerData.StartPositionIsProviderDefault = false; + } StreamingEvents.EmitMessageDelivered(streamProviderName, consumerData, batch, Silo); return newToken; @@ -2098,6 +2364,18 @@ private partial void LogWarningIncompatibleQueueProgress( StreamSequenceToken otherToken, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Cannot establish delivery checkpoint progress for queue {Queue}. {Reason} Receiver/provider recovery is required; later reads or initialization alone do not establish replay continuity." + )] + private partial void LogErrorQueueRecoveryRequired(QueueIdLogRecord queue, string reason); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Delivery progress is constrained for subscription {SubscriptionId} on stream {StreamId} in queue {Queue}. Continuous replay is required to resume progress; unavailable data requires receiver/provider recovery or an explicit subscription resolution." + )] + private partial void LogWarningDeliveryRecoveryRequired(QueueIdLogRecord queue, GuidId subscriptionId, QualifiedStreamId streamId); + [LoggerMessage( Level = LogLevel.Warning, EventId = (int)ErrorCode.PersistentStreamPullingAgent_13, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index 07d7d0c8969..d200ecd4ce7 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -55,6 +55,15 @@ internal sealed class StreamConsumerData public bool HasDeliveryProgressError; [NonSerialized] public StreamSequenceToken? UnconfirmedDeliveryToken; + // Unlike LastToken, this anchor must survive later fallback deliveries and handshake responses. + [NonSerialized] + public StreamHandshakeToken? DeliveryRecoveryToken; + [NonSerialized] + public IQueueCacheCursor? DeliveryRecoveryCursor; + [NonSerialized] + public bool HasObservedRecoveryStart; + [NonSerialized] + public long CursorVersion; public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { @@ -66,10 +75,17 @@ public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, ISt internal void SafeDisposeCursor(ILogger logger) { + CursorVersion++; IsCaughtUp = false; PendingBatch = null; if (Cursor is { } cursor) { + if (ReferenceEquals(DeliveryRecoveryCursor, cursor)) + { + DeliveryRecoveryCursor = null; + HasObservedRecoveryStart = false; + } + Cursor = null; // kill cursor activity and ensure it does not start again on this consumer data. try diff --git a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs index e77c8f21a3a..cbc6f1bca1c 100644 --- a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs +++ b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs @@ -19,9 +19,13 @@ internal sealed class StreamConsumerCollection [Id(2)] public bool StreamRegistered { get; set; } - // Not serialized - runtime-only in-flight registration tracking. + // A completed, nonnull task retains pending registration work for the next pump to retry. [NonSerialized] public Task? RegistrationTask; + [NonSerialized] + public StreamSequenceToken? RegistrationStartToken; + [NonSerialized] + public IQueueCacheCursor? RegistrationCursor; public StreamConsumerCollection(DateTime now) { @@ -62,6 +66,7 @@ public IEnumerable AllConsumers() public void DisposeAll(ILogger logger) { + DisposeRegistrationCursor(logger); foreach (StreamConsumerData consumer in queueData.Values) { consumer.SafeDisposeCursor(logger); @@ -69,6 +74,20 @@ public void DisposeAll(ILogger logger) queueData.Clear(); } + public void DisposeRegistrationCursor(ILogger logger) + { + var cursor = RegistrationCursor; + RegistrationCursor = null; + try + { + cursor?.Dispose(); + } + catch (Exception exception) + { + Utils.LogIgnoredException(logger, exception, "Disposing the pending stream registration cursor."); + } + } + public int Count { get { return queueData.Count; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index ba1559e7467..af6c8d010d5 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -230,7 +230,7 @@ public async Task ReadFromQueue_CachesDequeuedMessagesBeforeObservingCancellatio [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task RegisterStream_RemovesCacheEntryWhenProducerRegistrationTerminates() + public async Task RegisterStream_RetainsCacheEntryWhenProducerRegistrationTerminates() { var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); @@ -240,7 +240,12 @@ public async Task RegisterStream_RemovesCacheEntryWhenProducerRegistrationTermin await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); - Assert.Empty(await testAccessor.GetPubSubCache()); + var stream = Assert.Single(await testAccessor.GetPubSubCache()); + Assert.Equal(streamId, stream.Key); + Assert.False(stream.Value.StreamRegistered); + Assert.NotNull(stream.Value.RegistrationTask); + Assert.True(stream.Value.RegistrationTask.IsCompleted); + Assert.Equal(1, Assert.IsType(stream.Value.RegistrationStartToken).SequenceNumber); } [TestSuite("BVT")] @@ -400,7 +405,8 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, StreamPullingAgentOptions? options = null, - IStreamFilter? streamFilter = null) + IStreamFilter? streamFilter = null, + IBackoffProvider? deliveryBackoff = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -443,7 +449,7 @@ private static PersistentStreamPullingAgent CreateAgent( queueAdapter, queueAdapterCache!, new NoOpStreamDeliveryFailureHandler(), - new FixedBackoff(TimeSpan.FromMilliseconds(1)), + deliveryBackoff ?? new FixedBackoff(TimeSpan.FromMilliseconds(1)), new FixedBackoff(TimeSpan.FromMilliseconds(1)), timeProvider ?? TimeProvider.System, shared); @@ -956,6 +962,8 @@ public void Purge() } } + public void PurgeOldest() => cache.RemoveOldestMessage(); + private sealed class CacheDataAdapter : ICacheDataAdapter { public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) @@ -1057,11 +1065,14 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public List DeliveredBatches { get; } = new(); public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); public Queue DeliveryResponses { get; } = new(); public StreamHandshakeToken? HandshakeResponse { get; set; } = requestedToken; + public Exception? DeliveryException { get; set; } + public int HandshakeCount { get; private set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1071,10 +1082,16 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public async Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) { + DeliveredBatches.Add(item); DeliveredTokens.Add(item.SequenceToken); DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); await releaseDelivery.Task; + if (DeliveryException is { } exception) + { + throw exception; + } + return DeliveryResponses.TryDequeue(out var response) ? response : null; } @@ -1086,7 +1103,11 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke return Task.CompletedTask; } - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(HandshakeResponse); + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + { + HandshakeCount++; + return Task.FromResult(HandshakeResponse); + } public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -2828,10 +2849,13 @@ public async Task Shutdown_DoesNotSkipFailedRegistrationOrInitialAttachment(bool GuidId.GetGuidId(Guid.NewGuid()), failedId, GrainId.Create("test", Guid.NewGuid().ToString())); var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(call => - Task.FromResult>(call.ArgAt(0).Equals(failedId) + .ReturnsForAnyArgs(call => !subscriberAttachment + ? Task.FromException>(new InvalidOperationException("Injected registration failure.")) + : Task.FromResult>(call.ArgAt(0).Equals(failedId) ? new HashSet { subscription } : new HashSet())); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(_ => throw new OperationCanceledException("Injected registration retry cancellation.")); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var queueCache = new RecordingQueueCache(); var adapterCache = Substitute.For(); @@ -2844,9 +2868,9 @@ public async Task Shutdown_DoesNotSkipFailedRegistrationOrInitialAttachment(bool new TestBatchContainer(otherId.StreamId, new EventSequenceTokenV2(200)) ])); - // Missing pubsub terminates registration. A missing RuntimeClient fails attachment - // before a consumer record exists, after producer registration has succeeded. - var agent = CreateAgent(subscriberAttachment ? pubSub : null, queueId, receiver, adapterCache); + // Canceling the retry wait terminates registration. A missing RuntimeClient fails + // attachment before a consumer record exists, after producer registration succeeds. + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, deliveryBackoff: backoff); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); @@ -2857,24 +2881,25 @@ await Task.WhenAll(streams.Values.Select(stream => stream.RegistrationTask ?? Ta { Assert.True(streams[failedId].StreamRegistered); Assert.Equal(0, streams[failedId].Count); + var pendingRegistration = streams[failedId].RegistrationTask; + Assert.NotNull(pendingRegistration); + Assert.True(pendingRegistration.IsCompleted); } else { - Assert.Empty(await accessor.GetPubSubCache()); + var retainedStreams = await accessor.GetPubSubCache(); + Assert.True(retainedStreams.ContainsKey(failedId)); + Assert.All(retainedStreams.Values, stream => + { + Assert.False(stream.StreamRegistered); + Assert.NotNull(stream.RegistrationTask); + Assert.True(stream.RegistrationTask.IsCompleted); + }); } await accessor.Shutdown(); - if (unknownPosition) - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - else - { - var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); - Assert.NotNull(checkpoint); - Assert.Equal(100, checkpoint.SequenceNumber); - } + Assert.Empty(queueCache.DeliveryProgressTokens); } [TestSuite("BVT")] @@ -2936,6 +2961,115 @@ public async Task Shutdown_DoesNotAdvanceToReadCancelledAfterReceiverReturns(boo await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Recovery_InitialGroupedFailureRequiresFirstBatchForInclusiveReplay( + bool purgeFirstBatch, bool explicitEarliestRequest) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var firstToken = new EventSequenceTokenV2(10); + var lastToken = new EventSequenceTokenV2(20); + var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>( + [ + new TestBatchContainer(streamId.StreamId, firstToken), + new TestBatchContainer(streamId.StreamId, lastToken), + ]), + Task.FromResult>([])); + var observer = new RecordingConsumer(explicitEarliestRequest + ? StreamHandshakeToken.CreateStartPositionToken(StreamSubscriptionStartPosition.EarliestAvailable) + : null) + { + DeliveryException = new InvalidOperationException("Injected grouped delivery failure."), + }; + observer.ReleaseDelivery(); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions + { + BatchContainerBatchSize = 2, + InitialSubscriptionStartPosition = explicitEarliestRequest + ? StreamSubscriptionStartPosition.Latest + : StreamSubscriptionStartPosition.EarliestAvailable, + MaxEventDeliveryTime = TimeSpan.FromMilliseconds(100), + }, + deliveryBackoff: new FixedBackoff(TimeSpan.FromMilliseconds(150))); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); + consumer.IsRegistered = true; + Assert.Null(consumer.LastProcessedToken); + Assert.False(consumer.HasDeliveryProgressError); + + await accessor.RunConsumerCursor(consumer).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.NotEmpty(observer.DeliveredBatches); + Assert.All(observer.DeliveredBatches, batch => Assert.Equal(new[] { 10L, 20L }, + Assert.IsType(batch).BatchContainers.Select(item => item.SequenceToken.SequenceNumber))); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Equal(firstToken, Assert.IsType(consumer.DeliveryRecoveryToken).Token); + Assert.Null(consumer.LastProcessedToken); + Assert.Equal(lastToken, consumer.UnconfirmedDeliveryToken); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + + if (purgeFirstBatch) + { + queueCache.PurgeOldest(); + var acceptedOldPosition = queueCache.TryGetCacheCursor(streamId.StreamId, firstToken); + Assert.Equal(QueueCacheCursorResultKind.Success, acceptedOldPosition.Kind); + using var probe = Assert.IsAssignableFrom(acceptedOldPosition.Cursor); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, probe.MoveNextWithResult().Kind); + Assert.Equal(20, Assert.IsType(probe.GetCurrent(out _)).SequenceToken.SequenceNumber); + } + + observer.DeliveryException = null; + observer.DeliveredBatches.Clear(); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + Assert.NotEmpty(observer.DeliveredBatches); + Assert.All(observer.DeliveredBatches, batch => Assert.Equal( + purgeFirstBatch ? [20L] : new[] { 10L, 20L }, + Assert.IsType(batch).BatchContainers.Select(item => item.SequenceToken.SequenceNumber))); + Assert.Equal(purgeFirstBatch, consumer.HasDeliveryProgressError); + Assert.Equal(!purgeFirstBatch, consumer.IsCaughtUp); + if (purgeFirstBatch) + { + Assert.Equal(firstToken, Assert.IsType(consumer.DeliveryRecoveryToken).Token); + Assert.Null(consumer.LastProcessedToken); + } + else + { + Assert.Null(consumer.DeliveryRecoveryToken); + Assert.Equal(lastToken, consumer.LastProcessedToken); + } + + await accessor.Shutdown(); + if (purgeFirstBatch) + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + Assert.Equal(lastToken, Assert.Single(queueCache.DeliveryProgressTokens)); + } + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -3027,10 +3161,19 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( } failedCursor.Received(1).MoveNextWithResult(); - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(consumer.IsCaughtUp); + var recoveredInitially = !unknownPosition && progressKind != "rewind"; + Assert.Equal(!recoveredInitially, consumer.HasDeliveryProgressError); + Assert.Equal(recoveredInitially, consumer.IsCaughtUp); Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Same(safeToken, consumer.LastProcessedToken); + if (recoveredInitially) + { + Assert.Equal(progressKind == "reattach" ? 300 : 200, + Assert.IsType(consumer.LastProcessedToken).SequenceNumber); + } + else + { + Assert.Same(safeToken, consumer.LastProcessedToken); + } Assert.Equal(0, consumer.PendingHandshakes); if (progressKind == "filter") { @@ -3048,15 +3191,24 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( Assert.All(observer.DeliveredTokens, token => Assert.Same(laterToken, token)); } - // A delivery-token rewind retains the first newer batch on current providers. - // Accepting its redelivery must still not erase the earlier cursor failure. + // A later rewind cursor is not proof of replay from the original safe anchor. + // The next pump must retry that anchor before releasing the error. Assert.Null(consumer.UnconfirmedDeliveryToken); if (expireStream) { timeProvider.Advance(TimeSpan.FromDays(1)); Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); + if (unknownPosition) + { + Assert.Same(stream, Assert.Single(await accessor.GetPubSubCache()).Value); + Assert.True(consumer.HasDeliveryProgressError); + } + else + { + Assert.False(consumer.HasDeliveryProgressError); + Assert.Empty(await accessor.GetPubSubCache()); + } } await accessor.Shutdown(); @@ -3069,7 +3221,7 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( { var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); Assert.NotNull(checkpoint); - Assert.Equal(10, checkpoint.SequenceNumber); + Assert.Equal(recoveredInitially || expireStream ? 200 : 10, checkpoint.SequenceNumber); } } @@ -3144,7 +3296,11 @@ public async Task Shutdown_PreservesReplayRequirementAfterRewindToEmptyCursor(bo { timeProvider.Advance(TimeSpan.FromDays(1)); Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); + var retainedStream = Assert.Single(await accessor.GetPubSubCache()); + Assert.Equal(streamId, retainedStream.Key); + Assert.True(retainedStream.Value.Contains(consumer.SubscriptionId)); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); } await accessor.Shutdown(); @@ -3252,7 +3408,8 @@ public async Task Shutdown_PreservesReplayFloorForEarlierRequest( if (!recoveryIssued) { recoveryIssued = true; - return QueueCacheCursorResult.FromCursor(recoveryCursor); + return QueueCacheCursorResult.FromCursor( + reattach ? new EmptyQueueCacheCursor() : recoveryCursor); } if (call.ArgAt(1)?.SequenceNumber == earlierToken.SequenceNumber) @@ -3282,7 +3439,14 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( Assert.True(consumer.HasDeliveryProgressError); Assert.False(consumer.IsCaughtUp); Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); + if (reattach) + { + Assert.Empty(observer.DeliveredTokens); + } + else + { + Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); + } await accessor.Shutdown(); if (unknownPosition || tokenKind == "explicit" || reattach && tokenKind == "implicit") @@ -3370,21 +3534,50 @@ public async Task Shutdown_PreservesReplayFloorAfterAccountingFailure(bool failC Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); Assert.True(idleConsumer.IsCaughtUp); Assert.Equal(1, Assert.Single(idleObserver.DeliveredTokens).SequenceNumber); - Assert.Same(accountingError, await Assert.ThrowsAsync( - () => accessor.ReadFromQueue(queueId, receiver, 1000))); + if (failCacheAdd) + { + Assert.Same(accountingError, await Assert.ThrowsAsync( + () => accessor.ReadFromQueue(queueId, receiver, 1000))); + } + else + { + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + streams = await accessor.GetPubSubCache(); - Assert.False(streams.ContainsKey(failedId)); - Assert.False(streams.ContainsKey(tailId)); + await Task.WhenAll(streams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (failCacheAdd) + { + Assert.False(streams.ContainsKey(failedId)); + Assert.False(streams.ContainsKey(tailId)); + } + else + { + Assert.True(streams.ContainsKey(failedId)); + Assert.Equal(includeUnreadTail, streams.ContainsKey(tailId)); + var pendingRegistration = streams[failedId].RegistrationTask; + Assert.NotNull(pendingRegistration); + Assert.True(pendingRegistration.IsCompleted); + Assert.False(streams[failedId].StreamRegistered); + Assert.Equal(100, Assert.IsType(streams[failedId].RegistrationStartToken).SequenceNumber); + } + await pubSub.DidNotReceive().RegisterProducer(failedId, Arg.Any(), Arg.Any()); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(activeConsumer.IsCaughtUp); - Assert.Equal(200, Assert.Single(activeObserver.DeliveredTokens).SequenceNumber); - await accessor.Shutdown(); + Assert.Equal(failCacheAdd, await accessor.ReadFromQueue(queueId, receiver, 1000)); + if (failCacheAdd) + { + Assert.True(activeConsumer.IsCaughtUp); + Assert.Equal(200, Assert.Single(activeObserver.DeliveredTokens).SequenceNumber); + } + else + { + Assert.Empty(activeObserver.DeliveredTokens); + await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); + } - var checkpoint = Assert.Single(backingCache.DeliveryProgressTokens); - Assert.NotNull(checkpoint); - Assert.True(checkpoint.SequenceNumber <= 100, - $"Fetched position 100 was never accounted for, but shutdown advanced to {checkpoint.SequenceNumber}."); + await accessor.Shutdown(); + Assert.Empty(backingCache.DeliveryProgressTokens); StreamConsumerData AddConsumer(QualifiedStreamId streamId, RecordingConsumer observer) { @@ -3398,6 +3591,742 @@ StreamConsumerData AddConsumer(QualifiedStreamId streamId, RecordingConsumer obs } } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("delivery", "throw", false)] + [InlineData("delivery", "acquire-miss", false)] + [InlineData("delivery", "advance-miss", false)] + [InlineData("implicit", "throw", false)] + [InlineData("implicit", "acquire-miss", false)] + [InlineData("implicit", "advance-miss", false)] + [InlineData("explicit", "throw", false)] + [InlineData("explicit", "acquire-miss", false)] + [InlineData("delivery", "throw", true)] + [InlineData("delivery", "acquire-miss", true)] + [InlineData("delivery", "advance-miss", true)] + [InlineData("implicit", "throw", true)] + [InlineData("implicit", "acquire-miss", true)] + [InlineData("implicit", "advance-miss", true)] + [InlineData("explicit", "throw", true)] + [InlineData("explicit", "acquire-miss", true)] + [InlineData("delivery", "none", false)] + [InlineData("implicit", "none", false)] + [InlineData("explicit", "none", false)] + [InlineData("delivery", "none", true)] + [InlineData("implicit", "none", true)] + [InlineData("explicit", "none", true)] + public async Task Shutdown_PreservesReplayAfterHandshakeFallback(string tokenKind, string failureMode, bool reattach) + { + var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(_ => registration.Task); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var latestToken = new EventSequenceTokenV2(200); + var requestedPosition = new EventSequenceTokenV2(tokenKind == "explicit" ? 50 : 100); + var requestedToken = tokenKind == "delivery" + ? StreamHandshakeToken.CreateDeliveyToken(requestedPosition) + : StreamHandshakeToken.CreateStartToken(requestedPosition); + var latestBatch = new TestBatchContainer(streamId.StreamId, latestToken); + var acquisitionError = new InvalidOperationException("Injected handshake cursor acquisition failure."); + var missingCursor = Substitute.For(); + missingCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.FromCacheMiss(new("requested", "low", "high"))); + var queueCache = Substitute.For(); + var requestedCursorCount = 0; + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + if (call.ArgAt(1)?.SequenceNumber == requestedPosition.SequenceNumber) + { + requestedCursorCount++; + switch (failureMode) + { + case "throw": + throw acquisitionError; + case "acquire-miss": + return QueueCacheCursorResult.FromCacheMiss(new("requested", "low", "high")); + case "advance-miss": + return QueueCacheCursorResult.FromCursor(missingCursor); + } + } + + return QueueCacheCursorResult.FromCursor( + new ScriptedQueueCursor([latestBatch], streamId.StreamId, null)); + }); + var progress = new List(); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => progress.Add(call.ArgAt(0))); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([latestBatch])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var registrationTask = stream.RegistrationTask; + Assert.NotNull(registrationTask); + Assert.False(registrationTask.IsCompleted); + var subscriptionId = tokenKind == "implicit" + ? SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid()) + : SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); + Assert.Equal(tokenKind == "implicit", SubscriptionMarker.IsImplicitSubscription(subscriptionId)); + var observer = new RecordingConsumer(reattach ? null : requestedToken); + observer.ReleaseDelivery(); + var consumer = stream.AddConsumer( + GuidId.GetGuidId(subscriptionId), streamId, observer, null, DateTime.UtcNow); + Assert.False(consumer.IsRegistered); + Assert.False(consumer.HasDeliveryProgressError); + + // The real initial-registration path finds this consumer instead of resolving a grain reference. + registration.SetResult(new HashSet + { + new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), + }); + await registrationTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (reattach) + { + Assert.True(consumer.IsCaughtUp); + Assert.False(consumer.HasDeliveryProgressError); + Assert.Equal(latestToken, consumer.LastProcessedToken); + observer.DeliveredTokens.Clear(); + observer.HandshakeResponse = requestedToken; + await agent.RunOrQueueTask(() => agent.AddSubscriber( + consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, + TestContext.Current.CancellationToken)); + } + + Assert.True(stream.StreamRegistered); + Assert.True(consumer.IsRegistered); + Assert.Equal(0, consumer.PendingHandshakes); + Assert.True(requestedCursorCount >= 1); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.NotEmpty(observer.DeliveredTokens); + Assert.All(observer.DeliveredTokens, token => Assert.Same(latestToken, token)); + if (failureMode == "throw") + { + Assert.Contains(acquisitionError, observer.Errors); + } + else if (failureMode == "advance-miss") + { + missingCursor.Received().MoveNextWithResult(); + } + + await accessor.Shutdown(); + + if (failureMode == "none") + { + Assert.False(consumer.HasDeliveryProgressError); + Assert.Equal(latestToken, Assert.Single(progress)); + } + else + { + if (tokenKind == "delivery") + { + Assert.Equal(requestedPosition, Assert.Single(progress)); + Assert.Equal(requestedPosition, consumer.LastProcessedToken); + } + else + { + Assert.Empty(progress); + Assert.Null(consumer.LastProcessedToken); + } + + Assert.True(consumer.HasDeliveryProgressError); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, false, true)] + [InlineData(false, true, true)] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + public async Task Shutdown_PreservesReplayAfterUnorderedRead(bool separateReads, bool unknownFirst, bool nullContainer) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var knownId = StreamId.Create("namespace", Guid.NewGuid()); + var unknownId = StreamId.Create("namespace", Guid.NewGuid()); + var knownToken = new EventSequenceTokenV2(200); + IBatchContainer knownBatch = new TestBatchContainer(knownId, knownToken); + IBatchContainer unknownBatch = nullContainer ? null! : new TestBatchContainer(unknownId, null!); + var firstBatch = unknownFirst ? unknownBatch : knownBatch; + var secondBatch = unknownFirst ? knownBatch : unknownBatch; + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>(separateReads ? [firstBatch] : [firstBatch, secondBatch]), + Task.FromResult>([secondBatch])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + + await ReadAndFinishRegistration(); + if (separateReads) + { + await ReadAndFinishRegistration(); + } + + var streams = await accessor.GetPubSubCache(); + Assert.Equal(nullContainer ? 1 : 2, streams.Count); + Assert.All(streams.Values, stream => + { + Assert.True(stream.StreamRegistered); + Assert.Null(stream.RegistrationTask); + Assert.Equal(0, stream.Count); + }); + await accessor.Shutdown(); + + if (nullContainer) + { + Assert.Equal(knownToken, Assert.Single(queueCache.DeliveryProgressTokens)); + } + else + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + + async Task ReadAndFinishRegistration() + { + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var pendingStreams = await accessor.GetPubSubCache(); + await Task.WhenAll(pendingStreams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("exact", 1)] + [InlineData("exact", 2)] + [InlineData("fallback", 1)] + [InlineData("fallback", 2)] + [InlineData("empty", 1)] + [InlineData("empty", 2)] + public async Task Recovery_OnlyAcknowledgedExactReplayReleasesProgress(string replayKind, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100))]), + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(replayKind == "fallback" ? 500 : 200))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.Cursor = queueCache.GetCacheCursor(streamId.StreamId, null); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Equal(100, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); + Assert.False(consumer.HasDeliveryProgressError); + + var failedCursor = Substitute.For(); + var cursorError = new InvalidOperationException("Injected cursor failure after acknowledged position 100."); + if (replayKind == "fallback") + { + failedCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.FromCacheMiss(new("200", "500", "500"))); + } + else + { + failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); + } + consumer.Cursor = failedCursor; + if (replayKind != "exact") + { + queueCache.AddToCache([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]); + queueCache.Purge(); + } + + if (replayKind == "empty") + { + await accessor.RunConsumerCursor(consumer); + } + else + { + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + + if (replayKind == "fallback") + { + failedCursor.Received(1).MoveNextWithResult(); + Assert.Empty(observer.Errors); + } + else + { + Assert.Contains(cursorError, observer.Errors); + } + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + var caughtUp = consumer.IsCaughtUp; + await accessor.Shutdown(); + var expectedCheckpoint = replayKind == "exact" ? 200 : 100; + Assert.Equal(expectedCheckpoint, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + Assert.Equal(replayKind != "exact", consumer.HasDeliveryProgressError); + Assert.Equal(replayKind == "exact", caughtUp); + Assert.Equal( + replayKind == "empty" ? [100L] : new[] { 100L, replayKind == "fallback" ? 500L : 200L }, + observer.DeliveredTokens.Select(token => token.SequenceNumber)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Recovery_ExplicitStartSurvivesFallbackAndIdleCleanup(bool cleanupBeforeRecovery, bool cleanupAfterRecovery) + { + var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(_ => registration.Task); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var timeProvider = new FakeTimeProvider(); + var backingCache = new PurgeablePooledQueueCache(); + backingCache.AddToCache( + [ + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(50)), + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), + ]); + var queueCache = Substitute.For(); + queueCache.GetMaxAddCount().Returns(1000); + queueCache.When(cache => cache.AddToCache(Arg.Any>())) + .Do(call => backingCache.AddToCache(call.Arg>())); + var failEarlierCursor = true; + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + var token = call.ArgAt(1); + if (failEarlierCursor && token?.SequenceNumber == 50) + { + return QueueCacheCursorResult.FromCacheMiss(new("50", "100", "500")); + } + + return backingCache.TryGetCacheCursor(call.ArgAt(0), token); + }); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => backingCache.UpdateDeliveryProgress(call.ArgAt(0), call.Arg())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(500))]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var registrationTask = stream.RegistrationTask; + Assert.NotNull(registrationTask); + var observer = new RecordingConsumer(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(50))); + observer.ReleaseDelivery(); + var consumer = stream.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + registration.SetResult(new HashSet + { + new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), + }); + await registrationTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Null(consumer.LastProcessedToken); + Assert.Equal(500, Assert.Single(observer.DeliveredTokens).SequenceNumber); + Assert.Equal(500, Assert.IsType(Assert.IsType(consumer.LastToken).Token).SequenceNumber); + if (cleanupBeforeRecovery) + { + timeProvider.Advance(TimeSpan.FromDays(1)); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + Assert.True((await accessor.GetPubSubCache()).ContainsKey(streamId), "Unfulfilled inclusive replay must survive inactivity cleanup."); + } + + observer.DeliveredTokens.Clear(); + failEarlierCursor = false; + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + Assert.Equal(new[] { 50L, 100L, 500L }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.False(consumer.HasDeliveryProgressError); + Assert.True(consumer.IsCaughtUp); + if (cleanupAfterRecovery) + { + timeProvider.Advance(TimeSpan.FromDays(1)); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + Assert.Empty(await accessor.GetPubSubCache()); + } + + await accessor.Shutdown(); + Assert.Equal(500, Assert.IsType(Assert.Single(backingCache.DeliveryProgressTokens)).SequenceNumber); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("delivery", false, false, 1)] + [InlineData("delivery", false, true, 2)] + [InlineData("delivery", true, false, 2)] + [InlineData("delivery", true, true, 1)] + [InlineData("start", false, false, 2)] + [InlineData("start", false, true, 1)] + [InlineData("start", true, false, 1)] + [InlineData("start", true, true, 2)] + [InlineData("none", false, false, 1)] + [InlineData("none", false, false, 2)] + public async Task Recovery_StaleCallbackCannotAcknowledgeReattachedCursor( + string tokenKind, bool reuseCursor, bool lateRewind, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var attemptedToken = new EventSequenceTokenV2(200); + var earlierToken = new EventSequenceTokenV2(50); + var batch = new TestBatchContainer(streamId.StreamId, attemptedToken); + var oldCursor = Substitute.For(); + oldCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); + oldCursor.GetCurrent(out _).Returns(batch); + IQueueCacheCursor reboundCursor = reuseCursor ? oldCursor : new EmptyQueueCacheCursor(); + var queueCache = Substitute.For(); + var failEarlierRequest = true; + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + if (failEarlierRequest && call.ArgAt(1)?.SequenceNumber == 50) + { + return QueueCacheCursorResult.FromCacheMiss(new("50", "100", "200")); + } + + return QueueCacheCursorResult.FromCursor(reboundCursor); + }); + var progress = new List(); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => progress.Add(call.ArgAt(0))); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([batch])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var observer = new RecordingConsumer(); + if (lateRewind) + { + observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(400))); + } + + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new EventSequenceTokenV2(100); + consumer.LastToken = StreamHandshakeToken.CreateDeliveyToken(consumer.LastProcessedToken); + consumer.Cursor = oldCursor; + var delivery = accessor.RunConsumerCursor(consumer); + try + { + await observer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(delivery.IsCompleted); + Assert.Equal(100, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); + Assert.False(consumer.HasDeliveryProgressError); + if (tokenKind != "none") + { + observer.HandshakeResponse = tokenKind == "start" + ? StreamHandshakeToken.CreateStartToken(earlierToken) + : StreamHandshakeToken.CreateDeliveyToken(earlierToken); + await Reattach(); + Assert.True(consumer.HasDeliveryProgressError); + + // Rebind an exact-but-empty recovery cursor while the old delivery is still blocked. + // Some providers recycle the same cursor object, so identity alone cannot identify the callback. + failEarlierRequest = false; + await Reattach(); + Assert.Same(reboundCursor, consumer.Cursor); + Assert.Same(reboundCursor, consumer.DeliveryRecoveryCursor); + Assert.Equal(reuseCursor, ReferenceEquals(oldCursor, consumer.Cursor)); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Equal(0, consumer.PendingHandshakes); + Assert.Same(observer.HandshakeResponse, consumer.LastToken); + } + + observer.ReleaseDelivery(); + await delivery.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + if (tokenKind == "none") + { + Assert.False(consumer.HasDeliveryProgressError); + Assert.True(consumer.IsCaughtUp); + Assert.Null(consumer.UnconfirmedDeliveryToken); + Assert.Equal(200, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); + Assert.Equal(200, Assert.IsType( + Assert.IsType(consumer.LastToken).Token).SequenceNumber); + } + else + { + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(consumer.IsCaughtUp); + Assert.Same(observer.HandshakeResponse, consumer.LastToken); + Assert.Equal(earlierToken, consumer.DeliveryRecoveryToken?.Token); + Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); + Assert.Equal(tokenKind == "start" ? null : earlierToken, consumer.LastProcessedToken); + } + + await accessor.Shutdown(); + if (tokenKind == "start") + { + Assert.Empty(progress); + } + else + { + Assert.Equal(tokenKind == "none" ? 200 : 50, + Assert.IsType(Assert.Single(progress)).SequenceNumber); + } + } + finally + { + observer.ReleaseDelivery(); + } + + Task Reattach() => agent.RunOrQueueTask(() => agent.AddSubscriber( + consumer.SubscriptionId, streamId, GrainId.Create("test", "reattached-subscriber"), null, + TestContext.Current.CancellationToken)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Recovery_RetriesCanceledRegistrationFromOriginalCachedPosition() + { + var firstRegistration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var subscriptions = new HashSet + { + new(subscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), + }; + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(_ => firstRegistration.Task, _ => Task.FromResult>(subscriptions)); + var backoff = Substitute.For(); + // Cancel the first attempt's retry wait, without stopping the agent. + var cancelFirstWait = true; + backoff.Next(Arg.Any()).Returns(_ => + { + if (cancelFirstWait) + { + cancelFirstWait = false; + throw new OperationCanceledException("Transient registration retry cancellation."); + } + + return TimeSpan.Zero; + }); + var queueCache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>( + [ + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), + ]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, deliveryBackoff: backoff); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var firstTask = stream.RegistrationTask; + Assert.NotNull(firstTask); + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + stream.AddConsumer(subscriptionId, streamId, observer, null, DateTime.UtcNow); + firstRegistration.SetException(new InvalidOperationException("Transient pubsub registration failure.")); + await firstTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + var streams = await accessor.GetPubSubCache(); + await Task.WhenAll(streams.Values.Select(value => value.RegistrationTask ?? Task.CompletedTask)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await pubSub.Received(2).RegisterProducer(streamId, Arg.Any(), Arg.Any()); + Assert.Equal(new[] { 100L, 200L }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.Equal(1, observer.HandshakeCount); + await accessor.Shutdown(); + Assert.Equal(200, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Recovery_RetriesOnlyFailedInitialAttachmentsWithoutNewMessages() + { + var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken).ReturnsForAnyArgs(_ => registration.Task); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var failedId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var healthyId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); + var queueCache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>( + [ + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), + ]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var firstTask = stream.RegistrationTask; + Assert.NotNull(firstTask); + var healthy = new RecordingConsumer(); + healthy.ReleaseDelivery(); + stream.AddConsumer(healthyId, streamId, healthy, null, DateTime.UtcNow); + registration.SetResult(new HashSet + { + new(failedId, streamId, GrainId.Create("test", "initially-unavailable-subscriber")), + new(healthyId, streamId, GrainId.Create("test", "healthy-subscriber")), + }); + await firstTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, healthy.HandshakeCount); + Assert.Equal(new[] { 100L, 200L }, healthy.DeliveredTokens.Select(token => token.SequenceNumber)); + // The first attachment failed resolving its reference. Make it available for the retry. + var recovered = new RecordingConsumer(); + recovered.ReleaseDelivery(); + stream.AddConsumer(failedId, streamId, recovered, null, DateTime.UtcNow); + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + var streams = await accessor.GetPubSubCache(); + await Task.WhenAll(streams.Values.Select(value => value.RegistrationTask ?? Task.CompletedTask)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(1, recovered.HandshakeCount); + Assert.Equal(new[] { 100L, 200L }, recovered.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.Equal(1, healthy.HandshakeCount); + Assert.Equal(new[] { 100L, 200L }, healthy.DeliveredTokens.Select(token => token.SequenceNumber)); + await accessor.Shutdown(); + Assert.Equal(200, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Recovery_RemovingConsumerReleasesOnlyItsReplayConstraint(bool faultFirstConsumer, bool removeSecondConsumer) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var firstObserver = new RecordingConsumer(); + firstObserver.ReleaseDelivery(); + var first = AddConsumer(firstObserver, 100); + var second = AddConsumer(new RecordingConsumer(), 150); + await accessor.RunConsumerCursor(first); + await accessor.RunConsumerCursor(second); + Assert.True(first.HasDeliveryProgressError); + Assert.True(second.HasDeliveryProgressError); + if (faultFirstConsumer) + { + firstObserver.HandshakeResponse = new UnknownHandshakeToken(); + await agent.RunOrQueueTask(() => agent.AddSubscriber( + first.SubscriptionId, streamId, GrainId.Create("test", "faulted-subscriber"), null, + TestContext.Current.CancellationToken)); + await pubSub.Received(1).FaultSubscription(streamId, first.SubscriptionId, Arg.Any()); + } + else + { + await agent.RunOrQueueTask(() => agent.RemoveSubscriber( + first.SubscriptionId, streamId, TestContext.Current.CancellationToken)); + } + + Assert.False(stream.Contains(first.SubscriptionId)); + Assert.True(stream.Contains(second.SubscriptionId)); + Assert.True(second.HasDeliveryProgressError); + if (removeSecondConsumer) + { + await agent.RunOrQueueTask(() => agent.RemoveSubscriber( + second.SubscriptionId, streamId, TestContext.Current.CancellationToken)); + } + + await accessor.Shutdown(); + Assert.Equal(removeSecondConsumer ? 200 : 150, + Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + + StreamConsumerData AddConsumer(RecordingConsumer observer, long safePosition) + { + var consumer = stream.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new EventSequenceTokenV2(safePosition); + consumer.LastToken = StreamHandshakeToken.CreateDeliveyToken(consumer.LastProcessedToken); + var cursor = Substitute.For(); + cursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected unresolved cursor failure.")); + consumer.Cursor = cursor; + return consumer; + } + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] From 7cf24c1b11b57fa06dbdeb9ff4cc5679fdcf412f Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 16:30:27 -0700 Subject: [PATCH 3/9] fix(streaming): preserve recovery cursors when new data arrives - Skip refresh for tracked recovery cursors to preserve replay anchors and cache-miss detection without changing normal idle cursor wake-up. - Cover empty-cache recovery followed by new reads for inclusive and exclusive tokens, missing/restored replay, and grouped delivery. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 7 +- .../PersistentStreamPullingAgentTests.cs | 108 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 1cd455e9315..af6f980c4c7 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -1555,7 +1555,12 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq consumerData.IsCaughtUp = false; if (consumerData.IsRegistered) { - consumerData.Cursor?.Refresh(startToken); + // Preserve recovery positions so the next move can detect missing replay data. + if (consumerData.Cursor is { } cursor && !ReferenceEquals(cursor, consumerData.DeliveryRecoveryCursor)) + { + cursor.Refresh(startToken); + } + if (consumerData.State == StreamConsumerDataState.Inactive) { // wake up inactive consumers diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index af6c8d010d5..cb2faaea397 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -3988,6 +3988,114 @@ public async Task Recovery_ExplicitStartSurvivesFallbackAndIdleCleanup(bool clea Assert.Equal(500, Assert.IsType(Assert.Single(backingCache.DeliveryProgressTokens)).SequenceNumber); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false, 1)] + [InlineData(false, false, 2)] + [InlineData(false, true, 1)] + [InlineData(false, true, 2)] + [InlineData(true, false, 1)] + [InlineData(true, false, 2)] + [InlineData(true, true, 1)] + [InlineData(true, true, 2)] + public async Task Recovery_NewReadPreservesEmptyCursorReplayBoundary( + bool inclusiveReplay, bool replayAvailable, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var requiredToken = new EventSequenceTokenV2(100); + var latestToken = new EventSequenceTokenV2(500); + var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); + queueCache.AddToCache( + [ + new TestBatchContainer(streamId.StreamId, requiredToken), + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), + ]); + queueCache.Purge(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>(replayAvailable + ? + [ + new TestBatchContainer(streamId.StreamId, requiredToken), + new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), + new TestBatchContainer(streamId.StreamId, latestToken), + ] + : [new TestBatchContainer(streamId.StreamId, latestToken)])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, requiredToken, DateTime.UtcNow); + var observer = new RecordingConsumer(inclusiveReplay + ? StreamHandshakeToken.CreateStartToken(requiredToken) + : StreamHandshakeToken.CreateDeliveyToken(requiredToken)); + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); + consumer.IsRegistered = true; + Assert.False(consumer.HasDeliveryProgressError); + + var failedCursor = Substitute.For(); + var cursorError = new InvalidOperationException("Injected cursor failure before an empty-cache recovery."); + failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); + consumer.SafeDisposeCursor(NullLogger.Instance); + consumer.Cursor = failedCursor; + await accessor.RunConsumerCursor(consumer); + + Assert.Contains(cursorError, observer.Errors); + Assert.Empty(observer.DeliveredBatches); + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(consumer.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + var recoveryCursor = consumer.Cursor; + Assert.NotNull(recoveryCursor); + Assert.Same(recoveryCursor, consumer.DeliveryRecoveryCursor); + var recoveryToken = consumer.DeliveryRecoveryToken; + Assert.Equal(requiredToken, Assert.IsAssignableFrom(recoveryToken).Token); + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Equal(!replayAvailable, consumer.HasDeliveryProgressError); + Assert.Equal(replayAvailable, consumer.IsCaughtUp); + // A missing range must invalidate the recovery cursor, not silently reposition it to the new read. + Assert.Null(consumer.DeliveryRecoveryCursor); + if (replayAvailable) + { + Assert.Null(consumer.DeliveryRecoveryToken); + Assert.Equal(latestToken, consumer.LastProcessedToken); + } + else + { + Assert.Same(recoveryToken, consumer.DeliveryRecoveryToken); + Assert.Equal(inclusiveReplay ? null : requiredToken, consumer.LastProcessedToken); + } + + var deliveredTokens = observer.DeliveredBatches + .SelectMany(batch => batch is BatchContainerBatch group ? group.BatchContainers.AsEnumerable() : [batch]) + .Select(batch => batch.SequenceToken.SequenceNumber); + Assert.Equal(replayAvailable ? new[] { 100L, 200L, 500L } : [500L], deliveredTokens); + await accessor.Shutdown(); + if (inclusiveReplay && !replayAvailable) + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + Assert.Equal(replayAvailable ? latestToken : requiredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + } + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] From 5686e95b9068461f85d1ccaefb2136e2c24cd91c Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 18:07:59 -0700 Subject: [PATCH 4/9] fix(streaming): bound automatic checkpoint recovery retries - Bound replay recovery and outer registration restarts with provider backoff, a shared delivery deadline, and a finite attempt limit. - Release exhausted recovery resources while preserving checkpoint constraints and normal delivery of new traffic. - Reject detached handshake continuations and resume messages received during terminal failure notifications. - Reset unknown progress per receiver lifetime and recover null cursors. - Cover retry limits, failure policy, cleanup, and lifecycle races. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 262 ++++++- .../QueueStreamDataStructures.cs | 49 ++ .../StreamConsumerCollection.cs | 4 + .../PersistentStreamPullingAgentTests.cs | 720 +++++++++++++++++- 4 files changed, 995 insertions(+), 40 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index af6f980c4c7..4fe380318f0 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -22,6 +22,8 @@ namespace Orleans.Streams internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPersistentStreamPullingAgent, PersistentStreamPullingAgent.ITestAccessor { private const int ReadLoopRetryMax = 6; + // Like queue reads, automatic recovery stays finite even when the delivery timeout is unlimited. + private const int RecoveryAttemptMax = 6; private const int StreamInactivityCheckFrequency = 10; private readonly IBackoffProvider deliveryBackoffProvider; private readonly IBackoffProvider queueReaderBackoffProvider; @@ -48,6 +50,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; private StreamSequenceToken? _lastReadToken; + private StreamSequenceToken? _retainedDeliveryProgress; private bool _hasUnknownReplayPosition; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -158,7 +161,10 @@ public Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; + // Progress belongs to this receiver lifetime, not the reusable pulling-agent instance. _lastReadToken = null; + _retainedDeliveryProgress = null; + _hasUnknownReplayPosition = false; pendingConsumerRecoveries.Clear(); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; @@ -354,6 +360,7 @@ private async Task AddSubscriber_Impl( data.IsCaughtUp = false; data.PendingStartToken ??= cacheToken; data.PendingHandshakes++; + var attached = false; try { if (await DoHandshakeWithConsumer(data, cacheToken)) @@ -364,15 +371,19 @@ private async Task AddSubscriber_Impl( data.PendingStartToken = null; } data.IsRegistered = true; + attached = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); - if (data.State == StreamConsumerDataState.Inactive) - RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so } } finally { data.PendingHandshakes--; } + + if (attached && data.PendingHandshakes == 0 && data.State == StreamConsumerDataState.Inactive) + { + RunConsumerCursor(data, useCurrentCursor: true).Ignore(); + } } internal static StreamSequenceToken? GetInitialDeliveryProgress( @@ -389,7 +400,7 @@ private async Task DoHandshakeWithConsumer( StreamConsumerData consumerData, StreamSequenceToken? cacheToken) { - if (IsShutdown) return false; + if (IsShutdown || consumerData.IsRemoved) return false; StreamHandshakeToken? requestedHandshakeToken = null; StreamHandshakeToken? effectiveHandshakeToken = null; @@ -405,11 +416,12 @@ private async Task DoHandshakeWithConsumer( i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId), AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException - (exception, i) => exception is not ClientNotAvailableException && !IsShutdown, + (exception, i) => exception is not ClientNotAvailableException && !IsShutdown && !consumerData.IsRemoved, this.options.MaxEventDeliveryTime, deliveryBackoffProvider, cancellationToken: CancellationToken.None); + if (IsShutdown || consumerData.IsRemoved) return false; effectiveHandshakeToken = requestedHandshakeToken; if (effectiveHandshakeToken is null && ShouldApplyInitialSubscriptionStartPosition(consumerData)) { @@ -488,6 +500,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception exception) { + if (consumerData.IsRemoved) return false; RecordDeliveryProgressError(consumerData, effectiveHandshakeToken); exceptionOccured = exception; } @@ -801,6 +814,11 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea if (!pubSubCache.TryGetValue(streamId, out var streamData)) return; + if (streamData.TryGetConsumer(subscriptionId, out var consumer)) + { + pendingConsumerRecoveries.Remove(consumer); + } + // remove consumer bool removed = streamData.RemoveConsumer(subscriptionId, logger); if (removed) @@ -1053,8 +1071,8 @@ private void CleanupPubSubCache(DateTime now) { if (tuple.Value.IsInactive(now, options.StreamInactivityPeriod) && tuple.Value.RegistrationTask is null - && (queueCache is null - || TryGetStreamDeliveryProgress(tuple.Value, out var requiredToken) && requiredToken is null)) + && !tuple.Value.AllConsumers().Any(consumer => consumer.PendingHandshakes != 0 + || pendingConsumerRecoveries.Contains(consumer))) { inactiveStreams ??= new(); inactiveStreams.Add(tuple.Key); @@ -1070,6 +1088,7 @@ private void CleanupPubSubCache(DateTime now) { if (pubSubCache.Remove(streamId, out var streamData)) { + RetainStreamDeliveryProgress(streamData); streamData.DisposeAll(logger); StreamingEvents.EmitStreamInactive(streamProviderName, streamId.StreamId, options.StreamInactivityPeriod, Silo); } @@ -1093,7 +1112,8 @@ private void NotifyDeliveryProgress() private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) { earliest = _lastReadToken; - if (_hasUnknownReplayPosition) + if (_hasUnknownReplayPosition + || _retainedDeliveryProgress is not null && !IncludeProgress(_retainedDeliveryProgress, ref earliest)) { return false; } @@ -1110,6 +1130,16 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return true; } + private void RetainStreamDeliveryProgress(StreamConsumerCollection streamData) + { + // Reclaim inactive subscription records without forgetting their unresolved handoff constraint. + if (!TryGetStreamDeliveryProgress(streamData, out var token) + || token is not null && !IncludeProgress(token, ref _retainedDeliveryProgress)) + { + _hasUnknownReplayPosition = true; + } + } + private bool TryGetStreamDeliveryProgress(StreamConsumerCollection streamConsumers, out StreamSequenceToken? earliest) { earliest = null; @@ -1266,7 +1296,7 @@ private void TrackDeliveryRecoveryCursor( IQueueCacheCursor cursor, StreamHandshakeToken? request) { - if (consumer.HasDeliveryProgressError + if (consumer.HasDeliveryProgressError && consumer.DeliveryRecovery is not { Stopped: true } && consumer.DeliveryRecoveryToken is { Token: { } required } recovery && request?.Token is { } requested && TryCompareQueueProgress(requested, required, out var comparison) @@ -1322,6 +1352,9 @@ private void RecordAcknowledgedDeliveryProgress( consumer.DeliveryRecoveryToken = null; consumer.DeliveryRecoveryCursor = null; consumer.HasObservedRecoveryStart = false; + consumer.DeliveryRecovery = null; + consumer.DeliveryRecoveryFailureReported = false; + consumer.DeliveryRecoveryFailureToken = null; pendingConsumerRecoveries.Remove(consumer); } @@ -1337,8 +1370,10 @@ private void RecordAcknowledgedDeliveryProgress( private void RecordDeliveryProgressError( StreamConsumerData consumer, StreamHandshakeToken? replayRequest = null, - StreamSequenceToken? failedToken = null) + StreamSequenceToken? failedToken = null, + long? startedAt = null) { + if (consumer.IsRemoved) return; var request = NormalizeRecoveryToken(consumer, replayRequest); if (!consumer.HasDeliveryProgressError) { @@ -1374,6 +1409,7 @@ private void RecordDeliveryProgressError( if (!consumer.HasDeliveryProgressError) { + consumer.DeliveryRecovery = new(startedAt ?? _timeProvider.GetTimestamp()); LogWarningDeliveryRecoveryRequired(new(QueueId), consumer.SubscriptionId, consumer.StreamId); } @@ -1381,7 +1417,10 @@ private void RecordDeliveryProgressError( consumer.IsCaughtUp = false; consumer.DeliveryRecoveryCursor = null; consumer.HasObservedRecoveryStart = false; - pendingConsumerRecoveries.Add(consumer); + if (consumer.DeliveryRecovery is not { Stopped: true }) + { + pendingConsumerRecoveries.Add(consumer); + } } private void RetryPendingConsumers() @@ -1392,7 +1431,7 @@ private void RetryPendingConsumers() || !stream.TryGetConsumer(consumer.SubscriptionId, out var current) || !ReferenceEquals(current, consumer) || !consumer.HasDeliveryProgressError - || consumer.DeliveryRecoveryToken is null) + || consumer.DeliveryRecovery is { Stopped: true }) { pendingConsumerRecoveries.Remove(consumer); } @@ -1416,6 +1455,28 @@ private void RetryPendingWork(CancellationToken cancellationToken) } } + private async Task StopConsumerRecovery(StreamConsumerData consumer, CancellationToken cancellationToken) + { + if (consumer.DeliveryRecovery is not { Stopped: false } recovery) + { + return; + } + + recovery.Stopped = true; + pendingConsumerRecoveries.Remove(consumer); + consumer.SafeDisposeCursor(logger); + consumer.PendingStartToken = null; + consumer.PendingContinuationToken = null; + consumer.UnconfirmedDeliveryToken = null; + LogWarningDeliveryRecoveryStopped(new(QueueId), consumer.SubscriptionId, consumer.StreamId, recovery.Attempts); + if (!consumer.DeliveryRecoveryFailureReported) + { + // A replay boundary is not necessarily a failed event (failure handlers can dead-letter this token). + await ErrorProtocol(consumer, new StreamEventDeliveryFailureException(consumer.StreamId), + true, null, null, cancellationToken: cancellationToken); + } + } + private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => EventSequenceTokenCompatibility.Compare(current, other) < 0; @@ -1440,6 +1501,22 @@ private void RegisterStream( return; } + var recovery = streamData.RegistrationRecovery ??= new(_timeProvider.GetTimestamp()); + if (recovery.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) + { + recovery.Stopped = true; + RetainStreamDeliveryProgress(streamData); + foreach (var consumer in streamData.AllConsumers()) + { + pendingConsumerRecoveries.Remove(consumer); + } + pubSubCache.Remove(streamId); + streamData.DisposeAll(logger); + LogWarningStreamRegistrationRecoveryStopped(new(QueueId), streamId, recovery.Attempts); + return; + } + + if (!recovery.TryBeginAttempt(_timeProvider)) return; streamData.RegistrationTask = RegisterStreamAsync(); async Task RegisterStreamAsync() @@ -1503,11 +1580,13 @@ async Task RegisterStreamAsync() { streamData.RegistrationTask = null; streamData.RegistrationStartToken = null; + streamData.RegistrationRecovery = null; streamData.DisposeRegistrationCursor(logger); } else { streamData.RegistrationTask = Task.CompletedTask; + recovery.FinishAttempt(_timeProvider, deliveryBackoffProvider); if (IsShutdown || !pubSubCache.TryGetValue(streamId, out var current) || !ReferenceEquals(current, streamData)) { @@ -1555,6 +1634,12 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq consumerData.IsCaughtUp = false; if (consumerData.IsRegistered) { + if (consumerData.DeliveryRecovery is { Stopped: true } && consumerData.Cursor is null) + { + // A nonfaulting subscription still receives new traffic, but does not restart abandoned replay. + consumerData.PendingContinuationToken ??= startToken; + } + // Preserve recovery positions so the next move can detect missing replay data. if (consumerData.Cursor is { } cursor && !ReferenceEquals(cursor, consumerData.DeliveryRecoveryCursor)) { @@ -1578,27 +1663,62 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq } } - private async Task RunConsumerCursor(StreamConsumerData consumerData, CancellationToken cancellationToken = default) + private async Task RunConsumerCursor(StreamConsumerData consumerData, CancellationToken cancellationToken = default, bool useCurrentCursor = false) { TagList? tags = null; + StreamRecoveryState? recoveryAttempt = null; + var ownsCursor = false; try { // double check in case of interleaving - if (consumerData.State == StreamConsumerDataState.Active || - consumerData.Cursor is null) return; + if (IsShutdown || cancellationToken.IsCancellationRequested + || consumerData.IsRemoved || consumerData.State == StreamConsumerDataState.Active + || consumerData.PendingHandshakes != 0) return; - consumerData.State = StreamConsumerDataState.Active; - if (consumerData.HasDeliveryProgressError && consumerData.DeliveryRecoveryToken is not null + if (consumerData.HasDeliveryProgressError && consumerData.PendingHandshakes == 0 + && consumerData.DeliveryRecovery is { Stopped: false } recovery) + { + if (consumerData.DeliveryRecoveryToken is null && !useCurrentCursor + || recovery.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) + { + consumerData.State = StreamConsumerDataState.Active; + ownsCursor = true; + await StopConsumerRecovery(consumerData, cancellationToken); + return; + } + + if (!recovery.TryBeginAttempt(_timeProvider)) return; + recoveryAttempt = recovery; + } + + var needsRecovery = consumerData.HasDeliveryProgressError && consumerData.DeliveryRecoveryToken is not null + && consumerData.DeliveryRecovery is not { Stopped: true } + && !useCurrentCursor && consumerData.PendingHandshakes == 0 - && !ReferenceEquals(consumerData.Cursor, consumerData.DeliveryRecoveryCursor)) + && (consumerData.Cursor is null || !ReferenceEquals(consumerData.Cursor, consumerData.DeliveryRecoveryCursor)); + var continueNewTraffic = consumerData.DeliveryRecovery is { Stopped: true } + && consumerData.Cursor is null && consumerData.PendingContinuationToken is not null; + if (consumerData.Cursor is null && !needsRecovery && !continueNewTraffic) return; + + consumerData.State = StreamConsumerDataState.Active; + ownsCursor = true; + var replacedCursor = needsRecovery || continueNewTraffic; + if (needsRecovery) { consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetRecoveryCursor(consumerData); } + else if (continueNewTraffic) + { + var token = consumerData.PendingContinuationToken; + consumerData.PendingContinuationToken = null; + consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, token); + } var deliveredAny = false; while (!IsShutdown && !cancellationToken.IsCancellationRequested && consumerData.Cursor is not null) { + if (consumerData.PendingHandshakes != 0) break; var deliveryCursor = consumerData.Cursor; var cursorVersion = consumerData.CursorVersion; var batchCursor = options.BatchContainerBatchSize > 1 @@ -1616,6 +1736,17 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati { RecordDeliveryProgressError(consumerData); consumerData.SafeDisposeCursor(logger); + if (replacedCursor || consumerData.DeliveryRecovery is { Stopped: true }) + { + break; + } + + if (recoveryAttempt is null) + { + recoveryAttempt = consumerData.DeliveryRecovery!; + if (!recoveryAttempt.TryBeginAttempt(_timeProvider)) break; + } + replacedCursor = true; consumerData.Cursor = GetCacheMissRecoveryCursor(consumerData); continue; } @@ -1648,7 +1779,13 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati RecordDeliveryProgressError(consumerData); exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); - consumerData.Cursor = GetRecoveryCursor(consumerData); + if (recoveryAttempt is null && consumerData.DeliveryRecovery is { Stopped: false } recoveryState + && recoveryState.TryBeginAttempt(_timeProvider)) + { + recoveryAttempt = recoveryState; + replacedCursor = true; + consumerData.Cursor = GetRecoveryCursor(consumerData); + } } if (exceptionOccured is null) @@ -1662,6 +1799,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } } + long? deliveryStartedAt = null; try { if (_streamInstruments?.PersistentStreamSentMessages.Enabled is true) @@ -1685,14 +1823,26 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.UnconfirmedDeliveryToken = nextBatch.ProgressToken; } + var deliveryTime = options.MaxEventDeliveryTime; + if (consumerData.DeliveryRecovery is { Stopped: false } activeRecovery) + { + deliveryTime = activeRecovery.RemainingTime(_timeProvider, deliveryTime); + if (options.MaxEventDeliveryTime > TimeSpan.Zero && deliveryTime <= TimeSpan.Zero) + { + await StopConsumerRecovery(consumerData, cancellationToken); + break; + } + } + deliveryPending = true; + deliveryStartedAt = _timeProvider.GetTimestamp(); StreamHandshakeToken? newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( i => DeliverBatchToConsumer(consumerData, batch, cursorVersion, cancellationToken), - AsyncExecutorWithRetries.INFINITE_RETRIES, + consumerData.DeliveryRecovery is { Stopped: false } ? 1 : AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException (exception, i) => exception is not ClientNotAvailableException && !IsShutdown && consumerData.CursorVersion == cursorVersion, - this.options.MaxEventDeliveryTime, + deliveryTime, deliveryBackoffProvider, cancellationToken: cancellationToken); deliveryPending = false; @@ -1793,7 +1943,8 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati continue; } - RecordDeliveryProgressError(consumerData, failedToken: nextBatch.Batch?.SequenceToken); + RecordDeliveryProgressError(consumerData, failedToken: nextBatch.Batch?.SequenceToken, + startedAt: deliveryStartedAt); if (batchCursor is not null && nextBatch.Batch is not null) { batchCursor.RecordDeliveryFailure(nextBatch.Batch); @@ -1829,20 +1980,45 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } } } - consumerData.State = StreamConsumerDataState.Inactive; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { RecordDeliveryProgressError(consumerData); // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception LogErrorRunConsumerCursor(exc); - consumerData.State = StreamConsumerDataState.Inactive; throw; } + finally + { + if (ownsCursor) + { + try + { + if (recoveryAttempt is not null && ReferenceEquals(recoveryAttempt, consumerData.DeliveryRecovery) + && !recoveryAttempt.Stopped && !IsShutdown) + { + recoveryAttempt.FinishAttempt(_timeProvider, deliveryBackoffProvider); + if (recoveryAttempt.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) + { + await StopConsumerRecovery(consumerData, cancellationToken); + } + } + } + finally + { + consumerData.State = StreamConsumerDataState.Inactive; + if (!IsShutdown && !consumerData.IsRemoved && consumerData.PendingHandshakes == 0 + && consumerData.DeliveryRecovery is { Stopped: true } + && consumerData.PendingContinuationToken is not null) + { + RunConsumerCursor(consumerData).Ignore(); + } + } + } + } } private readonly struct ConsumerBatch @@ -2058,6 +2234,7 @@ private async Task ErrorProtocol( bool forceFaultSubscription = false, CancellationToken cancellationToken = default) { + if (consumerData.IsRemoved) return true; // for loss of client, we just remove the subscription if (exceptionOccured is ClientNotAvailableException) { @@ -2066,8 +2243,24 @@ private async Task ErrorProtocol( return true; } + if (consumerData.HasDeliveryProgressError) + { + if (consumerData.DeliveryRecovery is { Stopped: false } + && consumerData.DeliveryRecoveryFailureReported && !forceFaultSubscription + && consumerData.DeliveryRecoveryFailureIsDelivery == isDeliveryError + && Equals(consumerData.DeliveryRecoveryFailureToken, token)) + { + return false; + } + + consumerData.DeliveryRecoveryFailureReported = true; + consumerData.DeliveryRecoveryFailureIsDelivery = isDeliveryError; + consumerData.DeliveryRecoveryFailureToken = token; + } + // notify consumer about the error or that the data is not available. await DeliverErrorToConsumer(consumerData, exceptionOccured, batch, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + if (consumerData.IsRemoved) return true; // record that there was a delivery failure if (isDeliveryError) { @@ -2080,6 +2273,7 @@ await streamFailureHandler.OnSubscriptionFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } + if (consumerData.IsRemoved) return true; // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if ((forceFaultSubscription || streamFailureHandler.ShouldFaultSubsriptionOnError) && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) @@ -2093,13 +2287,17 @@ await DeliverErrorToConsumer( batch, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); + if (consumerData.IsRemoved) return true; // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId, cancellationToken); } finally { // remove subscription - RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); + if (!consumerData.IsRemoved) + { + RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); + } } return true; } @@ -2371,7 +2569,7 @@ private partial void LogWarningIncompatibleQueueProgress( [LoggerMessage( Level = LogLevel.Error, - Message = "Cannot establish delivery checkpoint progress for queue {Queue}. {Reason} Receiver/provider recovery is required; later reads or initialization alone do not establish replay continuity." + Message = "Cannot establish delivery checkpoint progress for queue {Queue}. {Reason} Receiver/provider recovery is required; later reads in this receiver lifetime do not establish replay continuity." )] private partial void LogErrorQueueRecoveryRequired(QueueIdLogRecord queue, string reason); @@ -2381,6 +2579,18 @@ private partial void LogWarningIncompatibleQueueProgress( )] private partial void LogWarningDeliveryRecoveryRequired(QueueIdLogRecord queue, GuidId subscriptionId, QualifiedStreamId streamId); + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Stopping automatic replay after {Attempts} attempts for subscription {SubscriptionId} on stream {StreamId} in queue {Queue}. The unresolved delivery checkpoint constraint is retained; nonfaulting subscriptions can continue with new traffic." + )] + private partial void LogWarningDeliveryRecoveryStopped(QueueIdLogRecord queue, GuidId subscriptionId, QualifiedStreamId streamId, int attempts); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Stopping automatic registration recovery after {Attempts} attempts for stream {StreamId} in queue {Queue}. Registration resources are released and delivery checkpoint progress remains unknown." + )] + private partial void LogWarningStreamRegistrationRecoveryStopped(QueueIdLogRecord queue, QualifiedStreamId streamId, int attempts); + [LoggerMessage( Level = LogLevel.Warning, EventId = (int)ErrorCode.PersistentStreamPullingAgent_13, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index d200ecd4ce7..56b0fd6185b 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -1,5 +1,6 @@ using System; using Microsoft.Extensions.Logging; +using Orleans.Internal; using Orleans.Runtime; namespace Orleans.Streams @@ -33,8 +34,12 @@ internal sealed class StreamConsumerData [NonSerialized] public bool IsRegistered = false; [NonSerialized] + public bool IsRemoved; + [NonSerialized] public StreamSequenceToken? PendingStartToken; [NonSerialized] + public StreamSequenceToken? PendingContinuationToken; + [NonSerialized] public IBatchContainer? PendingBatch; [NonSerialized] public bool StartPositionIsProviderDefault; @@ -64,6 +69,14 @@ internal sealed class StreamConsumerData public bool HasObservedRecoveryStart; [NonSerialized] public long CursorVersion; + [NonSerialized] + public StreamRecoveryState? DeliveryRecovery; + [NonSerialized] + public bool DeliveryRecoveryFailureReported; + [NonSerialized] + public bool DeliveryRecoveryFailureIsDelivery; + [NonSerialized] + public StreamSequenceToken? DeliveryRecoveryFailureToken; public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { @@ -102,7 +115,43 @@ internal void SafeDisposeCursor(ILogger logger) catch { } Utils.LogIgnoredException(logger, ex, caller); } + + } + } + } + + internal sealed class StreamRecoveryState(long startedAt) + { + private long _lastAttempt; + private TimeSpan _retryDelay; + + public int Attempts { get; private set; } + public bool Stopped { get; set; } + + public TimeSpan RemainingTime(TimeProvider timeProvider, TimeSpan maximumTime) + => maximumTime <= TimeSpan.Zero + ? Timeout.InfiniteTimeSpan + : maximumTime - timeProvider.GetElapsedTime(startedAt); + + public bool IsExhausted(TimeProvider timeProvider, TimeSpan maximumTime, int maximumAttempts) + => Stopped || Attempts >= maximumAttempts + || maximumTime > TimeSpan.Zero && RemainingTime(timeProvider, maximumTime) <= TimeSpan.Zero; + + public bool TryBeginAttempt(TimeProvider timeProvider) + { + if (Attempts != 0 && timeProvider.GetElapsedTime(_lastAttempt) < _retryDelay) + { + return false; } + + Attempts++; + return true; + } + + public void FinishAttempt(TimeProvider timeProvider, IBackoffProvider backoff) + { + _lastAttempt = timeProvider.GetTimestamp(); + _retryDelay = backoff.Next(Attempts - 1); } } } diff --git a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs index cbc6f1bca1c..4ab1cee73b8 100644 --- a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs +++ b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs @@ -26,6 +26,8 @@ internal sealed class StreamConsumerCollection public StreamSequenceToken? RegistrationStartToken; [NonSerialized] public IQueueCacheCursor? RegistrationCursor; + [NonSerialized] + public StreamRecoveryState? RegistrationRecovery; public StreamConsumerCollection(DateTime now) { @@ -45,6 +47,7 @@ public bool RemoveConsumer(GuidId subscriptionId, ILogger logger) { if (!queueData.Remove(subscriptionId, out var consumer)) return false; + consumer.IsRemoved = true; consumer.SafeDisposeCursor(logger); return true; } @@ -69,6 +72,7 @@ public void DisposeAll(ILogger logger) DisposeRegistrationCursor(logger); foreach (StreamConsumerData consumer in queueData.Values) { + consumer.IsRemoved = true; consumer.SafeDisposeCursor(logger); } queueData.Clear(); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index cb2faaea397..f859ff8da7b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -406,7 +406,8 @@ private static PersistentStreamPullingAgent CreateAgent( TimeProvider? timeProvider = null, StreamPullingAgentOptions? options = null, IStreamFilter? streamFilter = null, - IBackoffProvider? deliveryBackoff = null) + IBackoffProvider? deliveryBackoff = null, + IStreamFailureHandler? failureHandler = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -448,8 +449,8 @@ private static PersistentStreamPullingAgent CreateAgent( options ?? new StreamPullingAgentOptions(), queueAdapter, queueAdapterCache!, - new NoOpStreamDeliveryFailureHandler(), - deliveryBackoff ?? new FixedBackoff(TimeSpan.FromMilliseconds(1)), + failureHandler ?? new NoOpStreamDeliveryFailureHandler(), + deliveryBackoff ?? new FixedBackoff(TimeSpan.Zero), new FixedBackoff(TimeSpan.FromMilliseconds(1)), timeProvider ?? TimeProvider.System, shared); @@ -1072,6 +1073,9 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public Queue DeliveryResponses { get; } = new(); public StreamHandshakeToken? HandshakeResponse { get; set; } = requestedToken; public Exception? DeliveryException { get; set; } + public Action? OnDelivery { get; set; } + public Action? OnHandshake { get; set; } + public Task? HandshakeTask { get; set; } public int HandshakeCount { get; private set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) @@ -1087,6 +1091,7 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); await releaseDelivery.Task; + OnDelivery?.Invoke(); if (DeliveryException is { } exception) { throw exception; @@ -1106,7 +1111,8 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) { HandshakeCount++; - return Task.FromResult(HandshakeResponse); + OnHandshake?.Invoke(); + return HandshakeTask ?? Task.FromResult(HandshakeResponse); } public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); @@ -2173,6 +2179,7 @@ public async Task ProviderDefaultEarliestRejectsInvalidCustomCacheResult() public async Task RunConsumerCursorStopsAfterInvalidMoveResult() { var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); var queueCache = new InvalidMoveQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); @@ -2181,7 +2188,7 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( pubSub, - QueueId.GetQueueId("queue", 0u, 0u), + queueId, queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); @@ -2204,6 +2211,14 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() Assert.True(queueCache.Cursor.IsDisposed); Assert.Equal(StreamConsumerDataState.Inactive, consumerData.State); Assert.Equal(0, queueCache.CursorAcquisitionCount); + + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + + Assert.Null(consumerData.Cursor); + Assert.True(consumerData.HasDeliveryProgressError); + Assert.Null(consumerData.DeliveryRecoveryToken); + Assert.Equal(0, queueCache.CursorAcquisitionCount); + await accessor.Shutdown(); } private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamConsumerExtension @@ -2997,7 +3012,7 @@ public async Task Recovery_InitialGroupedFailureRequiresFirstBatchForInclusiveRe DeliveryException = new InvalidOperationException("Injected grouped delivery failure."), }; observer.ReleaseDelivery(); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, new FakeTimeProvider(), options: new StreamPullingAgentOptions { BatchContainerBatchSize = 2, @@ -3201,12 +3216,12 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); if (unknownPosition) { - Assert.Same(stream, Assert.Single(await accessor.GetPubSubCache()).Value); + Assert.Empty(await accessor.GetPubSubCache()); Assert.True(consumer.HasDeliveryProgressError); } else { - Assert.False(consumer.HasDeliveryProgressError); + Assert.Equal(!recoveredInitially, consumer.HasDeliveryProgressError); Assert.Empty(await accessor.GetPubSubCache()); } } @@ -3221,7 +3236,7 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( { var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); Assert.NotNull(checkpoint); - Assert.Equal(recoveredInitially || expireStream ? 200 : 10, checkpoint.SequenceNumber); + Assert.Equal(recoveredInitially ? 200 : 10, checkpoint.SequenceNumber); } } @@ -3296,11 +3311,11 @@ public async Task Shutdown_PreservesReplayRequirementAfterRewindToEmptyCursor(bo { timeProvider.Advance(TimeSpan.FromDays(1)); Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var retainedStream = Assert.Single(await accessor.GetPubSubCache()); - Assert.Equal(streamId, retainedStream.Key); - Assert.True(retainedStream.Value.Contains(consumer.SubscriptionId)); + Assert.Empty(await accessor.GetPubSubCache()); Assert.True(consumer.HasDeliveryProgressError); - Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); + Assert.Null(consumer.UnconfirmedDeliveryToken); + Assert.Null(consumer.Cursor); + Assert.Null(consumer.PendingBatch); } await accessor.Shutdown(); @@ -3704,6 +3719,11 @@ await agent.RunOrQueueTask(() => agent.AddSubscriber( Assert.True(consumer.IsRegistered); Assert.Equal(0, consumer.PendingHandshakes); Assert.True(requestedCursorCount >= 1); + if (reattach && tokenKind == "implicit" && failureMode == "advance-miss") + { + // A failed replacement yields rather than immediately opening another recovery cursor. + await accessor.RunConsumerCursor(consumer); + } Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); Assert.NotEmpty(observer.DeliveredTokens); Assert.All(observer.DeliveredTokens, token => Assert.Same(latestToken, token)); @@ -3943,7 +3963,8 @@ public async Task Recovery_ExplicitStartSurvivesFallbackAndIdleCleanup(bool clea receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(500))]), Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { MaxEventDeliveryTime = TimeSpan.FromDays(2) }); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); @@ -4096,6 +4117,113 @@ public async Task Recovery_NewReadPreservesEmptyCursorReplayBoundary( } } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false, 1)] + [InlineData(false, false, 2)] + [InlineData(false, true, 1)] + [InlineData(false, true, 2)] + [InlineData(true, false, 1)] + [InlineData(true, false, 2)] + [InlineData(true, true, 1)] + [InlineData(true, true, 2)] + public async Task Recovery_AfterContractFailureReacquiresDisposedCursor( + bool inclusiveReplay, bool replayAvailable, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var requiredToken = new EventSequenceTokenV2(100); + var latestToken = new EventSequenceTokenV2(200); + var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>( + [ + new TestBatchContainer(streamId.StreamId, requiredToken), + new TestBatchContainer(streamId.StreamId, latestToken), + ]), + Task.FromResult>([])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, requiredToken, DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var observer = new RecordingConsumer(inclusiveReplay + ? StreamHandshakeToken.CreateStartToken(requiredToken) + : StreamHandshakeToken.CreateDeliveyToken(requiredToken)); + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, DateTime.UtcNow); + Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); + consumer.IsRegistered = true; + consumer.SafeDisposeCursor(NullLogger.Instance); + var invalidCursor = new InvalidMoveCursor(); + consumer.Cursor = invalidCursor; + + await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); + + Assert.True(invalidCursor.IsDisposed); + Assert.Null(consumer.Cursor); + Assert.True(consumer.HasDeliveryProgressError); + var recoveryToken = consumer.DeliveryRecoveryToken; + Assert.Equal(requiredToken, Assert.IsAssignableFrom(recoveryToken).Token); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + consumer.PendingHandshakes++; + await accessor.RunConsumerCursor(consumer); + Assert.Null(consumer.Cursor); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + consumer.PendingHandshakes--; + if (!replayAvailable) + { + queueCache.Purge(); + } + + await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + + Assert.NotNull(consumer.Cursor); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Equal(!replayAvailable, consumer.HasDeliveryProgressError); + Assert.Equal(replayAvailable, consumer.IsCaughtUp); + if (replayAvailable) + { + Assert.Null(consumer.DeliveryRecoveryToken); + Assert.Equal(latestToken, consumer.LastProcessedToken); + var deliveredTokens = observer.DeliveredBatches + .SelectMany(batch => batch is BatchContainerBatch group ? group.BatchContainers.AsEnumerable() : [batch]) + .Select(batch => batch.SequenceToken.SequenceNumber); + Assert.Equal(inclusiveReplay ? new[] { 100L, 200L } : [200L], deliveredTokens); + } + else + { + Assert.Same(recoveryToken, consumer.DeliveryRecoveryToken); + Assert.Same(consumer.Cursor, consumer.DeliveryRecoveryCursor); + Assert.Equal(inclusiveReplay ? null : requiredToken, consumer.LastProcessedToken); + Assert.Empty(observer.DeliveredBatches); + } + + await accessor.Shutdown(); + if (inclusiveReplay && !replayAvailable) + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + Assert.Equal(replayAvailable ? latestToken : requiredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + } + + await accessor.RunConsumerCursor(consumer); + Assert.Null(consumer.Cursor); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4476,6 +4604,570 @@ public async Task Shutdown_DoesNotReusePreviousReceiverReadBoundaryAfterReinitia queueCache.DeliveryProgressTokens.Select(token => token?.SequenceNumber)); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("none")] + [InlineData("ordered")] + [InlineData("unknown")] + public async Task Recovery_AfterReinitializeTracksOnlyCurrentReceiverReads(string nextRead) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, null!)]), + Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]), + Task.FromResult>([new TestBatchContainer(streamId.StreamId, + nextRead == "unknown" ? null! : new EventSequenceTokenV2(10))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + await accessor.Shutdown(); + + // A later ordered read cannot repair the unknown position within the same receiver lifetime. + Assert.Empty(queueCache.DeliveryProgressTokens); + await InitializeAgent(agent); + if (nextRead != "none") + { + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + + await accessor.Shutdown(); + + adapterCache.Received(2).CreateQueueCache(queueId); + if (nextRead == "ordered") + { + Assert.Equal(10, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + } + else if (nextRead == "unknown") + { + Assert.Empty(queueCache.DeliveryProgressTokens); + } + else + { + Assert.All(queueCache.DeliveryProgressTokens, token => Assert.Null(token)); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Recovery_StopsRetryingPersistentContractFailures(bool implicitSubscription, bool faultOnError) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var timeProvider = new FakeTimeProvider(); + var requiredToken = new EventSequenceTokenV2(100); + var latestToken = new EventSequenceTokenV2(200); + var queueCache = Substitute.For(); + queueCache.GetMaxAddCount().Returns(1000); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + var progress = new List(); + queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) + .Do(call => progress.Add(call.ArgAt(0))); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([new TestBatchContainer(streamId.StreamId, latestToken)])); + var failureHandler = Substitute.For(); + failureHandler.ShouldFaultSubsriptionOnError.Returns(faultOnError); + var stopped = false; + failureHandler.OnDeliveryFailure(default!, default!, default, default).ReturnsForAnyArgs(_ => + { + stopped = true; + return Task.CompletedTask; + }); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(_ => stopped ? TimeSpan.Zero : TimeSpan.FromSeconds(1)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + deliveryBackoff: backoff, failureHandler: failureHandler); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, requiredToken, timeProvider.GetUtcNow().UtcDateTime); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + observer.OnDelivery = () => observer.DeliveryException = observer.DeliveredTokens.Count == 1 + ? new InvalidOperationException("Transient failure delivering fresh traffic.") + : null; + var subscriptionId = implicitSubscription + ? SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid()) + : SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(subscriptionId), streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + consumer.IsRegistered = true; + consumer.LastProcessedToken = requiredToken; + consumer.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); + var cursors = new List(); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + if (!Equals(call.ArgAt(1), requiredToken)) + { + return QueueCacheCursorResult.FromCursor( + new ScriptedQueueCursor([new TestBatchContainer(streamId.StreamId, latestToken)], streamId.StreamId, null)); + } + + var cursor = new InvalidMoveCursor(); + cursors.Add(cursor); + return QueueCacheCursorResult.FromCursor(cursor); + }); + + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Single(cursors); + for (var i = 0; i < 20; i++) + { + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + Assert.Single(cursors); + + for (var i = 0; i < 10; i++) + { + timeProvider.Advance(TimeSpan.FromSeconds(1)); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + + Assert.Equal(6, cursors.Count); + Assert.All(cursors, cursor => Assert.True(cursor.IsDisposed)); + Assert.Null(consumer.PendingBatch); + var shouldFault = faultOnError && !implicitSubscription; + await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, null); + Assert.True(consumer.DeliveryRecovery!.Stopped); + await pubSub.Received(shouldFault ? 1 : 0).FaultSubscription(streamId, consumer.SubscriptionId, Arg.Any()); + for (var i = 0; i < 20; i++) + { + timeProvider.Advance(TimeSpan.FromSeconds(1)); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + Assert.Equal(6, cursors.Count); + if (shouldFault) + { + Assert.Empty(observer.DeliveredTokens); + } + else + { + Assert.Equal(new[] { latestToken, latestToken }, observer.DeliveredTokens); + } + await accessor.Shutdown(); + Assert.Equal(shouldFault ? latestToken : requiredToken, Assert.Single(progress)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("contract", -1, false)] + [InlineData("acquisition", -1, false)] + [InlineData("cache-miss", -1, false)] + [InlineData("empty", -1, false)] + [InlineData("contract", 2, false)] + [InlineData("empty", 2, false)] + [InlineData("contract", -1, true)] + public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( + string failure, int maximumSeconds, bool holdFailureNotification) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var timeProvider = new FakeTimeProvider(); + var requiredToken = new EventSequenceTokenV2(100); + var latestToken = new EventSequenceTokenV2(200); + var cache = Substitute.For(); + var cachedBatches = new List(); + cache.When(value => value.AddToCache(Arg.Any>())) + .Do(call => cachedBatches.AddRange(call.Arg>())); + cache.GetMaxAddCount().Returns(1000); + cache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + var freshTraffic = true; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(_ => Task.FromResult>(freshTraffic + ? [new TestBatchContainer(streamId.StreamId, latestToken)] + : [])); + var failureHandler = Substitute.For(); + var failureNotification = new TaskCompletionSource(); + var reportingStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + failureHandler.OnDeliveryFailure(default!, default!, default, default) + .ReturnsForAnyArgs(_ => + { + reportingStarted.SetResult(); + return holdFailureNotification ? failureNotification.Task : Task.CompletedTask; + }); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(TimeSpan.FromSeconds(1)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions + { + MaxEventDeliveryTime = maximumSeconds < 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromSeconds(maximumSeconds), + }, + deliveryBackoff: backoff, failureHandler: failureHandler); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, requiredToken, timeProvider.GetUtcNow().UtcDateTime); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + freshTraffic = false; + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + consumer.IsRegistered = true; + consumer.PendingStartToken = requiredToken; + consumer.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); + var acquisitions = 0; + var cursors = new List(); + cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => + { + if (consumer.DeliveryRecovery is { Stopped: true }) + { + var start = call.ArgAt(1)!; + return QueueCacheCursorResult.FromCursor(new ScriptedQueueCursor( + cachedBatches.Where(batch => batch.SequenceToken.SequenceNumber >= start.SequenceNumber).ToList(), + streamId.StreamId, null)); + } + Assert.Equal(requiredToken, call.ArgAt(1)); + acquisitions++; + if (failure == "acquisition") throw new InvalidOperationException("Injected acquisition failure."); + var cursor = Substitute.For(); + cursor.MoveNextWithResult().Returns(failure switch + { + "cache-miss" => QueueCacheCursorMoveResult.FromCacheMiss(new("100", "200", "200")), + "empty" => QueueCacheCursorMoveResult.NoData, + _ => default, + }); + cursors.Add(cursor); + return QueueCacheCursorResult.FromCursor(cursor); + }); + + var expectedAttempts = maximumSeconds < 0 ? 6 : maximumSeconds; + for (var i = 0; i < expectedAttempts - 1; i++) + { + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Equal(i + 1, consumer.DeliveryRecovery!.Attempts); + for (var j = 0; j < 10; j++) + { + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + Assert.Equal(i + 1, consumer.DeliveryRecovery.Attempts); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + } + + var finalAttempt = accessor.RunConsumerCursor(consumer); + if (maximumSeconds >= 0) + { + if (failure == "contract") + { + await Assert.ThrowsAnyAsync(() => finalAttempt); + } + else + { + await finalAttempt; + } + timeProvider.Advance(TimeSpan.FromSeconds(1)); + finalAttempt = accessor.RunConsumerCursor(consumer); + } + await reportingStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(consumer.DeliveryRecovery!.Stopped); + Assert.Null(consumer.Cursor); + Assert.Null(consumer.PendingBatch); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Null(consumer.LastProcessedToken); + Assert.All(cursors, cursor => cursor.Received(1).Dispose()); + Assert.Equal(expectedAttempts, consumer.DeliveryRecovery.Attempts); + Assert.Equal(failure == "empty" ? 1 : expectedAttempts, acquisitions); + if (holdFailureNotification) + { + Assert.False(finalAttempt.IsCompleted); + Assert.Equal(StreamConsumerDataState.Active, consumer.State); + for (var i = 0; i < 10; i++) + { + await accessor.RunConsumerCursor(consumer); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + Assert.Equal(expectedAttempts, acquisitions); + freshTraffic = true; + latestToken = new EventSequenceTokenV2(300); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + latestToken = new EventSequenceTokenV2(400); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + freshTraffic = false; + Assert.Equal(300, consumer.PendingContinuationToken!.SequenceNumber); + failureNotification.SetResult(); + } + if (failure is "contract" or "acquisition" && maximumSeconds < 0) + { + await Assert.ThrowsAnyAsync(() => finalAttempt); + } + else + { + await finalAttempt; + } + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + if (holdFailureNotification) + { + Assert.Equal(new long[] { 300, 400 }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); + } + await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, null); + + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + await accessor.Shutdown(); + cache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool failAttachment) + { + var registration = new TaskCompletionSource>(); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(registration.Task); + var timeProvider = new FakeTimeProvider(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var requiredToken = new EventSequenceTokenV2(100); + var pin = Substitute.For(); + var cache = Substitute.For(); + cache.GetMaxAddCount().Returns(1000); + var acquisitions = 0; + cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(_ => + { + acquisitions++; + if (failAttachment && acquisitions == 1) + { + return QueueCacheCursorResult.FromCursor(pin); + } + throw new InvalidOperationException("Injected registration cursor failure."); + }); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, requiredToken)]), + Task.FromResult>([])); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(TimeSpan.FromSeconds(1)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { MaxEventDeliveryTime = Timeout.InfiniteTimeSpan }, + deliveryBackoff: backoff); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var firstAttempt = stream.RegistrationTask!; + var subscriptions = new HashSet(); + var observer = new RecordingConsumer(); + StreamConsumerData? consumer = null; + if (failAttachment) + { + consumer = stream.AddConsumer(GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + subscriptions.Add(new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"))); + } + registration.SetResult(subscriptions); + await firstAttempt; + for (var attempt = 1; attempt < 6; attempt++) + { + Assert.Equal(attempt, stream.RegistrationRecovery!.Attempts); + var before = acquisitions; + for (var read = 0; read < 10; read++) + { + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } + Assert.Equal(before, acquisitions); + if (failAttachment) + { + Assert.Same(pin, stream.RegistrationCursor); + pin.DidNotReceive().Dispose(); + } + timeProvider.Advance(TimeSpan.FromSeconds(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + await stream.RegistrationTask!; + } + + Assert.Equal(6, stream.RegistrationRecovery!.Attempts); + Task? handshake = null; + var handshakeResponse = new TaskCompletionSource(); + if (consumer is not null) + { + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + observer.OnHandshake = () => handshakeStarted.SetResult(); + observer.HandshakeTask = handshakeResponse.Task; + handshake = accessor.DoHandshakeWithConsumer(consumer, requiredToken); + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + var finalAcquisitions = acquisitions; + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(stream.RegistrationRecovery.Stopped); + Assert.Empty(await accessor.GetPubSubCache()); + Assert.Null(stream.RegistrationCursor); + Assert.Equal(0, stream.Count); + if (failAttachment) pin.Received(1).Dispose(); + if (handshake is not null) + { + cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(_ => + { + acquisitions++; + return QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor()); + }); + handshakeResponse.SetResult(StreamHandshakeToken.CreateStartToken(requiredToken)); + Assert.False(await handshake); + Assert.True(consumer!.IsRemoved); + Assert.Null(consumer.Cursor); + await accessor.RunConsumerCursor(consumer); + Assert.Empty(observer.DeliveredTokens); + } + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Equal(finalAcquisitions, acquisitions); + await accessor.Shutdown(); + cache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Recovery_DoesNotRestartAnExpiredDeliveryWindow() + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var timeProvider = new FakeTimeProvider(); + var processed = new EventSequenceTokenV2(90); + var failed = new EventSequenceTokenV2(100); + var cache = new ScriptedQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>([new TestBatchContainer(streamId.StreamId, failed)]), + Task.FromResult>([])); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(_ => throw new OperationCanceledException("End the initial callback's retry wait.")); + var failureHandler = Substitute.For(); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { MaxEventDeliveryTime = TimeSpan.FromSeconds(1) }, + deliveryBackoff: backoff, failureHandler: failureHandler); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, failed, timeProvider.GetUtcNow().UtcDateTime); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var observer = new RecordingConsumer + { + OnDelivery = () => timeProvider.Advance(TimeSpan.FromSeconds(2)), + DeliveryException = new InvalidOperationException("Injected delivery failure."), + }; + observer.ReleaseDelivery(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + consumer.IsRegistered = true; + consumer.LastProcessedToken = processed; + consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, processed); + await accessor.RunConsumerCursor(consumer); + Assert.True(consumer.HasDeliveryProgressError); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(consumer.DeliveryRecovery!.Stopped); + Assert.Equal(0, consumer.DeliveryRecovery.Attempts); + Assert.Null(consumer.Cursor); + Assert.Null(consumer.PendingBatch); + Assert.Equal(failed, Assert.Single(observer.DeliveredTokens)); + await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, failed); + await failureHandler.DidNotReceive().OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, processed); + await accessor.Shutdown(); + Assert.Equal(processed, Assert.Single(cache.DeliveryProgressTokens)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Recovery_YieldsWhenTheReplacementCursorAlsoMisses() + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var cache = Substitute.For(); + cache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var agent = CreateAgent(pubSub, queueId, queueAdapterCache: adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, new RecordingConsumer(), null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new EventSequenceTokenV2(100); + var moves = 0; + var cursors = new List(); + IQueueCacheCursor CreateMissingCursor() + { + if (cursors.Count > 10) throw new InvalidOperationException("Recovery did not yield."); + var cursor = Substitute.For(); + cursor.MoveNextWithResult().Returns(_ => + { + moves++; + return QueueCacheCursorMoveResult.FromCacheMiss(new("100", "200", "200")); + }); + cursors.Add(cursor); + return cursor; + } + cache.TryGetCacheCursorAtPosition(streamId.StreamId, StreamSubscriptionStartPosition.EarliestAvailable) + .Returns(_ => QueueCacheCursorResult.FromCursor(CreateMissingCursor())); + consumer.Cursor = CreateMissingCursor(); + await accessor.RunConsumerCursor(consumer); + Assert.Equal(2, moves); + Assert.Equal(2, cursors.Count); + Assert.All(cursors, cursor => cursor.Received(1).Dispose()); + Assert.Null(consumer.Cursor); + Assert.True(consumer.HasDeliveryProgressError); + Assert.Equal(1, consumer.DeliveryRecovery!.Attempts); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + await accessor.Shutdown(); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] From e25aaafa21988ea333456606ab69957f1aaf1238 Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 20:15:10 -0700 Subject: [PATCH 5/9] fix(streaming): preserve completed checkpoints without removal scans - Retain proven progress when local subscription records are removed. - Do not advance to later reads solely because no subscribers remain. - Keep unsubscribe progress bookkeeping constant-time. - Handle canceled backoff without resetting bounded recovery budgets. - Clarify checkpoint semantics and cover lifecycle and scaling cases. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 52 ++- .../QueueStreamDataStructures.cs | 12 +- .../QueueAdapters/IQueueCache.cs | 7 +- .../PersistentStreamPullingAgentTests.cs | 432 ++++++++++++++++-- 4 files changed, 465 insertions(+), 38 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 4fe380318f0..9d64bd0e2a1 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -51,6 +51,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task _activePumpTask = Task.CompletedTask; private StreamSequenceToken? _lastReadToken; private StreamSequenceToken? _retainedDeliveryProgress; + private StreamSequenceToken? _completedDeliveryProgress; private bool _hasUnknownReplayPosition; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -164,6 +165,7 @@ public Task Initialize(CancellationToken cancellationToken) // Progress belongs to this receiver lifetime, not the reusable pulling-agent instance. _lastReadToken = null; _retainedDeliveryProgress = null; + _completedDeliveryProgress = null; _hasUnknownReplayPosition = false; pendingConsumerRecoveries.Clear(); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; @@ -816,6 +818,7 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea if (streamData.TryGetConsumer(subscriptionId, out var consumer)) { + RetainCompletedDeliveryProgress(consumer.LastProcessedToken); pendingConsumerRecoveries.Remove(consumer); } @@ -1084,6 +1087,11 @@ private void CleanupPubSubCache(DateTime now) return; } + if (TryGetDeliveryProgress(out var progress)) + { + RetainCompletedDeliveryProgress(progress); + } + foreach (var streamId in inactiveStreams) { if (pubSubCache.Remove(streamId, out var streamData)) @@ -1111,9 +1119,9 @@ private void NotifyDeliveryProgress() private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) { - earliest = _lastReadToken; - if (_hasUnknownReplayPosition - || _retainedDeliveryProgress is not null && !IncludeProgress(_retainedDeliveryProgress, ref earliest)) + earliest = _retainedDeliveryProgress; + var hasSubscriptions = false; + if (_hasUnknownReplayPosition) { return false; } @@ -1125,6 +1133,20 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) { return false; } + + hasSubscriptions |= streamConsumers.Count > 0; + } + + // Removed subscriptions preserve proven progress, not evidence that subsequent reads were delivered. + if (!hasSubscriptions && _completedDeliveryProgress is { } completed + && !IncludeProgress(completed, ref earliest)) + { + return false; + } + + if ((hasSubscriptions || earliest is not null) && _lastReadToken is { } read) + { + return IncludeProgress(read, ref earliest); } return true; @@ -1132,6 +1154,11 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) private void RetainStreamDeliveryProgress(StreamConsumerCollection streamData) { + foreach (var consumer in streamData.AllConsumers()) + { + RetainCompletedDeliveryProgress(consumer.LastProcessedToken); + } + // Reclaim inactive subscription records without forgetting their unresolved handoff constraint. if (!TryGetStreamDeliveryProgress(streamData, out var token) || token is not null && !IncludeProgress(token, ref _retainedDeliveryProgress)) @@ -1140,6 +1167,21 @@ private void RetainStreamDeliveryProgress(StreamConsumerCollection streamData) } } + private void RetainCompletedDeliveryProgress(StreamSequenceToken? token) + { + if (token is null + || _lastReadToken is { } read && !TryCompareQueueProgress(token, read, out _)) + { + return; + } + + if (_completedDeliveryProgress is null + || TryCompareQueueProgress(token, _completedDeliveryProgress, out var comparison) && comparison > 0) + { + _completedDeliveryProgress = token; + } + } + private bool TryGetStreamDeliveryProgress(StreamConsumerCollection streamConsumers, out StreamSequenceToken? earliest) { earliest = null; @@ -1586,7 +1628,7 @@ async Task RegisterStreamAsync() else { streamData.RegistrationTask = Task.CompletedTask; - recovery.FinishAttempt(_timeProvider, deliveryBackoffProvider); + recovery.FinishAttempt(_timeProvider, deliveryBackoffProvider, logger); if (IsShutdown || !pubSubCache.TryGetValue(streamId, out var current) || !ReferenceEquals(current, streamData)) { @@ -2000,7 +2042,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (recoveryAttempt is not null && ReferenceEquals(recoveryAttempt, consumerData.DeliveryRecovery) && !recoveryAttempt.Stopped && !IsShutdown) { - recoveryAttempt.FinishAttempt(_timeProvider, deliveryBackoffProvider); + recoveryAttempt.FinishAttempt(_timeProvider, deliveryBackoffProvider, logger); if (recoveryAttempt.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) { await StopConsumerRecovery(consumerData, cancellationToken); diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index 56b0fd6185b..38427afcd28 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -148,10 +148,18 @@ public bool TryBeginAttempt(TimeProvider timeProvider) return true; } - public void FinishAttempt(TimeProvider timeProvider, IBackoffProvider backoff) + public void FinishAttempt(TimeProvider timeProvider, IBackoffProvider backoff, ILogger logger) { _lastAttempt = timeProvider.GetTimestamp(); - _retryDelay = backoff.Next(Attempts - 1); + try + { + _retryDelay = backoff.Next(Attempts - 1); + } + catch (OperationCanceledException exception) + { + _retryDelay = TimeSpan.Zero; + Utils.LogIgnoredException(logger, exception, "Stream recovery backoff was canceled. Retaining bounded recovery without a retry delay."); + } } } } diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs index cd27e0da14f..aa97bb754f1 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs @@ -99,11 +99,12 @@ QueueCacheCursorResult TryGetCacheCursorAtPosition( bool IsUnderPressure(); /// - /// Updates the cache with the current delivery progress of all active subscriptions. + /// Updates the cache with the safe delivery-based checkpoint position. /// /// - /// The earliest last processed sequence token across registered subscriptions. - /// A value indicates that there are no active subscriptions. + /// The delivery position constrained by subscriptions which still have outstanding work. + /// Includes previously established progress when subscription records have been removed. + /// A value indicates that no delivery progress is known. /// The token is only valid for the duration of the call and must not be stored. /// /// The current UTC time. diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index f859ff8da7b..6e87480150e 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -604,6 +604,17 @@ public void RecordDeliveryFailure() } } + private static StreamConsumerData AddRegisteredConsumer( + StreamConsumerCollection stream, QualifiedStreamId streamId, IQueueCache cache, DateTime? now = null) + { + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + var consumer = stream.AddConsumer(GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now ?? DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, null); + return consumer; + } + private sealed class ScriptedQueueCache : IQueueCache { private readonly List messages = new(); @@ -1061,6 +1072,16 @@ private sealed class DerivedEventSequenceTokenV2(long sequenceNumber, int eventI protected override Type SequenceTokenCompatibilityDomain => typeof(EventSequenceToken); } + private sealed class CountingEventSequenceToken(long sequenceNumber, Action onComparison) + : EventSequenceTokenV2(sequenceNumber) + { + public override int CompareTo(StreamSequenceToken? other) + { + onComparison(); + return base.CompareTo(other); + } + } + private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = null) : IStreamConsumerExtension { private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -2930,7 +2951,7 @@ public async Task Shutdown_DoesNotAdvanceToReadCancelledAfterReceiverReturns(boo .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = StreamId.Create("namespace", Guid.NewGuid()); - var queueCache = new RecordingQueueCache(); + var queueCache = new ScriptedQueueCache(); var adapterCache = Substitute.For(); adapterCache.CreateQueueCache(queueId).Returns(queueCache); var readStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -2951,7 +2972,10 @@ public async Task Shutdown_DoesNotAdvanceToReadCancelledAfterReceiverReturns(boo var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); await accessor.RegisterStream(new("provider", streamId), new EventSequenceTokenV2(10), DateTime.UtcNow); + var consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, new("provider", streamId), queueCache); Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(consumer.IsCaughtUp); + Assert.Equal(10, consumer.LastProcessedToken!.SequenceNumber); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); var pump = accessor.RunQueuePump(queueId, cancellation.Token); await readStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -3814,7 +3838,7 @@ public async Task Shutdown_PreservesReplayAfterUnorderedRead(bool separateReads, if (nullContainer) { - Assert.Equal(knownToken, Assert.Single(queueCache.DeliveryProgressTokens)); + Assert.Null(Assert.Single(queueCache.DeliveryProgressTokens)); } else { @@ -4545,8 +4569,7 @@ await agent.RunOrQueueTask(() => agent.RemoveSubscriber( } await accessor.Shutdown(); - Assert.Equal(removeSecondConsumer ? 200 : 150, - Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + Assert.Equal(150, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); StreamConsumerData AddConsumer(RecordingConsumer observer, long safePosition) { @@ -4563,6 +4586,61 @@ StreamConsumerData AddConsumer(RecordingConsumer observer, long safePosition) } } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(1)] + [InlineData(16)] + [InlineData(256)] + public async Task Removal_RetainsProgressWithLinearTokenComparisons(int subscriberCount) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var queueCache = new RecordingQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var comparisons = 0; + Action onComparison = () => comparisons++; + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( + Task.FromResult>( + [new TestBatchContainer(streamId.StreamId, new CountingEventSequenceToken(subscriberCount + 1, onComparison))])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var stream = (await accessor.GetPubSubCache()).Single().Value; + var consumers = new List(); + for (var i = 0; i < subscriberCount; i++) + { + var consumer = stream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, new RecordingConsumer(), null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.LastProcessedToken = new CountingEventSequenceToken(i + 1, onComparison); + consumers.Add(consumer); + } + + comparisons = 0; + await agent.RunOrQueueTask(() => + { + foreach (var consumer in consumers) + { + agent.RemoveSubscriber_Impl(consumer.SubscriptionId, streamId); + } + return Task.CompletedTask; + }); + + Assert.Equal(2 * subscriberCount - 1, comparisons); + Assert.Empty(await accessor.GetPubSubCache()); + await accessor.Shutdown(); + Assert.Equal(subscriberCount, Assert.Single(queueCache.DeliveryProgressTokens)!.SequenceNumber); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -4576,9 +4654,10 @@ public async Task Shutdown_DoesNotReusePreviousReceiverReadBoundaryAfterReinitia .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new RecordingQueueCache(); + var queueCache = new ScriptedQueueCache(); + var nextQueueCache = new ScriptedQueueCache(); var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); + adapterCache.CreateQueueCache(queueId).Returns(queueCache, nextQueueCache); var receiver = Substitute.For(); receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]), @@ -4587,21 +4666,29 @@ public async Task Shutdown_DoesNotReusePreviousReceiverReadBoundaryAfterReinitia var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), DateTime.UtcNow); + var consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, queueCache); Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(consumer.IsCaughtUp); + Assert.Equal(200, consumer.LastProcessedToken!.SequenceNumber); + await agent.RunOrQueueTask(() => agent.RemoveSubscriber( + consumer.SubscriptionId, streamId, TestContext.Current.CancellationToken)); await accessor.Shutdown(); await InitializeAgent(agent); if (readAfterReinitialize) { await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, nextQueueCache); Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(consumer.IsCaughtUp); + Assert.Equal(10, consumer.LastProcessedToken!.SequenceNumber); } await accessor.Shutdown(); Assert.Equal( new long?[] { 200, readAfterReinitialize ? 10 : null }, - queueCache.DeliveryProgressTokens.Select(token => token?.SequenceNumber)); + queueCache.DeliveryProgressTokens.Concat(nextQueueCache.DeliveryProgressTokens).Select(token => token?.SequenceNumber)); } [TestSuite("BVT")] @@ -4619,8 +4706,9 @@ public async Task Recovery_AfterReinitializeTracksOnlyCurrentReceiverReads(strin var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); var queueCache = new RecordingQueueCache(); + var nextQueueCache = new ScriptedQueueCache(); var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); + adapterCache.CreateQueueCache(queueId).Returns(queueCache, nextQueueCache); var receiver = Substitute.For(); receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( Task.FromResult>([new TestBatchContainer(streamId.StreamId, null!)]), @@ -4641,6 +4729,10 @@ public async Task Recovery_AfterReinitializeTracksOnlyCurrentReceiverReads(strin if (nextRead != "none") { await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + if (nextRead == "ordered") + { + AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, nextQueueCache); + } Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); } @@ -4649,15 +4741,15 @@ public async Task Recovery_AfterReinitializeTracksOnlyCurrentReceiverReads(strin adapterCache.Received(2).CreateQueueCache(queueId); if (nextRead == "ordered") { - Assert.Equal(10, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); + Assert.Equal(10, Assert.IsType(Assert.Single(nextQueueCache.DeliveryProgressTokens)).SequenceNumber); } else if (nextRead == "unknown") { - Assert.Empty(queueCache.DeliveryProgressTokens); + Assert.Empty(nextQueueCache.DeliveryProgressTokens); } else { - Assert.All(queueCache.DeliveryProgressTokens, token => Assert.Null(token)); + Assert.Null(Assert.Single(nextQueueCache.DeliveryProgressTokens)); } } @@ -4771,22 +4863,23 @@ public async Task Recovery_StopsRetryingPersistentContractFailures(bool implicit Assert.Equal(new[] { latestToken, latestToken }, observer.DeliveredTokens); } await accessor.Shutdown(); - Assert.Equal(shouldFault ? latestToken : requiredToken, Assert.Single(progress)); + Assert.Equal(requiredToken, Assert.Single(progress)); } [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("contract", -1, false)] - [InlineData("acquisition", -1, false)] - [InlineData("cache-miss", -1, false)] - [InlineData("empty", -1, false)] - [InlineData("contract", 2, false)] - [InlineData("empty", 2, false)] - [InlineData("contract", -1, true)] + [InlineData("contract", -1, false, false)] + [InlineData("acquisition", -1, false, false)] + [InlineData("cache-miss", -1, false, false)] + [InlineData("empty", -1, false, false)] + [InlineData("contract", 2, false, false)] + [InlineData("empty", 2, false, false)] + [InlineData("contract", -1, true, false)] + [InlineData("empty", -1, false, true)] public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( - string failure, int maximumSeconds, bool holdFailureNotification) + string failure, int maximumSeconds, bool holdFailureNotification, bool cancelBackoff) { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) @@ -4821,7 +4914,9 @@ public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( return holdFailureNotification ? failureNotification.Task : Task.CompletedTask; }); var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(TimeSpan.FromSeconds(1)); + backoff.Next(Arg.Any()).Returns(_ => cancelBackoff + ? throw new OperationCanceledException("Injected cancellation calculating consumer backoff.") + : TimeSpan.FromSeconds(1)); var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, options: new StreamPullingAgentOptions { @@ -4870,9 +4965,16 @@ public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( var expectedAttempts = maximumSeconds < 0 ? 6 : maximumSeconds; for (var i = 0; i < expectedAttempts - 1; i++) { - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + if (cancelBackoff) + { + await accessor.RunConsumerCursor(consumer); + } + else + { + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + } Assert.Equal(i + 1, consumer.DeliveryRecovery!.Attempts); - for (var j = 0; j < 10; j++) + for (var j = 0; !cancelBackoff && j < 10; j++) { Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); } @@ -4948,9 +5050,10 @@ public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( [TestProvider("None")] [TestArea("Streaming")] [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false)] - [InlineData(true)] - public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool failAttachment) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool failAttachment, bool cancelBackoff) { var registration = new TaskCompletionSource>(); var pubSub = Substitute.For(); @@ -4980,7 +5083,9 @@ public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool fa Task.FromResult>([new TestBatchContainer(streamId.StreamId, requiredToken)]), Task.FromResult>([])); var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(TimeSpan.FromSeconds(1)); + backoff.Next(Arg.Any()).Returns(_ => cancelBackoff + ? throw new OperationCanceledException("Injected cancellation calculating registration backoff.") + : TimeSpan.FromSeconds(1)); var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, options: new StreamPullingAgentOptions { MaxEventDeliveryTime = Timeout.InfiniteTimeSpan }, deliveryBackoff: backoff); @@ -5004,7 +5109,7 @@ public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool fa { Assert.Equal(attempt, stream.RegistrationRecovery!.Attempts); var before = acquisitions; - for (var read = 0; read < 10; read++) + for (var read = 0; !cancelBackoff && read < 10; read++) { Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); } @@ -5058,6 +5163,277 @@ public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool fa cache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public void Recovery_BackoffCancellationDoesNotResetOrEscapeTheBudget(bool previousDelay) + { + var clock = new FakeTimeProvider(); + var recovery = new StreamRecoveryState(clock.GetTimestamp()); + var backoff = Substitute.For(); + backoff.Next(Arg.Any()).Returns(call => previousDelay && call.Arg() == 0 + ? TimeSpan.FromSeconds(10) + : throw new OperationCanceledException("Injected backoff cancellation.")); + for (var attempt = 0; attempt < 6; attempt++) + { + Assert.False(recovery.IsExhausted(clock, TimeSpan.FromMinutes(1), 6)); + Assert.True(recovery.TryBeginAttempt(clock)); + recovery.FinishAttempt(clock, backoff, NullLogger.Instance); + Assert.Equal(attempt + 1, recovery.Attempts); + backoff.Received(1).Next(attempt); + clock.Advance(TimeSpan.FromSeconds(previousDelay && attempt == 0 ? 10 : 1)); + } + Assert.True(recovery.IsExhausted(clock, Timeout.InfiniteTimeSpan, 6)); + Assert.Equal(TimeSpan.FromSeconds(previousDelay ? 45 : 54), recovery.RemainingTime(clock, TimeSpan.FromMinutes(1))); + Assert.True(recovery.IsExhausted(clock, TimeSpan.FromSeconds(6), 100)); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void Recovery_BackoffDoesNotSuppressOtherProviderErrors() + { + var clock = new FakeTimeProvider(); + var recovery = new StreamRecoveryState(clock.GetTimestamp()); + var backoff = Substitute.For(); + var failure = new InvalidOperationException("Injected provider error."); + backoff.Next(Arg.Any()).Returns(_ => throw failure); + Assert.True(recovery.TryBeginAttempt(clock)); + Assert.Same(failure, Assert.Throws(() => recovery.FinishAttempt(clock, backoff, NullLogger.Instance))); + Assert.Equal(1, recovery.Attempts); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("none", false)] + [InlineData("none", true)] + [InlineData("drained", false)] + [InlineData("drained", true)] + [InlineData("removed", false)] + [InlineData("removed", true)] + [InlineData("expired", false)] + [InlineData("expired", true)] + public async Task Shutdown_DistinguishesNoSubscriptionsFromDrainedSubscriptions(string subscriptionState, bool readMore) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var timeProvider = new FakeTimeProvider(); + var readToken = new EventSequenceTokenV2(200); + var cache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + IList messages = [new TestBatchContainer(streamId.StreamId, readToken)]; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(_ => Task.FromResult(messages)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, readToken, timeProvider.GetUtcNow().UtcDateTime); + var observer = new RecordingConsumer(); + observer.ReleaseDelivery(); + StreamConsumerData? consumer = null; + if (subscriptionState != "none") + { + consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); + consumer.IsRegistered = true; + consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, null); + } + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + messages = []; + if (consumer is not null) + { + Assert.True(consumer.IsCaughtUp); + Assert.Equal(readToken, Assert.Single(observer.DeliveredTokens)); + } + else + { + Assert.Empty(observer.DeliveredTokens); + } + if (subscriptionState == "removed") + { + await agent.RunOrQueueTask(() => + { + agent.RemoveSubscriber_Impl(consumer!.SubscriptionId, streamId); + return Task.CompletedTask; + }); + } + else if (subscriptionState == "expired") + { + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + } + if (readMore) + { + var nextToken = new EventSequenceTokenV2(400); + if (!(await accessor.GetPubSubCache()).ContainsKey(streamId)) + { + await accessor.RegisterStream(streamId, nextToken, timeProvider.GetUtcNow().UtcDateTime); + } + messages = [new TestBatchContainer(streamId.StreamId, nextToken)]; + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Equal(subscriptionState == "none" ? 0 : subscriptionState == "drained" ? 2 : 1, observer.DeliveredTokens.Count); + } + await accessor.Shutdown(); + var expected = subscriptionState == "none" ? (long?)null : readMore && subscriptionState == "drained" ? 400 : 200; + Assert.Equal(expected, Assert.Single(cache.DeliveryProgressTokens)?.SequenceNumber); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("registration", false)] + [InlineData("handshake", false)] + [InlineData("known-replay", false)] + [InlineData("known-replay", true)] + [InlineData("unknown-replay", false)] + [InlineData("unknown-replay", true)] + [InlineData("drained", false)] + public async Task Shutdown_CompletedProgressStillHonorsCurrentReplayRequirements(string requirement, bool expireAgain) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var clock = new FakeTimeProvider(); + var cache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + IList messages = [new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => Task.FromResult(messages)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, clock); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), clock.GetUtcNow().UtcDateTime); + var original = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, cache, clock.GetUtcNow().UtcDateTime); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(original.IsCaughtUp); + Assert.Equal(200, original.LastProcessedToken!.SequenceNumber); + messages = []; + clock.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + + await accessor.RegisterStream(streamId, new EventSequenceTokenV2(400), clock.GetUtcNow().UtcDateTime); + var stream = (await accessor.GetPubSubCache()).Single().Value; + if (requirement == "drained") + { + var current = AddRegisteredConsumer(stream, streamId, cache, clock.GetUtcNow().UtcDateTime); + messages = [new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(400))]; + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(current.IsCaughtUp); + Assert.Equal(400, current.LastProcessedToken!.SequenceNumber); + } + else if (requirement == "registration") + { + stream.RegistrationTask = Task.CompletedTask; + } + else + { + var current = stream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), streamId, new RecordingConsumer(), null, clock.GetUtcNow().UtcDateTime); + current.IsRegistered = true; + current.LastProcessedToken = requirement == "unknown-replay" ? null : new EventSequenceTokenV2(100); + current.PendingHandshakes = requirement == "handshake" ? 1 : 0; + current.HasDeliveryProgressError = requirement != "handshake"; + current.DeliveryRecovery = new StreamRecoveryState(clock.GetTimestamp()) { Stopped = true }; + } + if (expireAgain) + { + clock.Advance(TimeSpan.FromDays(1)); + Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.Empty(await accessor.GetPubSubCache()); + } + await accessor.Shutdown(); + if (requirement is "known-replay" or "drained") + { + Assert.Equal(requirement == "drained" ? 400 : 100, Assert.Single(cache.DeliveryProgressTokens)!.SequenceNumber); + } + else + { + Assert.Empty(cache.DeliveryProgressTokens); + } + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_RemovalDoesNotPreserveAnUnacknowledgedReadFromAnotherStream() + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var cache = new PurgeablePooledQueueCache(); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(cache); + var receiver = Substitute.For(); + IList messages = + [ + new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1)), + new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200)), + ]; + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => Task.FromResult(messages)); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(idleId, new EventSequenceTokenV2(1), DateTime.UtcNow); + await accessor.RegisterStream(activeId, new EventSequenceTokenV2(200), DateTime.UtcNow); + var streams = await accessor.GetPubSubCache(); + var idle = AddRegisteredConsumer(streams[idleId], idleId, cache); + var active = AddRegisteredConsumer(streams[activeId], activeId, cache); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + Assert.True(idle.IsCaughtUp); + Assert.True(active.IsCaughtUp); + Assert.Equal(200, active.LastProcessedToken!.SequenceNumber); + + var pending = new RecordingConsumer(); + active.StreamConsumer = pending; + active.PendingHandshakes = 1; + messages = [new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(400))]; + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + active.PendingHandshakes = 0; + var delivery = accessor.RunConsumerCursor(active); + try + { + await pending.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(StreamConsumerDataState.Active, active.State); + Assert.Equal(400, active.UnconfirmedDeliveryToken!.SequenceNumber); + await agent.RunOrQueueTask(() => agent.RemoveSubscriber( + idle.SubscriptionId, idleId, TestContext.Current.CancellationToken)); + await agent.RunOrQueueTask(() => agent.RemoveSubscriber( + active.SubscriptionId, activeId, TestContext.Current.CancellationToken)); + pending.ReleaseDelivery(); + await delivery.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(200, active.LastProcessedToken.SequenceNumber); + Assert.Empty(await accessor.GetPubSubCache()); + await accessor.Shutdown(); + Assert.Equal(200, Assert.Single(cache.DeliveryProgressTokens)!.SequenceNumber); + } + finally + { + pending.ReleaseDelivery(); + } + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] From 4406c54efd9e78418fd4637d0332907b6fa4642a Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Fri, 11 Sep 2026 20:59:59 -0700 Subject: [PATCH 6/9] fix(streaming): retain recovery for incompatible acknowledgments - Compare outstanding tokens before accepting checkpoint progress. - Preserve unresolved delivery state and the existing recovery budget. - Cover filtered acknowledgments and bounded recovery across token types. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 18 ++- .../PersistentStreamPullingAgentTests.cs | 147 ++++++++++++++++++ 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 9d64bd0e2a1..4d6ebfd0668 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -1381,6 +1381,20 @@ private void RecordAcknowledgedDeliveryProgress( return; } + var acknowledgesUnconfirmedDelivery = false; + if (token is not null && consumer.UnconfirmedDeliveryToken is { } attempted) + { + if (!TryCompareQueueProgress(token, attempted, out var attemptedComparison)) + { + // Preserve the replay obligation and retry budget before accepting any new progress. + RecordDeliveryProgressError(consumer); + RecordDeliveryProgress(consumer, null); + return; + } + + acknowledgesUnconfirmedDelivery = attemptedComparison >= 0; + } + // Opening or draining an empty recovery cursor does not prove that its missing data was replayed. if (consumer.HasDeliveryProgressError && ReferenceEquals(consumer.DeliveryRecoveryCursor, deliveryCursor) @@ -1401,9 +1415,7 @@ private void RecordAcknowledgedDeliveryProgress( } RecordDeliveryProgress(consumer, token); - if (token is not null - && consumer.UnconfirmedDeliveryToken is { } attempted - && !IsBefore(token, attempted)) + if (acknowledgesUnconfirmedDelivery) { consumer.UnconfirmedDeliveryToken = null; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 6e87480150e..722ba3613fa 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -5786,6 +5786,153 @@ public async Task Shutdown_PreservesCheckpointForIncompatibleTokens(bool provide Assert.Empty(await accessor.GetPubSubCache()); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("incompatible", false, 1)] + [InlineData("incompatible", false, 2)] + [InlineData("incompatible", true, 1)] + [InlineData("incompatible", true, 2)] + [InlineData("older", false, 1)] + [InlineData("older", false, 2)] + [InlineData("older", true, 1)] + [InlineData("older", true, 2)] + [InlineData("equal", false, 1)] + [InlineData("equal", false, 2)] + [InlineData("equal", true, 1)] + [InlineData("equal", true, 2)] + [InlineData("newer", false, 1)] + [InlineData("newer", false, 2)] + [InlineData("newer", true, 1)] + [InlineData("newer", true, 2)] + public async Task Recovery_FilteredAcknowledgmentHonorsUnconfirmedPosition( + string tokenKind, bool recovering, int batchSize) + { + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var timeProvider = new FakeTimeProvider(); + var now = timeProvider.GetUtcNow().UtcDateTime; + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var incompatible = tokenKind == "incompatible"; + var attemptedToken = new EventSequenceTokenV2(100); + StreamSequenceToken requiredToken = incompatible + ? new IsolatedProviderToken(50) : new EventSequenceTokenV2(50); + StreamSequenceToken progressToken = tokenKind switch + { + "incompatible" => new IsolatedProviderToken(200), + "older" => new EventSequenceTokenV2(75), + "equal" => new EventSequenceTokenV2(100), + _ => new EventSequenceTokenV2(200) + }; + var batch = Substitute.For(); + batch.StreamId.Returns(streamId.StreamId); + batch.SequenceToken.Returns(progressToken); + batch.GetEvents().Returns([Tuple.Create("payload", progressToken)]); + var filter = Substitute.For(); + filter.ShouldDeliver(streamId.StreamId, "payload", null).Returns(false); + var queueCache = Substitute.For(); + queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) + .Returns(_ => QueueCacheCursorResult.FromCursor(CreateCursor())); + var adapterCache = Substitute.For(); + adapterCache.CreateQueueCache(queueId).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([batch])); + var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, + options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, streamFilter: filter); + var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await accessor.RegisterStream(streamId, requiredToken, now); + Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); + var observer = new RecordingConsumer(); + var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( + GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), + streamId, observer, null, now); + consumer.IsRegistered = true; + consumer.Cursor = CreateCursor(); + consumer.UnconfirmedDeliveryToken = attemptedToken; + consumer.LastToken = recovering + ? StreamHandshakeToken.CreateDeliveyToken(requiredToken) + : StreamHandshakeToken.CreateStartToken(requiredToken); + consumer.LastProcessedToken = recovering ? requiredToken : null; + consumer.PendingStartToken = recovering ? null : requiredToken; + var originalRecovery = recovering ? new StreamRecoveryState(timeProvider.GetTimestamp()) : null; + consumer.HasDeliveryProgressError = recovering; + consumer.DeliveryRecovery = originalRecovery; + consumer.DeliveryRecoveryToken = recovering ? consumer.LastToken : null; + consumer.DeliveryRecoveryCursor = recovering ? consumer.Cursor : null; + + await accessor.RunConsumerCursor(consumer); + + Assert.Empty(observer.DeliveredTokens); + Assert.Empty(observer.Errors); + Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); + Assert.Equal(incompatible, consumer.HasDeliveryProgressError); + Assert.Equal(!incompatible && tokenKind != "older", consumer.IsCaughtUp); + if (incompatible || tokenKind == "older") + { + Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); + } + else + { + Assert.Null(consumer.UnconfirmedDeliveryToken); + } + + if (incompatible) + { + Assert.Null(consumer.LastProcessedToken); + Assert.Same(requiredToken, consumer.DeliveryRecoveryToken?.Token); + Assert.Equal(recovering, consumer.DeliveryRecoveryToken is DeliveryToken); + var recovery = Assert.IsType(consumer.DeliveryRecovery); + Assert.Equal(recovering ? 1 : 0, recovery.Attempts); + if (recovering) + { + Assert.Same(originalRecovery, recovery); + } + + // Repeated incompatible acknowledgments must not replace the recovery episode or its budget. + for (var attempt = recovery.Attempts; attempt < 6; attempt++) + { + await accessor.RunConsumerCursor(consumer); + Assert.Same(recovery, consumer.DeliveryRecovery); + Assert.Equal(attempt + 1, recovery.Attempts); + Assert.Null(consumer.LastProcessedToken); + } + + Assert.True(recovery.Stopped); + Assert.Null(consumer.Cursor); + Assert.Single(observer.Errors); + await accessor.RunConsumerCursor(consumer); + Assert.Equal(6, recovery.Attempts); + } + else + { + Assert.Same(progressToken, consumer.LastProcessedToken); + Assert.Null(consumer.DeliveryRecovery); + } + + await accessor.Shutdown(); + if (incompatible) + { + queueCache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); + } + else + { + queueCache.Received(1).UpdateDeliveryProgress(progressToken, Arg.Any()); + } + + IQueueCacheCursor CreateCursor() + { + var cursor = Substitute.For(); + cursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); + cursor.GetCurrent(out _).Returns(batch); + return cursor; + } + } + private sealed class IsolatedProviderToken(long sequenceNumber) : EventSequenceTokenV2(sequenceNumber); [TestSuite("BVT")] From d57798f1ee63fe428be3f22b279be529e6111a63 Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Sat, 12 Sep 2026 10:48:56 -0700 Subject: [PATCH 7/9] fix(streaming): simplify checkpoint progress with legacy fallback Remove PR-added replay recovery, retry budgets, and retained gaps. Keep proven drained-subscription advancement bounded by queue reads. Use original checkpoint calculation when the optimization is uncertain. Restore original delivery retries and skip-and-continue failure handling. Preserve baseline coverage and focused checkpoint fallback regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 1069 ++--- .../QueueStreamDataStructures.cs | 82 +- .../StreamConsumerCollection.cs | 26 +- .../QueueAdapters/IQueueCache.cs | 8 +- .../PersistentStreamPullingAgentTests.cs | 3633 +++-------------- 5 files changed, 770 insertions(+), 4048 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 4d6ebfd0668..a0f87792f7d 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -22,8 +22,6 @@ namespace Orleans.Streams internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPersistentStreamPullingAgent, PersistentStreamPullingAgent.ITestAccessor { private const int ReadLoopRetryMax = 6; - // Like queue reads, automatic recovery stays finite even when the delivery timeout is unlimited. - private const int RecoveryAttemptMax = 6; private const int StreamInactivityCheckFrequency = 10; private readonly IBackoffProvider deliveryBackoffProvider; private readonly IBackoffProvider queueReaderBackoffProvider; @@ -31,7 +29,6 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private readonly IStreamPubSub pubSub; private readonly IStreamFilter streamFilter; private readonly Dictionary pubSubCache; - private readonly HashSet pendingConsumerRecoveries = new(); private readonly StreamPullingAgentOptions options; private readonly ILogger logger; private readonly IQueueAdapterCache queueAdapterCache; @@ -50,9 +47,7 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; private StreamSequenceToken? _lastReadToken; - private StreamSequenceToken? _retainedDeliveryProgress; - private StreamSequenceToken? _completedDeliveryProgress; - private bool _hasUnknownReplayPosition; + private bool _useLegacyDeliveryProgress; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -162,12 +157,8 @@ public Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; - // Progress belongs to this receiver lifetime, not the reusable pulling-agent instance. _lastReadToken = null; - _retainedDeliveryProgress = null; - _completedDeliveryProgress = null; - _hasUnknownReplayPosition = false; - pendingConsumerRecoveries.Clear(); + _useLegacyDeliveryProgress = false; lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -314,7 +305,6 @@ await Task.WhenAll(inFlightRegistrations) LogWarningUnregisterProducer(exc); } pubSubCache.Clear(); - pendingConsumerRecoveries.Clear(); } public Task AddSubscriber( @@ -360,32 +350,28 @@ private async Task AddSubscriber_Impl( } data.IsCaughtUp = false; - data.PendingStartToken ??= cacheToken; + if (data.PendingHandshakes != 0 || data.State == StreamConsumerDataState.Active) + { + _useLegacyDeliveryProgress = true; + } + data.PendingHandshakes++; - var attached = false; try { if (await DoHandshakeWithConsumer(data, cacheToken)) { - RecordDeliveryProgress(data, GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken)); - if (!data.HasDeliveryProgressError && data.LastProcessedToken is not null) - { - data.PendingStartToken = null; - } + data.LastProcessedToken = GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken); + data.PendingStartToken = null; data.IsRegistered = true; - attached = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); + if (data.State == StreamConsumerDataState.Inactive) + RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so } } finally { data.PendingHandshakes--; } - - if (attached && data.PendingHandshakes == 0 && data.State == StreamConsumerDataState.Inactive) - { - RunConsumerCursor(data, useCurrentCursor: true).Ignore(); - } } internal static StreamSequenceToken? GetInitialDeliveryProgress( @@ -402,7 +388,7 @@ private async Task DoHandshakeWithConsumer( StreamConsumerData consumerData, StreamSequenceToken? cacheToken) { - if (IsShutdown || consumerData.IsRemoved) return false; + if (IsShutdown) return false; StreamHandshakeToken? requestedHandshakeToken = null; StreamHandshakeToken? effectiveHandshakeToken = null; @@ -418,12 +404,11 @@ private async Task DoHandshakeWithConsumer( i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId), AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException - (exception, i) => exception is not ClientNotAvailableException && !IsShutdown && !consumerData.IsRemoved, + (exception, i) => exception is not ClientNotAvailableException && !IsShutdown, this.options.MaxEventDeliveryTime, deliveryBackoffProvider, cancellationToken: CancellationToken.None); - if (IsShutdown || consumerData.IsRemoved) return false; effectiveHandshakeToken = requestedHandshakeToken; if (effectiveHandshakeToken is null && ShouldApplyInitialSubscriptionStartPosition(consumerData)) { @@ -439,8 +424,6 @@ private async Task DoHandshakeWithConsumer( && startPositionToken.Equals(consumerData.LastToken); if (!preservesCurrentPosition) { - RecordReplayRequest(consumerData, startPositionToken); - RecordDeliveryProgress(consumerData, null); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetCacheCursorAtStartPosition( consumerData.StreamId, @@ -451,9 +434,6 @@ private async Task DoHandshakeWithConsumer( else if (effectiveHandshakeToken is StartToken or DeliveryToken && effectiveHandshakeToken.Token is { } requestedToken) { - RecordReplayRequest(consumerData, effectiveHandshakeToken); - RecordDeliveryProgress( - consumerData, GetInitialDeliveryProgress(effectiveHandshakeToken, consumerData.LastProcessedToken)); consumerData.SafeDisposeCursor(logger); if (effectiveHandshakeToken is DeliveryToken || effectiveHandshakeToken is StartToken @@ -461,7 +441,7 @@ private async Task DoHandshakeWithConsumer( { // Delivery tokens and implicit recovery tokens identify already processed events. consumerData.Cursor = GetAdvancedCursorOrFallback( - consumerData, + consumerData.StreamId, requestedToken, cacheToken, out consumerData.PendingBatch); @@ -469,11 +449,6 @@ private async Task DoHandshakeWithConsumer( else { var result = queueCache.TryGetCacheCursor(consumerData.StreamId, requestedToken); - if (result.Kind == QueueCacheCursorResultKind.CacheMiss) - { - RecordDeliveryProgressError(consumerData, effectiveHandshakeToken); - } - consumerData.Cursor = result.Kind switch { QueueCacheCursorResultKind.Success => result.Cursor!, @@ -482,10 +457,6 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null QueueCacheCursorResultKind.CacheMiss => throw result.CacheMiss!.Value.ToException(), _ => throw new InvalidOperationException($"Unexpected cursor result: {result.Kind}."), }; - if (result.Kind == QueueCacheCursorResultKind.Success) - { - TrackDeliveryRecoveryCursor(consumerData, consumerData.Cursor, effectiveHandshakeToken); - } } } else if (effectiveHandshakeToken is not null) @@ -502,8 +473,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception exception) { - if (consumerData.IsRemoved) return false; - RecordDeliveryProgressError(consumerData, effectiveHandshakeToken); + _useLegacyDeliveryProgress = true; exceptionOccured = exception; } if (exceptionOccured != null) @@ -545,7 +515,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception) { - RecordDeliveryProgressError(consumerData); + _useLegacyDeliveryProgress = true; consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, null); } } @@ -578,7 +548,16 @@ private static QueueCacheCursorMoveResult AdvanceCursorPastToken( throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - var batch = GetCurrentBatchOrThrow(cursor); + var batch = cursor.GetCurrent(out var exception); + if (exception is not null) + { + throw exception; + } + + if (batch is null) + { + throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); + } var comparison = EventSequenceTokenCompatibility.Compare(batch.SequenceToken, token); if (comparison >= 0) @@ -613,30 +592,6 @@ private IQueueCacheCursor GetCacheCursorAtStartPosition( private IQueueCacheCursor GetRecoveryCursor(StreamConsumerData consumerData) { - if (consumerData.HasDeliveryProgressError) - { - if (consumerData.DeliveryRecoveryToken is { Token: { } requiredToken } recoveryToken) - { - var acquisition = queueCache!.TryGetCacheCursor(consumerData.StreamId, requiredToken); - if (acquisition.Kind == QueueCacheCursorResultKind.Success) - { - var recoveryCursor = acquisition.Cursor!; - if (recoveryToken is StartToken - || TryAdvanceRecoveryCursor(recoveryCursor, requiredToken, out consumerData.PendingBatch)) - { - TrackDeliveryRecoveryCursor(consumerData, recoveryCursor, recoveryToken); - return recoveryCursor; - } - } - else if (acquisition.Kind != QueueCacheCursorResultKind.CacheMiss) - { - throw new InvalidOperationException($"Unexpected cursor result: {acquisition.Kind}."); - } - } - - return GetCacheCursorOrThrow(consumerData.StreamId, null); - } - if (consumerData.LastProcessedToken is { } lastProcessedToken) { var result = queueCache!.TryGetCacheCursor(consumerData.StreamId, lastProcessedToken); @@ -706,17 +661,15 @@ private IQueueCacheCursor GetCacheCursorOrThrow(StreamId streamId, StreamSequenc } private IQueueCacheCursor GetAdvancedCursorOrFallback( - StreamConsumerData consumerData, + StreamId streamId, StreamSequenceToken token, StreamSequenceToken? fallbackToken, out IBatchContainer? pendingBatch) { pendingBatch = null; - var streamId = consumerData.StreamId; var acquisitionResult = queueCache!.TryGetCacheCursor(streamId, token); if (acquisitionResult.Kind == QueueCacheCursorResultKind.CacheMiss) { - RecordDeliveryProgressError(consumerData, StreamHandshakeToken.CreateDeliveyToken(token)); return fallbackToken is not null ? GetCacheCursorOrThrow(streamId, fallbackToken) : throw acquisitionResult.CacheMiss!.Value.ToException(); @@ -734,7 +687,6 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( var moveResult = AdvanceCursorPastToken(cursor, token, out pendingBatch); if (moveResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { - RecordDeliveryProgressError(consumerData, StreamHandshakeToken.CreateDeliveyToken(token)); return fallbackToken is not null ? GetCacheCursorOrThrow(streamId, fallbackToken) : throw moveResult.CacheMiss!.Value.ToException(); @@ -746,7 +698,6 @@ private IQueueCacheCursor GetAdvancedCursorOrFallback( } retainCursor = true; - TrackDeliveryRecoveryCursor(consumerData, cursor, StreamHandshakeToken.CreateDeliveyToken(token)); return cursor; } finally @@ -816,10 +767,13 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea if (!pubSubCache.TryGetValue(streamId, out var streamData)) return; - if (streamData.TryGetConsumer(subscriptionId, out var consumer)) + if (!_useLegacyDeliveryProgress && (!streamData.StreamRegistered || streamData.RegistrationTask is not null + || streamData.TryGetConsumer(subscriptionId, out var consumer) + && (!HasCaughtUpDeliveryProgress(consumer, streamData.LastReadToken) + || _lastReadToken is null + || !TryCompareQueueProgress(consumer.LastProcessedToken!, _lastReadToken, out _)))) { - RetainCompletedDeliveryProgress(consumer.LastProcessedToken); - pendingConsumerRecoveries.Remove(consumer); + _useLegacyDeliveryProgress = true; } // remove consumer @@ -831,11 +785,8 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea LogDebugRemovedConsumer(subscriptionId, streamId); } - if (streamData.Count == 0 && streamData.RegistrationTask is null) - { + if (streamData.Count == 0) pubSubCache.Remove(streamId); - streamData.DisposeAll(logger); - } } private Task RunQueuePump(QueueId queueId, CancellationToken cancellationToken) @@ -867,8 +818,6 @@ private async Task PumpQueue(QueueId queueId, CancellationToken cancellationToke if (IsShutdown || cancellationToken.IsCancellationRequested) return; // timer was already removed, last tick - RetryPendingWork(cancellationToken); - // loop through the queue until it is empty. while (!IsShutdown && !cancellationToken.IsCancellationRequested) // shutdown sets IsShutdown and cancels the timer token. { @@ -903,6 +852,7 @@ private async Task PumpQueue(QueueId queueId, CancellationToken cancellationToke bool ReadLoopRetryExceptionFilter(Exception e, int retryCounter) { + _useLegacyDeliveryProgress = true; LogErrorRetrying(retryCounter, new(queueId), e); return !cancellationToken.IsCancellationRequested && !IsShutdown; } @@ -924,10 +874,8 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? return false; } - RetryPendingWork(cancellationToken); - // Pause all queue reads so a cold stream's first batch stays pinned until registration completes. - if (pubSubCache.Values.Any(static stream => stream.RegistrationTask is not null)) + if (pubSubCache.Values.Any(static stream => stream.RegistrationTask is { IsCompleted: false })) { return false; } @@ -1015,38 +963,38 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // calls do not stall message delivery for other streams on the same queue. RegisterStream(streamId, startToken, now, CancellationToken.None); } + + if (pubSubCache.TryGetValue(streamId, out streamData)) + { + streamData.LastReadToken = group.Last().SequenceToken; + } } accountedFor = true; } finally { - // The receiver has consumed this read even if cache insertion or registration fails. if (!accountedFor) { - _hasUnknownReplayPosition = true; - LogErrorQueueRecoveryRequired(new(QueueId), "A fetched read could not be fully admitted to the cache and stream registry."); + _useLegacyDeliveryProgress = true; } } - if (IsShutdown || cancellationToken.IsCancellationRequested) - { - return false; - } + RecordReadProgress(multiBatch); + return !IsShutdown && !cancellationToken.IsCancellationRequested; + } - // Only publish the read boundary after every stream has been accounted for. - foreach (var batch in multiBatch) - { - if (batch is null) - { - continue; - } + private void RecordReadProgress(IList batches) + { + if (queueCache is null || _useLegacyDeliveryProgress) return; - if (batch.SequenceToken is not { } token) + foreach (var batch in batches) + { + if (batch?.SequenceToken is not { } token) { - _hasUnknownReplayPosition = true; - LogErrorQueueRecoveryRequired(new(myQueueId), $"A fetched batch for stream {batch.StreamId} has no sequence token."); - continue; + _useLegacyDeliveryProgress = true; + LogWarningUnknownReadProgress(new(QueueId)); + return; } if (_lastReadToken is null) @@ -1055,16 +1003,19 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? } else if (!TryCompareQueueProgress(token, _lastReadToken, out var comparison)) { - _hasUnknownReplayPosition = true; - LogErrorQueueRecoveryRequired(new(myQueueId), "Fetched batches have incompatible sequence tokens."); + return; } - else if (comparison > 0) + else if (comparison < 0) + { + _useLegacyDeliveryProgress = true; + LogWarningUnknownReadProgress(new(QueueId)); + return; + } + else { _lastReadToken = token; } } - - return true; } private void CleanupPubSubCache(DateTime now) @@ -1072,10 +1023,7 @@ private void CleanupPubSubCache(DateTime now) List? inactiveStreams = null; foreach (var tuple in pubSubCache) { - if (tuple.Value.IsInactive(now, options.StreamInactivityPeriod) - && tuple.Value.RegistrationTask is null - && !tuple.Value.AllConsumers().Any(consumer => consumer.PendingHandshakes != 0 - || pendingConsumerRecoveries.Contains(consumer))) + if (tuple.Value.IsInactive(now, options.StreamInactivityPeriod)) { inactiveStreams ??= new(); inactiveStreams.Add(tuple.Key); @@ -1087,16 +1035,18 @@ private void CleanupPubSubCache(DateTime now) return; } - if (TryGetDeliveryProgress(out var progress)) - { - RetainCompletedDeliveryProgress(progress); - } - foreach (var streamId in inactiveStreams) { if (pubSubCache.Remove(streamId, out var streamData)) { - RetainStreamDeliveryProgress(streamData); + if (!_useLegacyDeliveryProgress && (!streamData.StreamRegistered || streamData.RegistrationTask is not null + || streamData.AllConsumers().Any(consumer => + !HasCaughtUpDeliveryProgress(consumer, streamData.LastReadToken) + || _lastReadToken is null + || !TryCompareQueueProgress(consumer.LastProcessedToken!, _lastReadToken, out _)))) + { + _useLegacyDeliveryProgress = true; + } streamData.DisposeAll(logger); StreamingEvents.EmitStreamInactive(streamProviderName, streamId.StreamId, options.StreamInactivityPeriod, Silo); } @@ -1119,415 +1069,131 @@ private void NotifyDeliveryProgress() private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) { - earliest = _retainedDeliveryProgress; - var hasSubscriptions = false; - if (_hasUnknownReplayPosition) - { - return false; - } - - foreach (var streamConsumers in pubSubCache.Values) - { - if (!TryGetStreamDeliveryProgress(streamConsumers, out var current) - || current is not null && !IncludeProgress(current, ref earliest)) - { - return false; - } - - hasSubscriptions |= streamConsumers.Count > 0; - } - - // Removed subscriptions preserve proven progress, not evidence that subsequent reads were delivered. - if (!hasSubscriptions && _completedDeliveryProgress is { } completed - && !IncludeProgress(completed, ref earliest)) - { - return false; - } - - if ((hasSubscriptions || earliest is not null) && _lastReadToken is { } read) - { - return IncludeProgress(read, ref earliest); - } - - return true; - } - - private void RetainStreamDeliveryProgress(StreamConsumerCollection streamData) - { - foreach (var consumer in streamData.AllConsumers()) - { - RetainCompletedDeliveryProgress(consumer.LastProcessedToken); - } + var canAdvancePastDrainedSubscriptions = !_useLegacyDeliveryProgress + && _lastReadToken is not null + && pubSubCache.Values.All(static stream => stream.StreamRegistered && stream.RegistrationTask is null + && stream.AllConsumers().All(static consumer => consumer.PendingHandshakes == 0)); - // Reclaim inactive subscription records without forgetting their unresolved handoff constraint. - if (!TryGetStreamDeliveryProgress(streamData, out var token) - || token is not null && !IncludeProgress(token, ref _retainedDeliveryProgress)) + if (canAdvancePastDrainedSubscriptions + && TryGetSubscriptionDeliveryProgress(_lastReadToken, out earliest) + && !_useLegacyDeliveryProgress) { - _hasUnknownReplayPosition = true; - } - } - - private void RetainCompletedDeliveryProgress(StreamSequenceToken? token) - { - if (token is null - || _lastReadToken is { } read && !TryCompareQueueProgress(token, read, out _)) - { - return; + return true; } - if (_completedDeliveryProgress is null - || TryCompareQueueProgress(token, _completedDeliveryProgress, out var comparison) && comparison > 0) - { - _completedDeliveryProgress = token; - } + // Opting out restores the original calculation; it does not freeze normal delivery progress. + return TryGetSubscriptionDeliveryProgress(null, out earliest); } - private bool TryGetStreamDeliveryProgress(StreamConsumerCollection streamConsumers, out StreamSequenceToken? earliest) + private bool TryGetSubscriptionDeliveryProgress(StreamSequenceToken? readBoundary, out StreamSequenceToken? earliest) { earliest = null; - if (!streamConsumers.StreamRegistered - || streamConsumers.RegistrationTask is not null) - { - return false; - } + var hasSubscriptions = false; - foreach (var consumer in streamConsumers.AllConsumers()) + foreach (var streamConsumers in pubSubCache.Values) { - if (!consumer.IsRegistered || consumer.PendingHandshakes != 0) + if (streamConsumers.RegistrationTask is { IsCompleted: false }) { return false; } - // A drained subscription's last matching event need not pin other streams. - if (consumer.IsCaughtUp && !consumer.HasDeliveryProgressError) - { - continue; - } - - var current = consumer.LastProcessedToken; - if (current is null) + foreach (var consumer in streamConsumers.AllConsumers()) { - return false; - } - - if (earliest is null) - { - earliest = current; - continue; - } - - try - { - if (IsBefore(current, earliest)) + if (!consumer.IsRegistered) { - earliest = current; + return false; } - } - catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") - { - LogWarningIncompatibleDeliveryProgress( - new(QueueId), consumer.SubscriptionId, consumer.StreamId, current, earliest, exception); - earliest = null; - return false; - } - } - - return true; - } - - private bool IncludeProgress(StreamSequenceToken token, ref StreamSequenceToken? earliest) - { - if (earliest is null) - { - earliest = token; - } - else if (!TryCompareQueueProgress(token, earliest, out var comparison)) - { - return false; - } - else if (comparison < 0) - { - earliest = token; - } - - return true; - } - - private bool TryCompareQueueProgress(StreamSequenceToken token, StreamSequenceToken other, out int comparison) - { - try - { - comparison = EventSequenceTokenCompatibility.Compare(token, other); - return true; - } - catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") - { - LogWarningIncompatibleQueueProgress(new(QueueId), token, other, exception); - comparison = default; - return false; - } - } - - private void RecordDeliveryProgress(StreamConsumerData consumer, StreamSequenceToken? token) - { - // A reset cursor's later progress cannot prove that it did not skip earlier data. - // An earlier or unknown replay request must still lower the retained checkpoint floor. - if (!consumer.HasDeliveryProgressError || token is null) - { - consumer.LastProcessedToken = token; - if (token is not null) - { - consumer.PendingStartToken = null; - } - } - else if (consumer.LastProcessedToken is { } current) - { - if (!TryCompareQueueProgress(token, current, out var comparison)) - { - consumer.LastProcessedToken = null; - } - else if (comparison < 0) - { - consumer.LastProcessedToken = token; - } - } - } - - private void RecordReplayRequest(StreamConsumerData consumer, StreamHandshakeToken request) - { - if (consumer.HasDeliveryProgressError) - { - IncludeDeliveryRecoveryRequest(consumer, NormalizeRecoveryToken(consumer, request)); - consumer.DeliveryRecoveryCursor = null; - consumer.HasObservedRecoveryStart = false; - } - - if (request is StartToken) - { - consumer.PendingStartToken = request.Token; - } - else if (request is StartPositionToken) - { - consumer.PendingStartToken = null; - } - } - - private static StreamHandshakeToken? NormalizeRecoveryToken(StreamConsumerData consumer, StreamHandshakeToken? token) - => token is StartToken && SubscriptionMarker.IsImplicitSubscription(consumer.SubscriptionId.Guid) - ? StreamHandshakeToken.CreateDeliveyToken(token.Token!) - : token is StartToken or DeliveryToken ? token : null; - - private void IncludeDeliveryRecoveryRequest(StreamConsumerData consumer, StreamHandshakeToken? request) - { - if (consumer.DeliveryRecoveryToken?.Token is not { } current - || request?.Token is not { } requested - || !TryCompareQueueProgress(requested, current, out var comparison)) - { - consumer.DeliveryRecoveryToken = null; - return; - } - - if (comparison < 0 || comparison == 0 && request is StartToken) - { - consumer.DeliveryRecoveryToken = request; - } - } - - private void TrackDeliveryRecoveryCursor( - StreamConsumerData consumer, - IQueueCacheCursor cursor, - StreamHandshakeToken? request) - { - if (consumer.HasDeliveryProgressError && consumer.DeliveryRecovery is not { Stopped: true } - && consumer.DeliveryRecoveryToken is { Token: { } required } recovery - && request?.Token is { } requested - && TryCompareQueueProgress(requested, required, out var comparison) - && (comparison < 0 || comparison == 0 && (request is StartToken || recovery is DeliveryToken))) - { - consumer.DeliveryRecoveryCursor = cursor; - consumer.HasObservedRecoveryStart = false; - } - } - - private void ObserveDeliveryRecoveryStart(StreamConsumerData consumer, IBatchContainer batch) - { - if (!consumer.HasDeliveryProgressError) - { - if (consumer.LastProcessedToken is null && batch.SequenceToken is { } first - && (consumer.PendingStartToken is null - || TryCompareQueueProgress(first, consumer.PendingStartToken, out var startComparison) && startComparison < 0)) - { - consumer.PendingStartToken = first; - } - } - else if (ReferenceEquals(consumer.DeliveryRecoveryCursor, consumer.Cursor) - && consumer.DeliveryRecoveryToken is StartToken { Token: { } required } - && batch.SequenceToken is { } current - && TryCompareQueueProgress(current, required, out var anchorComparison) && anchorComparison == 0) - { - // Pooled caches can accept a purged token as an exclusive boundary. Inclusive recovery must see it. - consumer.HasObservedRecoveryStart = true; - } - } - private void RecordAcknowledgedDeliveryProgress( - StreamConsumerData consumer, - StreamSequenceToken? token, - IQueueCacheCursor deliveryCursor, - long cursorVersion) - { - if (consumer.CursorVersion != cursorVersion || !ReferenceEquals(consumer.Cursor, deliveryCursor)) - { - return; - } - - var acknowledgesUnconfirmedDelivery = false; - if (token is not null && consumer.UnconfirmedDeliveryToken is { } attempted) - { - if (!TryCompareQueueProgress(token, attempted, out var attemptedComparison)) - { - // Preserve the replay obligation and retry budget before accepting any new progress. - RecordDeliveryProgressError(consumer); - RecordDeliveryProgress(consumer, null); - return; - } - - acknowledgesUnconfirmedDelivery = attemptedComparison >= 0; - } + var current = consumer.LastProcessedToken; + if (current is null) + { + return false; + } - // Opening or draining an empty recovery cursor does not prove that its missing data was replayed. - if (consumer.HasDeliveryProgressError - && ReferenceEquals(consumer.DeliveryRecoveryCursor, deliveryCursor) - && consumer.DeliveryRecoveryToken is { Token: { } required } recovery - && token is not null - && TryCompareQueueProgress(token, required, out var comparison) - && (recovery is DeliveryToken && comparison > 0 - || recovery is StartToken && comparison >= 0 && consumer.HasObservedRecoveryStart)) - { - consumer.HasDeliveryProgressError = false; - consumer.DeliveryRecoveryToken = null; - consumer.DeliveryRecoveryCursor = null; - consumer.HasObservedRecoveryStart = false; - consumer.DeliveryRecovery = null; - consumer.DeliveryRecoveryFailureReported = false; - consumer.DeliveryRecoveryFailureToken = null; - pendingConsumerRecoveries.Remove(consumer); - } + hasSubscriptions = true; + if (readBoundary is not null) + { + if (!TryCompareQueueProgress(current, readBoundary, out _)) + { + return false; + } - RecordDeliveryProgress(consumer, token); - if (acknowledgesUnconfirmedDelivery) - { - consumer.UnconfirmedDeliveryToken = null; - } - } + if (HasCaughtUpDeliveryProgress(consumer, streamConsumers.LastReadToken)) + { + continue; + } + } - private void RecordDeliveryProgressError( - StreamConsumerData consumer, - StreamHandshakeToken? replayRequest = null, - StreamSequenceToken? failedToken = null, - long? startedAt = null) - { - if (consumer.IsRemoved) return; - var request = NormalizeRecoveryToken(consumer, replayRequest); - if (!consumer.HasDeliveryProgressError) - { - var first = consumer.PendingStartToken; - if (failedToken is not null) - { - if (first is null) + if (earliest is null) { - first = failedToken; + earliest = current; + continue; } - else if (!TryCompareQueueProgress(failedToken, first, out var comparison)) + + try { - first = null; + if (IsBefore(current, earliest)) + { + earliest = current; + } } - else if (comparison < 0) + catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") { - first = failedToken; + LogWarningIncompatibleDeliveryProgress( + new(QueueId), consumer.SubscriptionId, consumer.StreamId, current, earliest, exception); + earliest = null; + return false; } } - - consumer.DeliveryRecoveryToken = replayRequest is not null - ? request - : consumer.LastProcessedToken is { } processed - ? StreamHandshakeToken.CreateDeliveyToken(processed) - : first is { } start - ? StreamHandshakeToken.CreateStartToken(start) - : null; - } - else if (replayRequest is not null) - { - IncludeDeliveryRecoveryRequest(consumer, request); } - if (!consumer.HasDeliveryProgressError) + if (readBoundary is not null && hasSubscriptions) { - consumer.DeliveryRecovery = new(startedAt ?? _timeProvider.GetTimestamp()); - LogWarningDeliveryRecoveryRequired(new(QueueId), consumer.SubscriptionId, consumer.StreamId); - } - - consumer.HasDeliveryProgressError = true; - consumer.IsCaughtUp = false; - consumer.DeliveryRecoveryCursor = null; - consumer.HasObservedRecoveryStart = false; - if (consumer.DeliveryRecovery is not { Stopped: true }) - { - pendingConsumerRecoveries.Add(consumer); - } - } - - private void RetryPendingConsumers() - { - foreach (var consumer in pendingConsumerRecoveries.ToArray()) - { - if (!pubSubCache.TryGetValue(consumer.StreamId, out var stream) - || !stream.TryGetConsumer(consumer.SubscriptionId, out var current) - || !ReferenceEquals(current, consumer) - || !consumer.HasDeliveryProgressError - || consumer.DeliveryRecovery is { Stopped: true }) + if (earliest is null) { - pendingConsumerRecoveries.Remove(consumer); + earliest = readBoundary; } - else if (consumer.IsRegistered && consumer.PendingHandshakes == 0 - && consumer.State == StreamConsumerDataState.Inactive) + else { - RunConsumerCursor(consumer).Ignore(); + if (!TryCompareQueueProgress(readBoundary, earliest, out var comparison)) + { + return false; + } + + if (comparison < 0) + { + earliest = readBoundary; + } } } - } - private void RetryPendingWork(CancellationToken cancellationToken) - { - RetryPendingConsumers(); - foreach (var pending in pubSubCache - .Where(static pair => pair.Value.RegistrationTask is { IsCompleted: true }) - .ToArray()) - { - RegisterStream(pending.Key, pending.Value.RegistrationStartToken, - _timeProvider.GetUtcNow().UtcDateTime, cancellationToken); - } + return true; } - private async Task StopConsumerRecovery(StreamConsumerData consumer, CancellationToken cancellationToken) + private bool HasCaughtUpDeliveryProgress(StreamConsumerData consumer, StreamSequenceToken? streamReadBoundary) + => consumer.IsCaughtUp + && consumer.State == StreamConsumerDataState.Inactive + && consumer.PendingHandshakes == 0 + && consumer.PendingBatch is null + && consumer.Cursor is not null + && consumer.LastProcessedToken is { } progress + && streamReadBoundary is not null + && TryCompareQueueProgress(progress, streamReadBoundary, out var comparison) + && comparison >= 0; + + private bool TryCompareQueueProgress(StreamSequenceToken token, StreamSequenceToken other, out int comparison) { - if (consumer.DeliveryRecovery is not { Stopped: false } recovery) + try { - return; + comparison = EventSequenceTokenCompatibility.Compare(token, other); + return true; } - - recovery.Stopped = true; - pendingConsumerRecoveries.Remove(consumer); - consumer.SafeDisposeCursor(logger); - consumer.PendingStartToken = null; - consumer.PendingContinuationToken = null; - consumer.UnconfirmedDeliveryToken = null; - LogWarningDeliveryRecoveryStopped(new(QueueId), consumer.SubscriptionId, consumer.StreamId, recovery.Attempts); - if (!consumer.DeliveryRecoveryFailureReported) + catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") { - // A replay boundary is not necessarily a failed event (failure handlers can dead-letter this token). - await ErrorProtocol(consumer, new StreamEventDeliveryFailureException(consumer.StreamId), - true, null, null, cancellationToken: cancellationToken); + _useLegacyDeliveryProgress = true; + LogWarningIncompatibleQueueProgress(new(QueueId), token, other, exception); + comparison = default; + return false; } } @@ -1536,7 +1202,7 @@ private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken ot private void RegisterStream( QualifiedStreamId streamId, - StreamSequenceToken? firstToken, + StreamSequenceToken firstToken, DateTime now, CancellationToken cancellationToken) { @@ -1545,39 +1211,20 @@ private void RegisterStream( return; } - if (!pubSubCache.TryGetValue(streamId, out var streamData)) - { - streamData = new StreamConsumerCollection(now) { RegistrationStartToken = firstToken }; - pubSubCache.Add(streamId, streamData); - } - else if (streamData.RegistrationTask is null || !streamData.RegistrationTask.IsCompleted) - { - return; - } + var streamData = new StreamConsumerCollection(now); - var recovery = streamData.RegistrationRecovery ??= new(_timeProvider.GetTimestamp()); - if (recovery.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) - { - recovery.Stopped = true; - RetainStreamDeliveryProgress(streamData); - foreach (var consumer in streamData.AllConsumers()) - { - pendingConsumerRecoveries.Remove(consumer); - } - pubSubCache.Remove(streamId); - streamData.DisposeAll(logger); - LogWarningStreamRegistrationRecoveryStopped(new(QueueId), streamId, recovery.Attempts); - return; - } - - if (!recovery.TryBeginAttempt(_timeProvider)) return; + // Create a fake cursor to point into a cache. + // That way we will not purge the event from the cache, until we talk to pub sub. + // This will help ensure the "casual consistency" between pre-existing subscripton (of a potentially new already subscribed consumer) + // and later production. + var pinCursor = queueCache is null ? null : GetCacheCursorOrThrow(streamId, firstToken); streamData.RegistrationTask = RegisterStreamAsync(); + pubSubCache.Add(streamId, streamData); async Task RegisterStreamAsync() { await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); - var completed = false; try { if (IsShutdown || cancellationToken.IsCancellationRequested) @@ -1585,12 +1232,6 @@ async Task RegisterStreamAsync() return; } - // Retain the original boundary and pin across retries, including failures before producer registration. - if (queueCache is not null) - { - streamData.RegistrationCursor ??= GetCacheCursorOrThrow(streamId, streamData.RegistrationStartToken); - } - var subscribers = await RegisterAsStreamProducer(streamId, cancellationToken); if (IsShutdown || cancellationToken.IsCancellationRequested) @@ -1605,72 +1246,78 @@ async Task RegisterStreamAsync() LogDebugGotBackSubscribers(subscribers.Count, streamId); - completed = true; + // Await all initial subscriber handshakes before releasing the pin cursor so + // that the first batch cannot be purged from the cache before each subscriber + // has established its own cursor. if (subscribers.Count > 0) { - var addSubscriptionTasks = new List>(subscribers.Count); + var addSubscriptionTasks = new List(subscribers.Count); foreach (PubSubSubscriptionState item in subscribers) { addSubscriptionTasks.Add(SubscribeWithIsolation(item)); } - completed = (await Task.WhenAll(addSubscriptionTasks)).All(static attached => attached); + await Task.WhenAll(addSubscriptionTasks); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - completed = false; - // The retained stream entry remains pending for the next uncanceled pump. + RemoveCanceledRegistration(); } catch (Exception exception) { - completed = false; - LogWarningFailedToRegisterStream(streamId, exception); - StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); + FailRegistration(exception); } finally { - if (completed) - { - streamData.RegistrationTask = null; - streamData.RegistrationStartToken = null; - streamData.RegistrationRecovery = null; - streamData.DisposeRegistrationCursor(logger); - } - else - { - streamData.RegistrationTask = Task.CompletedTask; - recovery.FinishAttempt(_timeProvider, deliveryBackoffProvider, logger); - if (IsShutdown - || !pubSubCache.TryGetValue(streamId, out var current) || !ReferenceEquals(current, streamData)) - { - streamData.DisposeRegistrationCursor(logger); - } - } + streamData.RegistrationTask = null; + pinCursor?.Dispose(); } } - async Task SubscribeWithIsolation(PubSubSubscriptionState item) + void RemoveCanceledRegistration() { - if (IsShutdown) + _useLegacyDeliveryProgress = true; + if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) + && ReferenceEquals(cachedStreamData, streamData)) { - return false; + pubSubCache.Remove(streamId); } - if (streamData.TryGetConsumer(item.SubscriptionId, out var existing) && existing.IsRegistered) + streamData.DisposeAll(logger); + } + + void FailRegistration(Exception exception) + { + _useLegacyDeliveryProgress = true; + LogWarningFailedToRegisterStream(streamId, exception); + StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); + + // Only reached when producer registration itself fails. + if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) + && ReferenceEquals(cachedStreamData, streamData)) { - return existing.PendingHandshakes == 0; + pubSubCache.Remove(streamId); + } + + streamData.DisposeAll(logger); + } + + async Task SubscribeWithIsolation(PubSubSubscriptionState item) + { + if (IsShutdown) + { + return; } try { - await AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, item.FilterData, streamData.RegistrationStartToken); - return !streamData.TryGetConsumer(item.SubscriptionId, out var consumer) || consumer.IsRegistered; + await AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, item.FilterData, firstToken); } catch (Exception exception) { + _useLegacyDeliveryProgress = true; LogWarningFailedToAddSubscription(item.Stream, exception); - return false; } } } @@ -1685,21 +1332,9 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq } // Some consumer might not be fully registered yet - consumerData.IsCaughtUp = false; if (consumerData.IsRegistered) { - if (consumerData.DeliveryRecovery is { Stopped: true } && consumerData.Cursor is null) - { - // A nonfaulting subscription still receives new traffic, but does not restart abandoned replay. - consumerData.PendingContinuationToken ??= startToken; - } - - // Preserve recovery positions so the next move can detect missing replay data. - if (consumerData.Cursor is { } cursor && !ReferenceEquals(cursor, consumerData.DeliveryRecoveryCursor)) - { - cursor.Refresh(startToken); - } - + consumerData.Cursor?.Refresh(startToken); if (consumerData.State == StreamConsumerDataState.Inactive) { // wake up inactive consumers @@ -1717,64 +1352,20 @@ private void StartInactiveCursors(StreamConsumerCollection streamData, StreamSeq } } - private async Task RunConsumerCursor(StreamConsumerData consumerData, CancellationToken cancellationToken = default, bool useCurrentCursor = false) + private async Task RunConsumerCursor(StreamConsumerData consumerData, CancellationToken cancellationToken = default) { TagList? tags = null; - StreamRecoveryState? recoveryAttempt = null; - var ownsCursor = false; try { // double check in case of interleaving - if (IsShutdown || cancellationToken.IsCancellationRequested - || consumerData.IsRemoved || consumerData.State == StreamConsumerDataState.Active - || consumerData.PendingHandshakes != 0) return; - - if (consumerData.HasDeliveryProgressError && consumerData.PendingHandshakes == 0 - && consumerData.DeliveryRecovery is { Stopped: false } recovery) - { - if (consumerData.DeliveryRecoveryToken is null && !useCurrentCursor - || recovery.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) - { - consumerData.State = StreamConsumerDataState.Active; - ownsCursor = true; - await StopConsumerRecovery(consumerData, cancellationToken); - return; - } - - if (!recovery.TryBeginAttempt(_timeProvider)) return; - recoveryAttempt = recovery; - } - - var needsRecovery = consumerData.HasDeliveryProgressError && consumerData.DeliveryRecoveryToken is not null - && consumerData.DeliveryRecovery is not { Stopped: true } - && !useCurrentCursor - && consumerData.PendingHandshakes == 0 - && (consumerData.Cursor is null || !ReferenceEquals(consumerData.Cursor, consumerData.DeliveryRecoveryCursor)); - var continueNewTraffic = consumerData.DeliveryRecovery is { Stopped: true } - && consumerData.Cursor is null && consumerData.PendingContinuationToken is not null; - if (consumerData.Cursor is null && !needsRecovery && !continueNewTraffic) return; + if (consumerData.State == StreamConsumerDataState.Active || + consumerData.Cursor is null) return; consumerData.State = StreamConsumerDataState.Active; - ownsCursor = true; - var replacedCursor = needsRecovery || continueNewTraffic; - if (needsRecovery) - { - consumerData.SafeDisposeCursor(logger); - consumerData.Cursor = GetRecoveryCursor(consumerData); - } - else if (continueNewTraffic) - { - var token = consumerData.PendingContinuationToken; - consumerData.PendingContinuationToken = null; - consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, token); - } - + consumerData.IsCaughtUp = false; var deliveredAny = false; while (!IsShutdown && !cancellationToken.IsCancellationRequested && consumerData.Cursor is not null) { - if (consumerData.PendingHandshakes != 0) break; - var deliveryCursor = consumerData.Cursor; - var cursorVersion = consumerData.CursorVersion; var batchCursor = options.BatchContainerBatchSize > 1 ? consumerData.Cursor as IQueueCacheCursorBatchDelivery : null; @@ -1782,25 +1373,13 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati ConsumerBatch nextBatch = default; Exception? exceptionOccured = null; var forceFaultSubscription = false; - var deliveryPending = false; try { nextBatch = GetBatchForConsumer(consumerData); if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { - RecordDeliveryProgressError(consumerData); + _useLegacyDeliveryProgress = true; consumerData.SafeDisposeCursor(logger); - if (replacedCursor || consumerData.DeliveryRecovery is { Stopped: true }) - { - break; - } - - if (recoveryAttempt is null) - { - recoveryAttempt = consumerData.DeliveryRecovery!; - if (!recoveryAttempt.TryBeginAttempt(_timeProvider)) break; - } - replacedCursor = true; consumerData.Cursor = GetCacheMissRecoveryCursor(consumerData); continue; } @@ -1812,9 +1391,15 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (!nextBatch.HasProgress) { - consumerData.IsCaughtUp = nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.NoData - && !consumerData.HasDeliveryProgressError - && consumerData.UnconfirmedDeliveryToken is null; + if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.NoData) + { + consumerData.IsCaughtUp = consumerData.LastProcessedToken is not null; + } + else + { + _useLegacyDeliveryProgress = true; + } + // Only emit cursor-drained when we transitioned from delivering to empty, // not on every empty poll. if (deliveredAny) @@ -1824,22 +1409,15 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (QueueCacheCursorContractException) { - RecordDeliveryProgressError(consumerData); consumerData.SafeDisposeCursor(logger); throw; } catch (Exception exc) { - RecordDeliveryProgressError(consumerData); + _useLegacyDeliveryProgress = true; exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); - if (recoveryAttempt is null && consumerData.DeliveryRecovery is { Stopped: false } recoveryState - && recoveryState.TryBeginAttempt(_timeProvider)) - { - recoveryAttempt = recoveryState; - replacedCursor = true; - consumerData.Cursor = GetRecoveryCursor(consumerData); - } + consumerData.Cursor = GetRecoveryCursor(consumerData); } if (exceptionOccured is null) @@ -1848,12 +1426,11 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is null) { - RecordAcknowledgedDeliveryProgress(consumerData, nextBatch.ProgressToken, deliveryCursor, cursorVersion); + consumerData.LastProcessedToken = nextBatch.ProgressToken; continue; } } - long? deliveryStartedAt = null; try { if (_streamInstruments?.PersistentStreamSentMessages.Enabled is true) @@ -1871,50 +1448,23 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (nextBatch.Batch is not null) { var batch = nextBatch.Batch; - if (consumerData.UnconfirmedDeliveryToken is null - || IsBefore(consumerData.UnconfirmedDeliveryToken, nextBatch.ProgressToken!)) - { - consumerData.UnconfirmedDeliveryToken = nextBatch.ProgressToken; - } - - var deliveryTime = options.MaxEventDeliveryTime; - if (consumerData.DeliveryRecovery is { Stopped: false } activeRecovery) - { - deliveryTime = activeRecovery.RemainingTime(_timeProvider, deliveryTime); - if (options.MaxEventDeliveryTime > TimeSpan.Zero && deliveryTime <= TimeSpan.Zero) - { - await StopConsumerRecovery(consumerData, cancellationToken); - break; - } - } - - deliveryPending = true; - deliveryStartedAt = _timeProvider.GetTimestamp(); StreamHandshakeToken? newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( - i => DeliverBatchToConsumer(consumerData, batch, cursorVersion, cancellationToken), - consumerData.DeliveryRecovery is { Stopped: false } ? 1 : AsyncExecutorWithRetries.INFINITE_RETRIES, + i => DeliverBatchToConsumer(consumerData, batch, cancellationToken), + AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not retry if the agent is shutting down, or if the exception is ClientNotAvailableException - (exception, i) => exception is not ClientNotAvailableException && !IsShutdown - && consumerData.CursorVersion == cursorVersion, - deliveryTime, + (exception, i) => exception is not ClientNotAvailableException && !IsShutdown, + this.options.MaxEventDeliveryTime, deliveryBackoffProvider, cancellationToken: cancellationToken); - deliveryPending = false; - if (consumerData.CursorVersion != cursorVersion) - { - continue; - } - - StreamSequenceToken? acknowledgedToken = null; if (newToken is not null) { - RecordReplayRequest(consumerData, newToken); + _useLegacyDeliveryProgress = true; consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; if (newToken is StartPositionToken startPositionToken) { - RecordDeliveryProgress(consumerData, null); + consumerData.LastProcessedToken = null; try { newCursor = GetCacheCursorAtStartPosition( @@ -1934,33 +1484,30 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati ?? throw new InvalidOperationException("A start handshake token must contain a stream sequence token."); if (SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { - acknowledgedToken = sequenceToken; - RecordDeliveryProgress(consumerData, acknowledgedToken); + consumerData.LastProcessedToken = sequenceToken; // An implicit recovery token identifies the last event processed by the prior activation. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( - consumerData, + consumerData.StreamId, sequenceToken, batch.SequenceToken, out pendingBatch); } else { - RecordDeliveryProgress(consumerData, null); + consumerData.LastProcessedToken = null; newCursor = GetCacheCursorOrThrow(consumerData.StreamId, sequenceToken); - TrackDeliveryRecoveryCursor(consumerData, newCursor, newToken); } } else if (newToken is DeliveryToken) { var sequenceToken = newToken.Token ?? throw new InvalidOperationException("A delivery handshake token must contain a stream sequence token."); - acknowledgedToken = sequenceToken; - RecordDeliveryProgress(consumerData, acknowledgedToken); + consumerData.LastProcessedToken = sequenceToken; // The handshake token points to an already processed event, so advance past it. // The current batch is the receiver's first available message if that token was evicted. newCursor = GetAdvancedCursorOrFallback( - consumerData, + consumerData.StreamId, sequenceToken, batch.SequenceToken, out pendingBatch); @@ -1977,28 +1524,14 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } else { - acknowledgedToken = nextBatch.ProgressToken; - RecordAcknowledgedDeliveryProgress(consumerData, acknowledgedToken, deliveryCursor, cursorVersion); - } - - // A rewind or an explicit start request does not acknowledge the attempted batch. - if (acknowledgedToken is { } processed - && consumerData.UnconfirmedDeliveryToken is { } attempted - && !IsBefore(processed, attempted)) - { - consumerData.UnconfirmedDeliveryToken = null; + // Track progress for the periodic delivery scan. + consumerData.LastProcessedToken = nextBatch.ProgressToken; } } } catch (Exception exc) { - if (deliveryPending && consumerData.CursorVersion != cursorVersion) - { - continue; - } - - RecordDeliveryProgressError(consumerData, failedToken: nextBatch.Batch?.SequenceToken, - startedAt: deliveryStartedAt); + _useLegacyDeliveryProgress = true; if (batchCursor is not null && nextBatch.Batch is not null) { batchCursor.RecordDeliveryFailure(nextBatch.Batch); @@ -2027,52 +1560,23 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati forceFaultSubscription, cancellationToken); if (faultedSubscription) return; - if (nextBatch.Batch is not null) - { - // Retry through the normal pump rather than skipping the failed delivery or spinning here. - break; - } } } + consumerData.State = StreamConsumerDataState.Inactive; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + _useLegacyDeliveryProgress = true; + consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { - RecordDeliveryProgressError(consumerData); + _useLegacyDeliveryProgress = true; // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception LogErrorRunConsumerCursor(exc); + consumerData.State = StreamConsumerDataState.Inactive; throw; } - finally - { - if (ownsCursor) - { - try - { - if (recoveryAttempt is not null && ReferenceEquals(recoveryAttempt, consumerData.DeliveryRecovery) - && !recoveryAttempt.Stopped && !IsShutdown) - { - recoveryAttempt.FinishAttempt(_timeProvider, deliveryBackoffProvider, logger); - if (recoveryAttempt.IsExhausted(_timeProvider, options.MaxEventDeliveryTime, RecoveryAttemptMax)) - { - await StopConsumerRecovery(consumerData, cancellationToken); - } - } - } - finally - { - consumerData.State = StreamConsumerDataState.Inactive; - if (!IsShutdown && !consumerData.IsRemoved && consumerData.PendingHandshakes == 0 - && consumerData.DeliveryRecovery is { Stopped: true } - && consumerData.PendingContinuationToken is not null) - { - RunConsumerCursor(consumerData).Ignore(); - } - } - } - } } private readonly struct ConsumerBatch @@ -2120,10 +1624,10 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - batchContainer = GetCurrentBatchOrThrow(cursor); + batchContainer = cursor.GetCurrent(out _) + ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); } - ObserveDeliveryRecoveryStart(consumerData, batchContainer); return ShouldDeliverBatch(streamId, batchContainer, filterData) ? new ConsumerBatch(batchContainer, batchContainer.SequenceToken) : new ConsumerBatch(null, batchContainer.SequenceToken); @@ -2133,11 +1637,9 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) int i = 0; var batchContainers = new List(); StreamSequenceToken? progressToken = null; - var sawBatch = pendingBatch is not null; if (pendingBatch is not null) { - ObserveDeliveryRecoveryStart(consumerData, pendingBatch); progressToken = pendingBatch.SequenceToken; if (ShouldDeliverBatch(streamId, pendingBatch, filterData)) { @@ -2164,9 +1666,8 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) throw new QueueCacheCursorContractException("The cursor move result is not initialized."); } - var batchContainer = GetCurrentBatchOrThrow(cursor); - ObserveDeliveryRecoveryStart(consumerData, batchContainer); - sawBatch = true; + var batchContainer = cursor.GetCurrent(out _) + ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); progressToken = batchContainer.SequenceToken; if (!ShouldDeliverBatch(streamId, batchContainer, filterData)) @@ -2178,7 +1679,7 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) if (progressToken is null) { - return new ConsumerBatch(sawBatch ? QueueCacheCursorMoveResult.Success : QueueCacheCursorMoveResult.NoData); + return new ConsumerBatch(QueueCacheCursorMoveResult.NoData); } return i == 0 @@ -2189,33 +1690,18 @@ private ConsumerBatch GetBatchForConsumer(StreamConsumerData consumerData) return default; } - private static IBatchContainer GetCurrentBatchOrThrow(IQueueCacheCursor cursor) - { - var batch = cursor.GetCurrent(out var exception); - if (exception is not null) - { - throw exception; - } - - return batch ?? throw new QueueCacheCursorContractException("A successful cursor move did not produce a current item."); - } - private sealed class QueueCacheCursorContractException(string message) : InvalidOperationException(message); private async Task DeliverBatchToConsumer( StreamConsumerData consumerData, IBatchContainer batch, - long cursorVersion, CancellationToken cancellationToken) { try { StreamHandshakeToken? newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch, cancellationToken); - if (consumerData.CursorVersion == cursorVersion) - { - consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token - consumerData.StartPositionIsProviderDefault = false; - } + consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token + consumerData.StartPositionIsProviderDefault = false; StreamingEvents.EmitMessageDelivered(streamProviderName, consumerData, batch, Silo); return newToken; @@ -2288,7 +1774,6 @@ private async Task ErrorProtocol( bool forceFaultSubscription = false, CancellationToken cancellationToken = default) { - if (consumerData.IsRemoved) return true; // for loss of client, we just remove the subscription if (exceptionOccured is ClientNotAvailableException) { @@ -2297,24 +1782,8 @@ private async Task ErrorProtocol( return true; } - if (consumerData.HasDeliveryProgressError) - { - if (consumerData.DeliveryRecovery is { Stopped: false } - && consumerData.DeliveryRecoveryFailureReported && !forceFaultSubscription - && consumerData.DeliveryRecoveryFailureIsDelivery == isDeliveryError - && Equals(consumerData.DeliveryRecoveryFailureToken, token)) - { - return false; - } - - consumerData.DeliveryRecoveryFailureReported = true; - consumerData.DeliveryRecoveryFailureIsDelivery = isDeliveryError; - consumerData.DeliveryRecoveryFailureToken = token; - } - // notify consumer about the error or that the data is not available. await DeliverErrorToConsumer(consumerData, exceptionOccured, batch, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); - if (consumerData.IsRemoved) return true; // record that there was a delivery failure if (isDeliveryError) { @@ -2327,7 +1796,6 @@ await streamFailureHandler.OnSubscriptionFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); } - if (consumerData.IsRemoved) return true; // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if ((forceFaultSubscription || streamFailureHandler.ShouldFaultSubsriptionOnError) && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) @@ -2341,17 +1809,13 @@ await DeliverErrorToConsumer( batch, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing | ConfigureAwaitOptions.ContinueOnCapturedContext); - if (consumerData.IsRemoved) return true; // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId, cancellationToken); } finally { // remove subscription - if (!consumerData.IsRemoved) - { - RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); - } + RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); } return true; } @@ -2424,6 +1888,15 @@ private readonly struct QueueIdLogRecord(QueueId queueId) public override string ToString() => queueId.ToStringWithHashCode(); } + [LoggerMessage(Level = LogLevel.Warning, + Message = "Cannot establish the read boundary for queue {QueueId}; using subscription-based delivery progress.")] + private partial void LogWarningUnknownReadProgress(QueueIdLogRecord queueId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Incompatible tokens {Token} and {Other} for queue {QueueId}; using subscription-based delivery progress.")] + private partial void LogWarningIncompatibleQueueProgress( + QueueIdLogRecord queueId, StreamSequenceToken token, StreamSequenceToken other, Exception exception); + [LoggerMessage( Level = LogLevel.Information, EventId = (int)ErrorCode.PersistentStreamPullingAgent_01, @@ -2611,40 +2084,6 @@ private partial void LogWarningIncompatibleDeliveryProgress( StreamSequenceToken otherToken, Exception exception); - [LoggerMessage( - Level = LogLevel.Warning, - Message = "Retaining the previous delivery watermark for queue {Queue}: token {Token} is incompatible with {OtherToken}." - )] - private partial void LogWarningIncompatibleQueueProgress( - QueueIdLogRecord queue, - StreamSequenceToken token, - StreamSequenceToken otherToken, - Exception exception); - - [LoggerMessage( - Level = LogLevel.Error, - Message = "Cannot establish delivery checkpoint progress for queue {Queue}. {Reason} Receiver/provider recovery is required; later reads in this receiver lifetime do not establish replay continuity." - )] - private partial void LogErrorQueueRecoveryRequired(QueueIdLogRecord queue, string reason); - - [LoggerMessage( - Level = LogLevel.Warning, - Message = "Delivery progress is constrained for subscription {SubscriptionId} on stream {StreamId} in queue {Queue}. Continuous replay is required to resume progress; unavailable data requires receiver/provider recovery or an explicit subscription resolution." - )] - private partial void LogWarningDeliveryRecoveryRequired(QueueIdLogRecord queue, GuidId subscriptionId, QualifiedStreamId streamId); - - [LoggerMessage( - Level = LogLevel.Warning, - Message = "Stopping automatic replay after {Attempts} attempts for subscription {SubscriptionId} on stream {StreamId} in queue {Queue}. The unresolved delivery checkpoint constraint is retained; nonfaulting subscriptions can continue with new traffic." - )] - private partial void LogWarningDeliveryRecoveryStopped(QueueIdLogRecord queue, GuidId subscriptionId, QualifiedStreamId streamId, int attempts); - - [LoggerMessage( - Level = LogLevel.Warning, - Message = "Stopping automatic registration recovery after {Attempts} attempts for stream {StreamId} in queue {Queue}. Registration resources are released and delivery checkpoint progress remains unknown." - )] - private partial void LogWarningStreamRegistrationRecoveryStopped(QueueIdLogRecord queue, QualifiedStreamId streamId, int attempts); - [LoggerMessage( Level = LogLevel.Warning, EventId = (int)ErrorCode.PersistentStreamPullingAgent_13, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index 38427afcd28..2b96fd1c4f3 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -1,6 +1,5 @@ using System; using Microsoft.Extensions.Logging; -using Orleans.Internal; using Orleans.Runtime; namespace Orleans.Streams @@ -34,49 +33,23 @@ internal sealed class StreamConsumerData [NonSerialized] public bool IsRegistered = false; [NonSerialized] - public bool IsRemoved; - [NonSerialized] public StreamSequenceToken? PendingStartToken; [NonSerialized] - public StreamSequenceToken? PendingContinuationToken; - [NonSerialized] public IBatchContainer? PendingBatch; [NonSerialized] public bool StartPositionIsProviderDefault; /// - /// The last proven-safe delivery position for this subscription. - /// Cannot advance after an unresolved delivery or cursor error, but earlier replay requests can lower it. + /// The sequence token of the last batch processed (delivered or filtered) by this subscription. + /// Used by the pulling agent's periodic scan to compute the delivery-based checkpoint watermark. /// [NonSerialized] public StreamSequenceToken? LastProcessedToken; - // Inactive also describes interrupted delivery, so it does not prove that the cursor drained. [NonSerialized] public bool IsCaughtUp; [NonSerialized] public int PendingHandshakes; - [NonSerialized] - public bool HasDeliveryProgressError; - [NonSerialized] - public StreamSequenceToken? UnconfirmedDeliveryToken; - // Unlike LastToken, this anchor must survive later fallback deliveries and handshake responses. - [NonSerialized] - public StreamHandshakeToken? DeliveryRecoveryToken; - [NonSerialized] - public IQueueCacheCursor? DeliveryRecoveryCursor; - [NonSerialized] - public bool HasObservedRecoveryStart; - [NonSerialized] - public long CursorVersion; - [NonSerialized] - public StreamRecoveryState? DeliveryRecovery; - [NonSerialized] - public bool DeliveryRecoveryFailureReported; - [NonSerialized] - public bool DeliveryRecoveryFailureIsDelivery; - [NonSerialized] - public StreamSequenceToken? DeliveryRecoveryFailureToken; public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { @@ -88,17 +61,10 @@ public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, ISt internal void SafeDisposeCursor(ILogger logger) { - CursorVersion++; IsCaughtUp = false; PendingBatch = null; if (Cursor is { } cursor) { - if (ReferenceEquals(DeliveryRecoveryCursor, cursor)) - { - DeliveryRecoveryCursor = null; - HasObservedRecoveryStart = false; - } - Cursor = null; // kill cursor activity and ensure it does not start again on this consumer data. try @@ -115,50 +81,6 @@ internal void SafeDisposeCursor(ILogger logger) catch { } Utils.LogIgnoredException(logger, ex, caller); } - - } - } - } - - internal sealed class StreamRecoveryState(long startedAt) - { - private long _lastAttempt; - private TimeSpan _retryDelay; - - public int Attempts { get; private set; } - public bool Stopped { get; set; } - - public TimeSpan RemainingTime(TimeProvider timeProvider, TimeSpan maximumTime) - => maximumTime <= TimeSpan.Zero - ? Timeout.InfiniteTimeSpan - : maximumTime - timeProvider.GetElapsedTime(startedAt); - - public bool IsExhausted(TimeProvider timeProvider, TimeSpan maximumTime, int maximumAttempts) - => Stopped || Attempts >= maximumAttempts - || maximumTime > TimeSpan.Zero && RemainingTime(timeProvider, maximumTime) <= TimeSpan.Zero; - - public bool TryBeginAttempt(TimeProvider timeProvider) - { - if (Attempts != 0 && timeProvider.GetElapsedTime(_lastAttempt) < _retryDelay) - { - return false; - } - - Attempts++; - return true; - } - - public void FinishAttempt(TimeProvider timeProvider, IBackoffProvider backoff, ILogger logger) - { - _lastAttempt = timeProvider.GetTimestamp(); - try - { - _retryDelay = backoff.Next(Attempts - 1); - } - catch (OperationCanceledException exception) - { - _retryDelay = TimeSpan.Zero; - Utils.LogIgnoredException(logger, exception, "Stream recovery backoff was canceled. Retaining bounded recovery without a retry delay."); } } } diff --git a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs index 4ab1cee73b8..01e2671333d 100644 --- a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs +++ b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs @@ -19,15 +19,12 @@ internal sealed class StreamConsumerCollection [Id(2)] public bool StreamRegistered { get; set; } - // A completed, nonnull task retains pending registration work for the next pump to retry. + // Not serialized - runtime-only in-flight registration tracking. [NonSerialized] public Task? RegistrationTask; + [NonSerialized] - public StreamSequenceToken? RegistrationStartToken; - [NonSerialized] - public IQueueCacheCursor? RegistrationCursor; - [NonSerialized] - public StreamRecoveryState? RegistrationRecovery; + public StreamSequenceToken? LastReadToken; public StreamConsumerCollection(DateTime now) { @@ -47,7 +44,6 @@ public bool RemoveConsumer(GuidId subscriptionId, ILogger logger) { if (!queueData.Remove(subscriptionId, out var consumer)) return false; - consumer.IsRemoved = true; consumer.SafeDisposeCursor(logger); return true; } @@ -69,29 +65,13 @@ public IEnumerable AllConsumers() public void DisposeAll(ILogger logger) { - DisposeRegistrationCursor(logger); foreach (StreamConsumerData consumer in queueData.Values) { - consumer.IsRemoved = true; consumer.SafeDisposeCursor(logger); } queueData.Clear(); } - public void DisposeRegistrationCursor(ILogger logger) - { - var cursor = RegistrationCursor; - RegistrationCursor = null; - try - { - cursor?.Dispose(); - } - catch (Exception exception) - { - Utils.LogIgnoredException(logger, exception, "Disposing the pending stream registration cursor."); - } - } - public int Count { get { return queueData.Count; } diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs index aa97bb754f1..c436a4eecd0 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs @@ -99,12 +99,12 @@ QueueCacheCursorResult TryGetCacheCursorAtPosition( bool IsUnderPressure(); /// - /// Updates the cache with the safe delivery-based checkpoint position. + /// Updates the cache with the current delivery progress of all active subscriptions. /// /// - /// The delivery position constrained by subscriptions which still have outstanding work. - /// Includes previously established progress when subscription records have been removed. - /// A value indicates that no delivery progress is known. + /// The earliest last processed sequence token across registered subscriptions. + /// When caught-up progress is known, drained subscriptions can instead advance to the queue's read boundary. + /// A value indicates that there are no active subscriptions. /// The token is only valid for the duration of the call and must not be stored. /// /// The current UTC time. diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 722ba3613fa..4d820c72e35 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -230,7 +230,7 @@ public async Task ReadFromQueue_CachesDequeuedMessagesBeforeObservingCancellatio [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task RegisterStream_RetainsCacheEntryWhenProducerRegistrationTerminates() + public async Task RegisterStream_RemovesCacheEntryWhenProducerRegistrationTerminates() { var queueId = QueueId.GetQueueId("queue", 0u, 0u); var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); @@ -240,12 +240,7 @@ public async Task RegisterStream_RetainsCacheEntryWhenProducerRegistrationTermin await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); - var stream = Assert.Single(await testAccessor.GetPubSubCache()); - Assert.Equal(streamId, stream.Key); - Assert.False(stream.Value.StreamRegistered); - Assert.NotNull(stream.Value.RegistrationTask); - Assert.True(stream.Value.RegistrationTask.IsCompleted); - Assert.Equal(1, Assert.IsType(stream.Value.RegistrationStartToken).SequenceNumber); + Assert.Empty(await testAccessor.GetPubSubCache()); } [TestSuite("BVT")] @@ -405,9 +400,7 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, StreamPullingAgentOptions? options = null, - IStreamFilter? streamFilter = null, - IBackoffProvider? deliveryBackoff = null, - IStreamFailureHandler? failureHandler = null) + IStreamFilter? filter = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -444,18 +437,27 @@ private static PersistentStreamPullingAgent CreateAgent( SystemTargetGrainId.Create(SystemTargetGrainId.CreateGrainType("persistent-stream-pulling-agent-test"), siloAddress), "provider", pubSub!, - streamFilter ?? new NoOpStreamFilter(), + filter ?? new NoOpStreamFilter(), queueId, options ?? new StreamPullingAgentOptions(), queueAdapter, queueAdapterCache!, - failureHandler ?? new NoOpStreamDeliveryFailureHandler(), - deliveryBackoff ?? new FixedBackoff(TimeSpan.Zero), + new NoOpStreamDeliveryFailureHandler(), + new FixedBackoff(TimeSpan.FromMilliseconds(1)), new FixedBackoff(TimeSpan.FromMilliseconds(1)), timeProvider ?? TimeProvider.System, shared); } + private sealed class RecordingSimpleQueueCache() + : SimpleQueueCache(256, NullLogger.Instance), IQueueCache + { + public List DeliveryProgressTokens { get; } = []; + + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => DeliveryProgressTokens.Add(earliestSubscriptionToken); + } + private sealed class RecordingQueueCache : IQueueCache { public int DeliveryProgressCallCount { get; private set; } @@ -604,17 +606,6 @@ public void RecordDeliveryFailure() } } - private static StreamConsumerData AddRegisteredConsumer( - StreamConsumerCollection stream, QualifiedStreamId streamId, IQueueCache cache, DateTime? now = null) - { - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - var consumer = stream.AddConsumer(GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now ?? DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, null); - return consumer; - } - private sealed class ScriptedQueueCache : IQueueCache { private readonly List messages = new(); @@ -897,7 +888,6 @@ public void RecordDeliveryFailure() private sealed class PurgeablePooledQueueCache : IQueueCache { private readonly PooledQueueCache cache; - public List DeliveryProgressTokens { get; } = []; public PurgeablePooledQueueCache(bool retainPurgeMetadata = false) @@ -974,8 +964,6 @@ public void Purge() } } - public void PurgeOldest() => cache.RemoveOldestMessage(); - private sealed class CacheDataAdapter : ICacheDataAdapter { public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) @@ -1072,32 +1060,14 @@ private sealed class DerivedEventSequenceTokenV2(long sequenceNumber, int eventI protected override Type SequenceTokenCompatibilityDomain => typeof(EventSequenceToken); } - private sealed class CountingEventSequenceToken(long sequenceNumber, Action onComparison) - : EventSequenceTokenV2(sequenceNumber) - { - public override int CompareTo(StreamSequenceToken? other) - { - onComparison(); - return base.CompareTo(other); - } - } - private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = null) : IStreamConsumerExtension { private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public List DeliveredBatches { get; } = new(); public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); - public Queue DeliveryResponses { get; } = new(); - public StreamHandshakeToken? HandshakeResponse { get; set; } = requestedToken; - public Exception? DeliveryException { get; set; } - public Action? OnDelivery { get; set; } - public Action? OnHandshake { get; set; } - public Task? HandshakeTask { get; set; } - public int HandshakeCount { get; private set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1107,18 +1077,11 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public async Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) { - DeliveredBatches.Add(item); DeliveredTokens.Add(item.SequenceToken); DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); await releaseDelivery.Task; - OnDelivery?.Invoke(); - if (DeliveryException is { } exception) - { - throw exception; - } - - return DeliveryResponses.TryDequeue(out var response) ? response : null; + return null; } public Task CompleteStream(GuidId subscriptionId, CancellationToken cancellationToken) => Task.CompletedTask; @@ -1129,12 +1092,7 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke return Task.CompletedTask; } - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) - { - HandshakeCount++; - OnHandshake?.Invoke(); - return HandshakeTask ?? Task.FromResult(HandshakeResponse); - } + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(requestedToken); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -2200,7 +2158,6 @@ public async Task ProviderDefaultEarliestRejectsInvalidCustomCacheResult() public async Task RunConsumerCursorStopsAfterInvalidMoveResult() { var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); var queueCache = new InvalidMoveQueueCache(); var queueAdapterCache = Substitute.For(); queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); @@ -2209,7 +2166,7 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); var agent = CreateAgent( pubSub, - queueId, + QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); @@ -2232,14 +2189,6 @@ public async Task RunConsumerCursorStopsAfterInvalidMoveResult() Assert.True(queueCache.Cursor.IsDisposed); Assert.Equal(StreamConsumerDataState.Inactive, consumerData.State); Assert.Equal(0, queueCache.CursorAcquisitionCount); - - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - - Assert.Null(consumerData.Cursor); - Assert.True(consumerData.HasDeliveryProgressError); - Assert.Null(consumerData.DeliveryRecoveryToken); - Assert.Equal(0, queueCache.CursorAcquisitionCount); - await accessor.Shutdown(); } private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamConsumerExtension @@ -2589,6 +2538,439 @@ await testAccessor.ReadFromQueue( Assert.Equal(attemptedToken, Assert.Single(queueCache.DeliveryProgressTokens)); } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, 1, false)] + [InlineData(false, 10, false)] + [InlineData(true, 1, false)] + [InlineData(true, 10, false)] + [InlineData(false, 1, true)] + [InlineData(false, 10, true)] + [InlineData(true, 1, true)] + [InlineData(true, 10, true)] + public async Task Checkpoint_DrainedSubscriptionsDoNotPinOtherStreams(bool pooled, int batchSize, bool filtered) + { + await using var scenario = await CreateCheckpointScenario(pooled, batchSize, filtered); + var batches = Enumerable.Range(1, 200) + .Select(sequence => (IBatchContainer)new TestBatchContainer( + sequence == 1 ? scenario.Idle.StreamId.StreamId : scenario.Busy.StreamId.StreamId, + new EventSequenceTokenV2(sequence))) + .ToList(); + + await scenario.Read(batches); + + Assert.True(scenario.Idle.IsCaughtUp); + Assert.True(scenario.Busy.IsCaughtUp); + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + if (filtered) + { + Assert.Empty(((ImmediateRecordingConsumer)scenario.Idle.StreamConsumer).DeliveredTokens); + Assert.Empty(((ImmediateRecordingConsumer)scenario.Busy.StreamConsumer).DeliveredTokens); + } + + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("active", 1L)] + [InlineData("pending-batch", 1L)] + [InlineData("handshake", 1L)] + [InlineData("disposed", 1L)] + [InlineData("not-drained", 1L)] + [InlineData("unknown-stream-read", 1L)] + [InlineData("unregistered-producer", 1L)] + [InlineData("unregistered-consumer", null)] + [InlineData("unknown-progress", null)] + public async Task Checkpoint_UnprovenSubscriptionsUseOriginalProgress(string condition, long? expected) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + var streams = await scenario.Accessor.GetPubSubCache(); + switch (condition) + { + case "active": + scenario.Idle.State = StreamConsumerDataState.Active; + break; + case "pending-batch": + scenario.Idle.PendingBatch = new TestBatchContainer(scenario.Idle.StreamId.StreamId, new EventSequenceTokenV2(1)); + break; + case "handshake": + scenario.Idle.PendingHandshakes = 1; + break; + case "disposed": + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + break; + case "not-drained": + scenario.Idle.IsCaughtUp = false; + break; + case "unknown-stream-read": + streams[scenario.Idle.StreamId].LastReadToken = null; + break; + case "unregistered-producer": + streams[scenario.Idle.StreamId].StreamRegistered = false; + break; + case "unregistered-consumer": + scenario.Idle.IsRegistered = false; + break; + case "unknown-progress": + scenario.Idle.LastProcessedToken = null; + break; + default: + throw new InvalidOperationException(condition); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown(); + if (expected.HasValue) + { + Assert.Equal(expected.Value, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + else + { + Assert.Empty(scenario.Checkpoints); + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_NewReadsMustBeAcknowledgedAfterDraining(bool pooled) + { + await using var scenario = await CreateCheckpointScenario(pooled); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + + Assert.Equal(3, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_EmptyCursorCannotHideAnUnacknowledgedRead(bool pooled) + { + await using var scenario = await CreateCheckpointScenario(pooled); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + if (scenario.Cache is PurgeablePooledQueueCache pooledCache) + { + pooledCache.Purge(); + scenario.Idle.Cursor = pooledCache.GetCacheCursor(scenario.Idle.StreamId, scenario.Idle.LastProcessedToken); + } + else + { + scenario.Idle.Cursor = new EmptyQueueCacheCursor(); + } + + scenario.Idle.State = StreamConsumerDataState.Inactive; + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + await scenario.AssertCheckpoint(1); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, 0)] + [InlineData(false, 1)] + [InlineData(true, 0)] + [InlineData(true, 1)] + public async Task Checkpoint_InitialResumeCanEstablishCaughtUpProgress(bool pooled, long acknowledged) + { + await using var scenario = await CreateCheckpointScenario(pooled); + scenario.Idle.IsRegistered = false; + var consumer = new RecordingConsumer(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(acknowledged))); + consumer.ReleaseDelivery(); + scenario.Idle.StreamConsumer = consumer; + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + + Assert.True(await scenario.Accessor.DoHandshakeWithConsumer(scenario.Idle, new EventSequenceTokenV2(1))); + scenario.Idle.LastProcessedToken = PersistentStreamPullingAgent.GetInitialDeliveryProgress(scenario.Idle.LastToken); + scenario.Idle.IsRegistered = true; + scenario.Idle.PendingStartToken = null; + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(acknowledged == 0 ? 1 : 0, consumer.DeliveredTokens.Count); + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_CursorFailureDoesNotFreezeLaterOriginalProgress(bool failCursor) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + if (failCursor) + { + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + } + + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + scenario.Idle.Cursor = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(3)); + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(3, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + await scenario.AssertCheckpoint(failCursor ? 3 : 200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_InFlightDeliveryConstrainsProgressUntilAcknowledged(bool acknowledge) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + var consumer = new RecordingConsumer(); + scenario.Busy.StreamConsumer = consumer; + scenario.Busy.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Busy, 200)); + scenario.Busy.State = StreamConsumerDataState.Inactive; + var delivery = scenario.Accessor.RunConsumerCursor(scenario.Busy); + + try + { + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (acknowledge) + { + consumer.ReleaseDelivery(); + await delivery; + } + + await scenario.AssertCheckpoint(acknowledge ? 200 : 2); + } + finally + { + consumer.ReleaseDelivery(); + await delivery; + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("undelivered")] + [InlineData("incompatible")] + [InlineData("pending-registration")] + public async Task Checkpoint_UncertainRemovalUsesRemainingOriginalProgress(string condition) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Idle, 200)); + + if (condition == "incompatible") + { + scenario.Idle.State = StreamConsumerDataState.Inactive; + scenario.Idle.LastProcessedToken = new IsolatedProviderToken(200); + } + else if (condition == "pending-registration") + { + var streams = await scenario.Accessor.GetPubSubCache(); + streams[scenario.Idle.StreamId].RegistrationTask = new TaskCompletionSource().Task; + } + + await scenario.Remove(scenario.Idle); + await scenario.AssertCheckpoint(2); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_NoSubscriptionsDoNotManufactureProgress(bool previouslySubscribed) + { + await using var scenario = await CreateCheckpointScenario(); + if (!previouslySubscribed) + { + var streams = await scenario.Accessor.GetPubSubCache(); + streams[scenario.Idle.StreamId].RemoveConsumer(scenario.Idle.SubscriptionId, NullLogger.Instance); + streams[scenario.Busy.StreamId].RemoveConsumer(scenario.Busy.SubscriptionId, NullLogger.Instance); + } + + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + if (previouslySubscribed) + { + await scenario.Remove(scenario.Idle); + await scenario.Remove(scenario.Busy); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown(); + Assert.Null(Assert.Single(scenario.Checkpoints)); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("ordered", 300L)] + [InlineData("unordered", 1L)] + [InlineData("unordered-batch", 1L)] + [InlineData("incompatible", 1L)] + [InlineData("missing-token", 1L)] + [InlineData("null-batch", 1L)] + public async Task Checkpoint_UncertainReadUsesOriginalProgress(string condition, long expected) + { + await using var scenario = await CreateCheckpointScenario(emptyCache: new RecordingQueueCache()); + scenario.Idle.LastProcessedToken = new EventSequenceTokenV2(1); + scenario.Busy.LastProcessedToken = new EventSequenceTokenV2(200); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + + var other = new QualifiedStreamId("provider", StreamId.Create("other", Guid.NewGuid())); + await scenario.Accessor.RegisterStream(other, new EventSequenceTokenV2(201), DateTime.UtcNow); + StreamSequenceToken? token = condition switch + { + "unordered" => new EventSequenceTokenV2(100), + "incompatible" => new IsolatedProviderToken(300), + "missing-token" => null, + _ => new EventSequenceTokenV2(300), + }; + IList batches = condition == "null-batch" + ? [null!] + : [new TestBatchContainer(other.StreamId, token!)]; + if (condition == "unordered-batch") + { + batches.Add(new TestBatchContainer(other.StreamId, new EventSequenceTokenV2(250))); + } + + await scenario.Read(batches); + await scenario.AssertCheckpoint(expected); + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Checkpoint_IncompatibleReadBoundaryStillAllowsOriginalSubscriberToken() + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + await scenario.Remove(scenario.Busy); + scenario.Idle.LastProcessedToken = new IsolatedProviderToken(10); + + await scenario.AssertCheckpoint(10); + Assert.IsType(Assert.Single(scenario.Checkpoints)); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_ReinitializeClearsReadBoundaryAndOptimizationOptOut(bool readAgain) + { + await using var scenario = await CreateCheckpointScenario(emptyCache: new RecordingQueueCache()); + scenario.Idle.LastProcessedToken = new EventSequenceTokenV2(1); + scenario.Busy.LastProcessedToken = new EventSequenceTokenV2(200); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + await scenario.Accessor.Shutdown(); + await InitializeAgent((PersistentStreamPullingAgent)scenario.Accessor); + + await scenario.Accessor.RegisterStream(scenario.Idle.StreamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + await scenario.Accessor.RegisterStream(scenario.Busy.StreamId, new EventSequenceTokenV2(200), DateTime.UtcNow); + var streams = await scenario.Accessor.GetPubSubCache(); + var restarted = scenario with + { + Idle = AddCheckpointConsumer(streams[scenario.Idle.StreamId], scenario.Idle.StreamId, scenario.Cache), + Busy = AddCheckpointConsumer(streams[scenario.Busy.StreamId], scenario.Busy.StreamId, scenario.Cache), + }; + restarted.Idle.LastProcessedToken = streams[scenario.Idle.StreamId].LastReadToken = new EventSequenceTokenV2(1); + restarted.Busy.LastProcessedToken = streams[scenario.Busy.StreamId].LastReadToken = new EventSequenceTokenV2(200); + restarted.Idle.IsCaughtUp = restarted.Busy.IsCaughtUp = true; + + if (readAgain) + { + await restarted.Read((restarted.Idle, 1), (restarted.Busy, 200)); + } + + await restarted.AssertCheckpoint(readAgain ? 200 : 1); + } + + private static async Task CreateCheckpointScenario( + bool pooled = false, int batchSize = 1, bool filtered = false, RecordingQueueCache? emptyCache = null) + { + IQueueCache cache; + List checkpoints; + if (emptyCache is not null) + { + cache = emptyCache; + checkpoints = emptyCache.DeliveryProgressTokens; + } + else if (pooled) + { + var pooledCache = new PurgeablePooledQueueCache(); + cache = pooledCache; + checkpoints = pooledCache.DeliveryProgressTokens; + } + else + { + var simpleCache = new RecordingSimpleQueueCache(); + cache = simpleCache; + checkpoints = simpleCache.DeliveryProgressTokens; + } + + var idleId = new QualifiedStreamId("provider", StreamId.Create("idle", Guid.NewGuid())); + var busyId = new QualifiedStreamId("provider", StreamId.Create("busy", Guid.NewGuid())); + var (accessor, _, idleStream) = await CreateInitializedAgentWithStream( + idleId, new EventSequenceTokenV2(1), cache, + new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, + filtered ? Substitute.For() : null); + await accessor.RegisterStream(busyId, new EventSequenceTokenV2(2), DateTime.UtcNow); + var busyStream = (await accessor.GetPubSubCache())[busyId]; + + return new(accessor, cache, checkpoints, + AddCheckpointConsumer(idleStream, idleId, cache), AddCheckpointConsumer(busyStream, busyId, cache)); + } + + private static StreamConsumerData AddCheckpointConsumer( + StreamConsumerCollection stream, QualifiedStreamId id, IQueueCache cache) + { + var consumer = stream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), id, new ImmediateRecordingConsumer(), null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.Cursor = cache.GetCacheCursor(id, null); + return consumer; + } + + private sealed record CheckpointScenario( + PersistentStreamPullingAgent.ITestAccessor Accessor, + IQueueCache Cache, + List Checkpoints, + StreamConsumerData Idle, + StreamConsumerData Busy) : IAsyncDisposable + { + public Task Read(params (StreamConsumerData Consumer, long Sequence)[] messages) + => Read(messages.Select(message => (IBatchContainer)new TestBatchContainer( + message.Consumer.StreamId.StreamId, new EventSequenceTokenV2(message.Sequence))).ToList()); + + public async Task Read(IList messages) + { + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(messages)); + await Accessor.ReadFromQueue(QueueId.GetQueueId("queue", 0u, 0u), receiver, 1000); + } + + public Task Remove(StreamConsumerData consumer) + { + var agent = (PersistentStreamPullingAgent)Accessor; + return agent.RunOrQueueTask(() => agent.RemoveSubscriber( + consumer.SubscriptionId, consumer.StreamId, TestContext.Current.CancellationToken)); + } + + public async Task AssertCheckpoint(long expected) + { + Checkpoints.Clear(); + await Accessor.Shutdown(); + Assert.Equal(expected, Assert.Single(Checkpoints)?.SequenceNumber); + } + + public ValueTask DisposeAsync() => new(Accessor.Shutdown()); + } + private static Task InitializeAgent(PersistentStreamPullingAgent agent) => agent.RunOrQueueTask(() => agent.Initialize(TestContext.Current.CancellationToken)); @@ -2599,7 +2981,8 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => QualifiedStreamId streamId, StreamSequenceToken registrationToken, IQueueCache queueCache, - StreamPullingAgentOptions options) + StreamPullingAgentOptions options, + IStreamFilter? filter = null) { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default) @@ -2612,7 +2995,8 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache, - options: options); + options: options, + filter: filter); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); await accessor.RegisterStream(streamId, registrationToken, DateTime.UtcNow); @@ -2780,2928 +3164,79 @@ public async Task RunQueuePump_ReadsAfterReinitialize() [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false, 200)] - [InlineData(true, false, 0)] - [InlineData(false, true, 1)] - public async Task Shutdown_AdvancesPastOnlyDrainedSubscriptions(bool holdFirstDelivery, bool holdNewDelivery, long expectedCheckpoint) + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_PushesEarliestDeliveryProgressTokenToCache() { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); var receiver = Substitute.For(); - IList messages = - [ - new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1)), - new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200)) - ]; receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(_ => Task.FromResult(messages)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + .Returns(Task.FromResult>(new List())); + receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); - await accessor.RegisterStream(idleId, new EventSequenceTokenV2(0), DateTime.UtcNow); - await accessor.RegisterStream(activeId, new EventSequenceTokenV2(0), DateTime.UtcNow); - var streams = await accessor.GetPubSubCache(); - var idle = new RecordingConsumer(); - var active = new RecordingConsumer(); - var pending = new RecordingConsumer(); - active.ReleaseDelivery(); - if (!holdFirstDelivery) - { - idle.ReleaseDelivery(); - } + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); - var idleData = AddConsumer(streams[idleId], idleId, idle); - var activeData = AddConsumer(streams[activeId], activeId, active); - try - { - await accessor.ReadFromQueue(queueId, receiver, 1000); - await idle.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await active.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(holdFirstDelivery ? 0 : 1, idleData.LastProcessedToken!.SequenceNumber); - Assert.Equal(200, activeData.LastProcessedToken!.SequenceNumber); - Assert.Equal(StreamConsumerDataState.Inactive, activeData.State); - Assert.Equal( - holdFirstDelivery ? StreamConsumerDataState.Active : StreamConsumerDataState.Inactive, - idleData.State); - Assert.Empty(idle.Errors); - Assert.Empty(active.Errors); - if (holdNewDelivery) - { - idleData.StreamConsumer = pending; - messages = - [ - new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(201)), - new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(300)) - ]; - await accessor.ReadFromQueue(queueId, receiver, 1000); - await pending.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(1, idleData.LastProcessedToken.SequenceNumber); - Assert.Equal(300, activeData.LastProcessedToken.SequenceNumber); - Assert.Equal(StreamConsumerDataState.Active, idleData.State); - } + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + Assert.Null(streamData.RegistrationTask); + queueCache.ClearDeliveryProgress(); - await accessor.Shutdown(); - var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); - Assert.NotNull(checkpoint); - Assert.Equal(expectedCheckpoint, checkpoint.SequenceNumber); - } - finally - { - idle.ReleaseDelivery(); - pending.ReleaseDelivery(); - } + var newestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: DateTime.UtcNow); + newestConsumer.IsRegistered = true; + newestConsumer.LastProcessedToken = new EventSequenceTokenV2(200); - StreamConsumerData AddConsumer(StreamConsumerCollection collection, QualifiedStreamId id, RecordingConsumer consumer) - { - var data = collection.AddConsumer(GuidId.GetGuidId(Guid.NewGuid()), id, consumer, null, DateTime.UtcNow); - data.IsRegistered = true; - data.LastProcessedToken = new EventSequenceTokenV2(0); - data.Cursor = queueCache.GetCacheCursor(id.StreamId, null); - return data; - } + var earliestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: DateTime.UtcNow); + earliestConsumer.IsRegistered = true; + earliestConsumer.LastProcessedToken = new EventSequenceTokenV2(95); + + await testAccessor.Shutdown(); + + Assert.Equal(earliestConsumer.LastProcessedToken, Assert.Single(queueCache.DeliveryProgressTokens)); } [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Shutdown_DoesNotSkipFailedRegistrationOrInitialAttachment(bool subscriberAttachment, bool unknownPosition) + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_PushesEarliestDeliveryProgressUsingBaseTokenPosition() { - var failedId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var otherId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var subscription = new PubSubSubscriptionState( - GuidId.GetGuidId(Guid.NewGuid()), failedId, GrainId.Create("test", Guid.NewGuid().ToString())); var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(call => !subscriberAttachment - ? Task.FromException>(new InvalidOperationException("Injected registration failure.")) - : Task.FromResult>(call.ArgAt(0).Equals(failedId) - ? new HashSet { subscription } - : new HashSet())); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(_ => throw new OperationCanceledException("Injected registration retry cancellation.")); + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var queueCache = new RecordingQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); var receiver = Substitute.For(); receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>( - [ - new TestBatchContainer(failedId.StreamId, unknownPosition ? null! : new EventSequenceTokenV2(100)), - new TestBatchContainer(otherId.StreamId, new EventSequenceTokenV2(200)) - ])); + .Returns(Task.FromResult>(new List())); + receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - // Canceling the retry wait terminates registration. A missing RuntimeClient fails - // attachment before a consumer record exists, after producer registration succeeds. - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, deliveryBackoff: backoff); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var streams = await accessor.GetPubSubCache(); - await Task.WhenAll(streams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - if (subscriberAttachment) - { - Assert.True(streams[failedId].StreamRegistered); - Assert.Equal(0, streams[failedId].Count); - var pendingRegistration = streams[failedId].RegistrationTask; - Assert.NotNull(pendingRegistration); - Assert.True(pendingRegistration.IsCompleted); - } - else - { - var retainedStreams = await accessor.GetPubSubCache(); - Assert.True(retainedStreams.ContainsKey(failedId)); - Assert.All(retainedStreams.Values, stream => - { - Assert.False(stream.StreamRegistered); - Assert.NotNull(stream.RegistrationTask); - Assert.True(stream.RegistrationTask.IsCompleted); - }); - } - - await accessor.Shutdown(); - - Assert.Empty(queueCache.DeliveryProgressTokens); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false)] - [InlineData(true)] - public async Task Shutdown_DoesNotAdvanceToReadCancelledAfterReceiverReturns(bool shutdownDuringRead) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = StreamId.Create("namespace", Guid.NewGuid()); - var queueCache = new ScriptedQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var readStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var finishRead = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var receiver = Substitute.For(); - var readCount = 0; - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => - { - if (++readCount == 1) - { - return Task.FromResult>([new TestBatchContainer(streamId, new EventSequenceTokenV2(10))]); - } - - readStarted.TrySetResult(); - return finishRead.Task; - }); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(new("provider", streamId), new EventSequenceTokenV2(10), DateTime.UtcNow); - var consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, new("provider", streamId), queueCache); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(consumer.IsCaughtUp); - Assert.Equal(10, consumer.LastProcessedToken!.SequenceNumber); - using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var pump = accessor.RunQueuePump(queueId, cancellation.Token); - await readStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Task? shutdown = null; - if (shutdownDuringRead) - { - shutdown = accessor.Shutdown(); - Assert.False(shutdown.IsCompleted); - } - else - { - cancellation.Cancel(); - } - - finishRead.SetResult([new TestBatchContainer(streamId, new EventSequenceTokenV2(100))]); - await pump.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await (shutdown ?? accessor.Shutdown()).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); - Assert.NotNull(checkpoint); - Assert.Equal(10, checkpoint.SequenceNumber); - await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Recovery_InitialGroupedFailureRequiresFirstBatchForInclusiveReplay( - bool purgeFirstBatch, bool explicitEarliestRequest) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var firstToken = new EventSequenceTokenV2(10); - var lastToken = new EventSequenceTokenV2(20); - var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>( - [ - new TestBatchContainer(streamId.StreamId, firstToken), - new TestBatchContainer(streamId.StreamId, lastToken), - ]), - Task.FromResult>([])); - var observer = new RecordingConsumer(explicitEarliestRequest - ? StreamHandshakeToken.CreateStartPositionToken(StreamSubscriptionStartPosition.EarliestAvailable) - : null) - { - DeliveryException = new InvalidOperationException("Injected grouped delivery failure."), - }; - observer.ReleaseDelivery(); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, new FakeTimeProvider(), - options: new StreamPullingAgentOptions - { - BatchContainerBatchSize = 2, - InitialSubscriptionStartPosition = explicitEarliestRequest - ? StreamSubscriptionStartPosition.Latest - : StreamSubscriptionStartPosition.EarliestAvailable, - MaxEventDeliveryTime = TimeSpan.FromMilliseconds(100), - }, - deliveryBackoff: new FixedBackoff(TimeSpan.FromMilliseconds(150))); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); - consumer.IsRegistered = true; - Assert.Null(consumer.LastProcessedToken); - Assert.False(consumer.HasDeliveryProgressError); - - await accessor.RunConsumerCursor(consumer).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.NotEmpty(observer.DeliveredBatches); - Assert.All(observer.DeliveredBatches, batch => Assert.Equal(new[] { 10L, 20L }, - Assert.IsType(batch).BatchContainers.Select(item => item.SequenceToken.SequenceNumber))); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Equal(firstToken, Assert.IsType(consumer.DeliveryRecoveryToken).Token); - Assert.Null(consumer.LastProcessedToken); - Assert.Equal(lastToken, consumer.UnconfirmedDeliveryToken); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - - if (purgeFirstBatch) - { - queueCache.PurgeOldest(); - var acceptedOldPosition = queueCache.TryGetCacheCursor(streamId.StreamId, firstToken); - Assert.Equal(QueueCacheCursorResultKind.Success, acceptedOldPosition.Kind); - using var probe = Assert.IsAssignableFrom(acceptedOldPosition.Cursor); - Assert.Equal(QueueCacheCursorMoveResultKind.Success, probe.MoveNextWithResult().Kind); - Assert.Equal(20, Assert.IsType(probe.GetCurrent(out _)).SequenceToken.SequenceNumber); - } - - observer.DeliveryException = null; - observer.DeliveredBatches.Clear(); - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - Assert.NotEmpty(observer.DeliveredBatches); - Assert.All(observer.DeliveredBatches, batch => Assert.Equal( - purgeFirstBatch ? [20L] : new[] { 10L, 20L }, - Assert.IsType(batch).BatchContainers.Select(item => item.SequenceToken.SequenceNumber))); - Assert.Equal(purgeFirstBatch, consumer.HasDeliveryProgressError); - Assert.Equal(!purgeFirstBatch, consumer.IsCaughtUp); - if (purgeFirstBatch) - { - Assert.Equal(firstToken, Assert.IsType(consumer.DeliveryRecoveryToken).Token); - Assert.Null(consumer.LastProcessedToken); - } - else - { - Assert.Null(consumer.DeliveryRecoveryToken); - Assert.Equal(lastToken, consumer.LastProcessedToken); - } - - await accessor.Shutdown(); - if (purgeFirstBatch) - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - else - { - Assert.Equal(lastToken, Assert.Single(queueCache.DeliveryProgressTokens)); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("delivery", false, false, 1)] - [InlineData("delivery", false, true, 1)] - [InlineData("delivery", true, false, 1)] - [InlineData("delivery", true, true, 1)] - [InlineData("filter", false, false, 1)] - [InlineData("filter", false, true, 1)] - [InlineData("filter", true, false, 1)] - [InlineData("filter", true, true, 1)] - [InlineData("rewind", false, false, 1)] - [InlineData("rewind", false, true, 1)] - [InlineData("rewind", true, false, 1)] - [InlineData("rewind", true, true, 1)] - [InlineData("reattach", false, false, 1)] - [InlineData("reattach", false, true, 1)] - [InlineData("reattach", true, false, 1)] - [InlineData("reattach", true, true, 1)] - [InlineData("delivery", false, false, 2)] - [InlineData("delivery", false, true, 2)] - [InlineData("delivery", true, false, 2)] - [InlineData("delivery", true, true, 2)] - [InlineData("filter", false, false, 2)] - [InlineData("filter", false, true, 2)] - [InlineData("filter", true, false, 2)] - [InlineData("filter", true, true, 2)] - [InlineData("rewind", false, false, 2)] - [InlineData("rewind", false, true, 2)] - [InlineData("rewind", true, false, 2)] - [InlineData("rewind", true, true, 2)] - [InlineData("reattach", false, false, 2)] - [InlineData("reattach", false, true, 2)] - [InlineData("reattach", true, false, 2)] - [InlineData("reattach", true, true, 2)] - public async Task Shutdown_PreservesReplayRequirementAfterLaterProgress( - string progressKind, bool unknownPosition, bool expireStream, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var timeProvider = new FakeTimeProvider(); - var now = timeProvider.GetUtcNow().UtcDateTime; - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new ScriptedQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - StreamSequenceToken? safeToken = unknownPosition ? null : new EventSequenceTokenV2(10); - var laterToken = new EventSequenceTokenV2(200); - var batch = Substitute.For(); - batch.StreamId.Returns(streamId.StreamId); - batch.SequenceToken.Returns(laterToken); - batch.GetEvents().Returns([Tuple.Create("payload", laterToken)]); - var filter = progressKind == "filter" ? Substitute.For() : null; - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([batch]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, streamFilter: filter); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), now); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var observer = new RecordingConsumer(); - if (progressKind == "rewind") - { - observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(100))); - } - - observer.ReleaseDelivery(); - var failedCursor = Substitute.For(); - failedCursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected cursor failure.")); - var consumer = stream.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now); - consumer.IsRegistered = true; - consumer.LastProcessedToken = safeToken; - consumer.Cursor = failedCursor; - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - if (progressKind == "reattach") - { - observer.HandshakeResponse = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(300)); - await agent.RunOrQueueTask(() => agent.AddSubscriber( - consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, - TestContext.Current.CancellationToken)); - } - - failedCursor.Received(1).MoveNextWithResult(); - var recoveredInitially = !unknownPosition && progressKind != "rewind"; - Assert.Equal(!recoveredInitially, consumer.HasDeliveryProgressError); - Assert.Equal(recoveredInitially, consumer.IsCaughtUp); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - if (recoveredInitially) - { - Assert.Equal(progressKind == "reattach" ? 300 : 200, - Assert.IsType(consumer.LastProcessedToken).SequenceNumber); - } - else - { - Assert.Same(safeToken, consumer.LastProcessedToken); - } - Assert.Equal(0, consumer.PendingHandshakes); - if (progressKind == "filter") - { - Assert.NotNull(filter); - filter.Received(1).ShouldDeliver(streamId.StreamId, "payload", null); - Assert.Null(consumer.LastToken); - Assert.Empty(observer.DeliveredTokens); - } - else - { - var deliveryToken = Assert.IsType(consumer.LastToken); - var sequenceToken = Assert.IsType(deliveryToken.Token); - Assert.Equal(progressKind == "reattach" ? 300 : 200, sequenceToken.SequenceNumber); - Assert.Equal(progressKind == "rewind" ? 2 : 1, observer.DeliveredTokens.Count); - Assert.All(observer.DeliveredTokens, token => Assert.Same(laterToken, token)); - } - - // A later rewind cursor is not proof of replay from the original safe anchor. - // The next pump must retry that anchor before releasing the error. - Assert.Null(consumer.UnconfirmedDeliveryToken); - - if (expireStream) - { - timeProvider.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - if (unknownPosition) - { - Assert.Empty(await accessor.GetPubSubCache()); - Assert.True(consumer.HasDeliveryProgressError); - } - else - { - Assert.Equal(!recoveredInitially, consumer.HasDeliveryProgressError); - Assert.Empty(await accessor.GetPubSubCache()); - } - } - - await accessor.Shutdown(); - - if (unknownPosition) - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - else - { - var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); - Assert.NotNull(checkpoint); - Assert.Equal(recoveredInitially ? 200 : 10, checkpoint.SequenceNumber); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false, 1)] - [InlineData(false, true, 1)] - [InlineData(true, false, 1)] - [InlineData(true, true, 1)] - [InlineData(false, false, 2)] - [InlineData(false, true, 2)] - [InlineData(true, false, 2)] - [InlineData(true, true, 2)] - public async Task Shutdown_PreservesReplayRequirementAfterRewindToEmptyCursor(bool unknownPosition, bool expireStream, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var timeProvider = new FakeTimeProvider(); - var now = timeProvider.GetUtcNow().UtcDateTime; - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - StreamSequenceToken? safeToken = unknownPosition ? null : new EventSequenceTokenV2(10); - var attemptedToken = new EventSequenceTokenV2(200); - var failedCursor = Substitute.For(); - failedCursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected cursor failure.")); - var recoveryCursor = Substitute.For(); - recoveryCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); - recoveryCursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, attemptedToken)); - var queueCache = Substitute.For(); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, attemptedToken)]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), now); - var observer = new RecordingConsumer(); - observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(100))); - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, now); - consumer.IsRegistered = true; - consumer.LastProcessedToken = safeToken; - consumer.Cursor = failedCursor; - // Model a provider whose attempted batch is no longer retained when the consumer rewinds. - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns( - QueueCacheCursorResult.FromCursor(recoveryCursor), - QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - - failedCursor.Received(1).MoveNextWithResult(); - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(consumer.IsCaughtUp); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Same(safeToken, consumer.LastProcessedToken); - Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); - var deliveryToken = Assert.IsType(consumer.LastToken); - var sequenceToken = Assert.IsType(deliveryToken.Token); - Assert.Equal(100, sequenceToken.SequenceNumber); - Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); - if (expireStream) - { - timeProvider.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Null(consumer.UnconfirmedDeliveryToken); - Assert.Null(consumer.Cursor); - Assert.Null(consumer.PendingBatch); - } - - await accessor.Shutdown(); - - if (unknownPosition) - { - queueCache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); - } - else - { - queueCache.Received(1).UpdateDeliveryProgress(safeToken, Arg.Any()); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("delivery", false, false, false)] - [InlineData("delivery", false, true, false)] - [InlineData("delivery", true, false, false)] - [InlineData("delivery", true, true, false)] - [InlineData("implicit", false, false, false)] - [InlineData("implicit", false, true, false)] - [InlineData("implicit", true, false, false)] - [InlineData("implicit", true, true, false)] - [InlineData("explicit", false, false, false)] - [InlineData("explicit", false, true, false)] - [InlineData("explicit", true, false, false)] - [InlineData("explicit", true, true, false)] - [InlineData("delivery", false, false, true)] - [InlineData("delivery", false, true, true)] - [InlineData("delivery", true, false, true)] - [InlineData("delivery", true, true, true)] - [InlineData("implicit", false, false, true)] - [InlineData("implicit", false, true, true)] - [InlineData("implicit", true, false, true)] - [InlineData("implicit", true, true, true)] - [InlineData("explicit", false, false, true)] - [InlineData("explicit", false, true, true)] - [InlineData("explicit", true, false, true)] - [InlineData("explicit", true, true, true)] - public async Task Shutdown_PreservesReplayFloorForEarlierRequest( - string tokenKind, bool unknownPosition, bool failCursorAcquisition, bool reattach) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var acknowledgedToken = new EventSequenceTokenV2(100); - var attemptedToken = new EventSequenceTokenV2(200); - var earlierToken = new EventSequenceTokenV2(50); - var requestedToken = tokenKind == "delivery" - ? StreamHandshakeToken.CreateDeliveyToken(earlierToken) - : StreamHandshakeToken.CreateStartToken(earlierToken); - var failedCursor = Substitute.For(); - var cursorError = new InvalidOperationException("Injected cursor failure before the rewind."); - failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); - var recoveryCursor = Substitute.For(); - recoveryCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); - recoveryCursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, attemptedToken)); - var queueCache = Substitute.For(); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - var progress = new List(); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => progress.Add(call.ArgAt(0))); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, attemptedToken)])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, acknowledgedToken, DateTime.UtcNow); - var observer = new RecordingConsumer(); - if (!reattach) - { - observer.DeliveryResponses.Enqueue(requestedToken); - } - - observer.ReleaseDelivery(); - var subscriptionId = SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); - if (tokenKind == "implicit") - { - subscriptionId = SubscriptionMarker.MarkAsImplictSubscriptionId(subscriptionId); - } - - Assert.Equal(tokenKind == "implicit", SubscriptionMarker.IsImplicitSubscription(subscriptionId)); - - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(subscriptionId), streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - // Seed the preceding acknowledgement; the unresolved error is raised by the cursor below. - consumer.LastProcessedToken = unknownPosition ? null : acknowledgedToken; - consumer.LastToken = unknownPosition ? null : StreamHandshakeToken.CreateDeliveyToken(acknowledgedToken); - consumer.Cursor = failedCursor; - var recoveryIssued = false; - var earlierCursorRequested = false; - var acquisitionError = new InvalidOperationException("Injected cursor acquisition failure at the earlier position."); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - if (!recoveryIssued) - { - recoveryIssued = true; - return QueueCacheCursorResult.FromCursor( - reattach ? new EmptyQueueCacheCursor() : recoveryCursor); - } - - if (call.ArgAt(1)?.SequenceNumber == earlierToken.SequenceNumber) - { - earlierCursorRequested = true; - if (failCursorAcquisition) - { - throw acquisitionError; - } - } - - return QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor()); - }); - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - if (reattach) - { - observer.HandshakeResponse = requestedToken; - await agent.RunOrQueueTask(() => agent.AddSubscriber( - consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, - TestContext.Current.CancellationToken)); - } - - Assert.True(earlierCursorRequested); - failedCursor.Received(1).MoveNextWithResult(); - Assert.Contains(cursorError, observer.Errors); - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(consumer.IsCaughtUp); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - if (reattach) - { - Assert.Empty(observer.DeliveredTokens); - } - else - { - Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); - } - await accessor.Shutdown(); - - if (unknownPosition || tokenKind == "explicit" || reattach && tokenKind == "implicit") - { - Assert.Null(consumer.LastProcessedToken); - Assert.Empty(progress); - } - else - { - Assert.Equal(earlierToken, consumer.LastProcessedToken); - Assert.Equal(earlierToken, Assert.Single(progress)); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Shutdown_PreservesReplayFloorAfterAccountingFailure(bool failCacheAdd, bool includeUnreadTail) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var failedId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var tailId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var backingCache = new PurgeablePooledQueueCache(); - var queueCache = Substitute.For(); - var accountingError = new InvalidOperationException("Injected synchronous fetched-batch accounting failure."); - queueCache.When(cache => cache.AddToCache(Arg.Any>())).Do(call => - { - var batches = call.Arg>(); - if (failCacheAdd && batches.Any(batch => batch.StreamId.Equals(failedId.StreamId))) - { - throw accountingError; - } - - backingCache.AddToCache(batches); - }); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - var streamId = call.ArgAt(0); - if (!failCacheAdd && streamId.Equals(failedId.StreamId)) - { - throw accountingError; - } - - return backingCache.TryGetCacheCursor(streamId, call.ArgAt(1)); - }); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => backingCache.UpdateDeliveryProgress(call.ArgAt(0), call.Arg())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - var failedRead = new List - { - new TestBatchContainer(failedId.StreamId, new EventSequenceTokenV2(100)), - }; - if (includeUnreadTail) - { - failedRead.Add(new TestBatchContainer(tailId.StreamId, new EventSequenceTokenV2(150))); - } - - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1))]), - Task.FromResult>(failedRead), - Task.FromResult>([new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(idleId, new EventSequenceTokenV2(0), DateTime.UtcNow); - await accessor.RegisterStream(activeId, new EventSequenceTokenV2(0), DateTime.UtcNow); - var streams = await accessor.GetPubSubCache(); - var idleObserver = new RecordingConsumer(); - var activeObserver = new RecordingConsumer(); - var idleConsumer = AddConsumer(idleId, idleObserver); - var activeConsumer = AddConsumer(activeId, activeObserver); - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(idleConsumer.IsCaughtUp); - Assert.Equal(1, Assert.Single(idleObserver.DeliveredTokens).SequenceNumber); - if (failCacheAdd) - { - Assert.Same(accountingError, await Assert.ThrowsAsync( - () => accessor.ReadFromQueue(queueId, receiver, 1000))); - } - else - { - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - - streams = await accessor.GetPubSubCache(); - await Task.WhenAll(streams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - if (failCacheAdd) - { - Assert.False(streams.ContainsKey(failedId)); - Assert.False(streams.ContainsKey(tailId)); - } - else - { - Assert.True(streams.ContainsKey(failedId)); - Assert.Equal(includeUnreadTail, streams.ContainsKey(tailId)); - var pendingRegistration = streams[failedId].RegistrationTask; - Assert.NotNull(pendingRegistration); - Assert.True(pendingRegistration.IsCompleted); - Assert.False(streams[failedId].StreamRegistered); - Assert.Equal(100, Assert.IsType(streams[failedId].RegistrationStartToken).SequenceNumber); - } - - await pubSub.DidNotReceive().RegisterProducer(failedId, Arg.Any(), Arg.Any()); - Assert.Equal(failCacheAdd, await accessor.ReadFromQueue(queueId, receiver, 1000)); - if (failCacheAdd) - { - Assert.True(activeConsumer.IsCaughtUp); - Assert.Equal(200, Assert.Single(activeObserver.DeliveredTokens).SequenceNumber); - } - else - { - Assert.Empty(activeObserver.DeliveredTokens); - await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); - } - - await accessor.Shutdown(); - Assert.Empty(backingCache.DeliveryProgressTokens); - - StreamConsumerData AddConsumer(QualifiedStreamId streamId, RecordingConsumer observer) - { - observer.ReleaseDelivery(); - var consumer = streams[streamId].AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.LastProcessedToken = new EventSequenceTokenV2(0); - consumer.Cursor = backingCache.GetCacheCursor(streamId.StreamId, null); - return consumer; - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("delivery", "throw", false)] - [InlineData("delivery", "acquire-miss", false)] - [InlineData("delivery", "advance-miss", false)] - [InlineData("implicit", "throw", false)] - [InlineData("implicit", "acquire-miss", false)] - [InlineData("implicit", "advance-miss", false)] - [InlineData("explicit", "throw", false)] - [InlineData("explicit", "acquire-miss", false)] - [InlineData("delivery", "throw", true)] - [InlineData("delivery", "acquire-miss", true)] - [InlineData("delivery", "advance-miss", true)] - [InlineData("implicit", "throw", true)] - [InlineData("implicit", "acquire-miss", true)] - [InlineData("implicit", "advance-miss", true)] - [InlineData("explicit", "throw", true)] - [InlineData("explicit", "acquire-miss", true)] - [InlineData("delivery", "none", false)] - [InlineData("implicit", "none", false)] - [InlineData("explicit", "none", false)] - [InlineData("delivery", "none", true)] - [InlineData("implicit", "none", true)] - [InlineData("explicit", "none", true)] - public async Task Shutdown_PreservesReplayAfterHandshakeFallback(string tokenKind, string failureMode, bool reattach) - { - var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(_ => registration.Task); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var latestToken = new EventSequenceTokenV2(200); - var requestedPosition = new EventSequenceTokenV2(tokenKind == "explicit" ? 50 : 100); - var requestedToken = tokenKind == "delivery" - ? StreamHandshakeToken.CreateDeliveyToken(requestedPosition) - : StreamHandshakeToken.CreateStartToken(requestedPosition); - var latestBatch = new TestBatchContainer(streamId.StreamId, latestToken); - var acquisitionError = new InvalidOperationException("Injected handshake cursor acquisition failure."); - var missingCursor = Substitute.For(); - missingCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.FromCacheMiss(new("requested", "low", "high"))); - var queueCache = Substitute.For(); - var requestedCursorCount = 0; - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - if (call.ArgAt(1)?.SequenceNumber == requestedPosition.SequenceNumber) - { - requestedCursorCount++; - switch (failureMode) - { - case "throw": - throw acquisitionError; - case "acquire-miss": - return QueueCacheCursorResult.FromCacheMiss(new("requested", "low", "high")); - case "advance-miss": - return QueueCacheCursorResult.FromCursor(missingCursor); - } - } - - return QueueCacheCursorResult.FromCursor( - new ScriptedQueueCursor([latestBatch], streamId.StreamId, null)); - }); - var progress = new List(); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => progress.Add(call.ArgAt(0))); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>([latestBatch])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var registrationTask = stream.RegistrationTask; - Assert.NotNull(registrationTask); - Assert.False(registrationTask.IsCompleted); - var subscriptionId = tokenKind == "implicit" - ? SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid()) - : SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); - Assert.Equal(tokenKind == "implicit", SubscriptionMarker.IsImplicitSubscription(subscriptionId)); - var observer = new RecordingConsumer(reattach ? null : requestedToken); - observer.ReleaseDelivery(); - var consumer = stream.AddConsumer( - GuidId.GetGuidId(subscriptionId), streamId, observer, null, DateTime.UtcNow); - Assert.False(consumer.IsRegistered); - Assert.False(consumer.HasDeliveryProgressError); - - // The real initial-registration path finds this consumer instead of resolving a grain reference. - registration.SetResult(new HashSet - { - new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), - }); - await registrationTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - if (reattach) - { - Assert.True(consumer.IsCaughtUp); - Assert.False(consumer.HasDeliveryProgressError); - Assert.Equal(latestToken, consumer.LastProcessedToken); - observer.DeliveredTokens.Clear(); - observer.HandshakeResponse = requestedToken; - await agent.RunOrQueueTask(() => agent.AddSubscriber( - consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"), null, - TestContext.Current.CancellationToken)); - } - - Assert.True(stream.StreamRegistered); - Assert.True(consumer.IsRegistered); - Assert.Equal(0, consumer.PendingHandshakes); - Assert.True(requestedCursorCount >= 1); - if (reattach && tokenKind == "implicit" && failureMode == "advance-miss") - { - // A failed replacement yields rather than immediately opening another recovery cursor. - await accessor.RunConsumerCursor(consumer); - } - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.NotEmpty(observer.DeliveredTokens); - Assert.All(observer.DeliveredTokens, token => Assert.Same(latestToken, token)); - if (failureMode == "throw") - { - Assert.Contains(acquisitionError, observer.Errors); - } - else if (failureMode == "advance-miss") - { - missingCursor.Received().MoveNextWithResult(); - } - - await accessor.Shutdown(); - - if (failureMode == "none") - { - Assert.False(consumer.HasDeliveryProgressError); - Assert.Equal(latestToken, Assert.Single(progress)); - } - else - { - if (tokenKind == "delivery") - { - Assert.Equal(requestedPosition, Assert.Single(progress)); - Assert.Equal(requestedPosition, consumer.LastProcessedToken); - } - else - { - Assert.Empty(progress); - Assert.Null(consumer.LastProcessedToken); - } - - Assert.True(consumer.HasDeliveryProgressError); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false, false)] - [InlineData(false, true, false)] - [InlineData(true, false, false)] - [InlineData(true, true, false)] - [InlineData(false, false, true)] - [InlineData(false, true, true)] - [InlineData(true, false, true)] - [InlineData(true, true, true)] - public async Task Shutdown_PreservesReplayAfterUnorderedRead(bool separateReads, bool unknownFirst, bool nullContainer) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var knownId = StreamId.Create("namespace", Guid.NewGuid()); - var unknownId = StreamId.Create("namespace", Guid.NewGuid()); - var knownToken = new EventSequenceTokenV2(200); - IBatchContainer knownBatch = new TestBatchContainer(knownId, knownToken); - IBatchContainer unknownBatch = nullContainer ? null! : new TestBatchContainer(unknownId, null!); - var firstBatch = unknownFirst ? unknownBatch : knownBatch; - var secondBatch = unknownFirst ? knownBatch : unknownBatch; - var queueCache = new RecordingQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>(separateReads ? [firstBatch] : [firstBatch, secondBatch]), - Task.FromResult>([secondBatch])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - - await ReadAndFinishRegistration(); - if (separateReads) - { - await ReadAndFinishRegistration(); - } - - var streams = await accessor.GetPubSubCache(); - Assert.Equal(nullContainer ? 1 : 2, streams.Count); - Assert.All(streams.Values, stream => - { - Assert.True(stream.StreamRegistered); - Assert.Null(stream.RegistrationTask); - Assert.Equal(0, stream.Count); - }); - await accessor.Shutdown(); - - if (nullContainer) - { - Assert.Null(Assert.Single(queueCache.DeliveryProgressTokens)); - } - else - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - - async Task ReadAndFinishRegistration() - { - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var pendingStreams = await accessor.GetPubSubCache(); - await Task.WhenAll(pendingStreams.Values.Select(stream => stream.RegistrationTask ?? Task.CompletedTask)) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("exact", 1)] - [InlineData("exact", 2)] - [InlineData("fallback", 1)] - [InlineData("fallback", 2)] - [InlineData("empty", 1)] - [InlineData("empty", 2)] - public async Task Recovery_OnlyAcknowledgedExactReplayReleasesProgress(string replayKind, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100))]), - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(replayKind == "fallback" ? 500 : 200))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.Cursor = queueCache.GetCacheCursor(streamId.StreamId, null); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Equal(100, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); - Assert.False(consumer.HasDeliveryProgressError); - - var failedCursor = Substitute.For(); - var cursorError = new InvalidOperationException("Injected cursor failure after acknowledged position 100."); - if (replayKind == "fallback") - { - failedCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.FromCacheMiss(new("200", "500", "500"))); - } - else - { - failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); - } - consumer.Cursor = failedCursor; - if (replayKind != "exact") - { - queueCache.AddToCache([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]); - queueCache.Purge(); - } - - if (replayKind == "empty") - { - await accessor.RunConsumerCursor(consumer); - } - else - { - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - - if (replayKind == "fallback") - { - failedCursor.Received(1).MoveNextWithResult(); - Assert.Empty(observer.Errors); - } - else - { - Assert.Contains(cursorError, observer.Errors); - } - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - var caughtUp = consumer.IsCaughtUp; - await accessor.Shutdown(); - var expectedCheckpoint = replayKind == "exact" ? 200 : 100; - Assert.Equal(expectedCheckpoint, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); - Assert.Equal(replayKind != "exact", consumer.HasDeliveryProgressError); - Assert.Equal(replayKind == "exact", caughtUp); - Assert.Equal( - replayKind == "empty" ? [100L] : new[] { 100L, replayKind == "fallback" ? 500L : 200L }, - observer.DeliveredTokens.Select(token => token.SequenceNumber)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Recovery_ExplicitStartSurvivesFallbackAndIdleCleanup(bool cleanupBeforeRecovery, bool cleanupAfterRecovery) - { - var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(_ => registration.Task); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var timeProvider = new FakeTimeProvider(); - var backingCache = new PurgeablePooledQueueCache(); - backingCache.AddToCache( - [ - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(50)), - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), - ]); - var queueCache = Substitute.For(); - queueCache.GetMaxAddCount().Returns(1000); - queueCache.When(cache => cache.AddToCache(Arg.Any>())) - .Do(call => backingCache.AddToCache(call.Arg>())); - var failEarlierCursor = true; - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - var token = call.ArgAt(1); - if (failEarlierCursor && token?.SequenceNumber == 50) - { - return QueueCacheCursorResult.FromCacheMiss(new("50", "100", "500")); - } - - return backingCache.TryGetCacheCursor(call.ArgAt(0), token); - }); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => backingCache.UpdateDeliveryProgress(call.ArgAt(0), call.Arg())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(500))]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { MaxEventDeliveryTime = TimeSpan.FromDays(2) }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var registrationTask = stream.RegistrationTask; - Assert.NotNull(registrationTask); - var observer = new RecordingConsumer(StreamHandshakeToken.CreateStartToken(new EventSequenceTokenV2(50))); - observer.ReleaseDelivery(); - var consumer = stream.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - registration.SetResult(new HashSet - { - new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), - }); - await registrationTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Null(consumer.LastProcessedToken); - Assert.Equal(500, Assert.Single(observer.DeliveredTokens).SequenceNumber); - Assert.Equal(500, Assert.IsType(Assert.IsType(consumer.LastToken).Token).SequenceNumber); - if (cleanupBeforeRecovery) - { - timeProvider.Advance(TimeSpan.FromDays(1)); - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - Assert.True((await accessor.GetPubSubCache()).ContainsKey(streamId), "Unfulfilled inclusive replay must survive inactivity cleanup."); - } - - observer.DeliveredTokens.Clear(); - failEarlierCursor = false; - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - Assert.Equal(new[] { 50L, 100L, 500L }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); - Assert.False(consumer.HasDeliveryProgressError); - Assert.True(consumer.IsCaughtUp); - if (cleanupAfterRecovery) - { - timeProvider.Advance(TimeSpan.FromDays(1)); - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - Assert.Empty(await accessor.GetPubSubCache()); - } - - await accessor.Shutdown(); - Assert.Equal(500, Assert.IsType(Assert.Single(backingCache.DeliveryProgressTokens)).SequenceNumber); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false, 1)] - [InlineData(false, false, 2)] - [InlineData(false, true, 1)] - [InlineData(false, true, 2)] - [InlineData(true, false, 1)] - [InlineData(true, false, 2)] - [InlineData(true, true, 1)] - [InlineData(true, true, 2)] - public async Task Recovery_NewReadPreservesEmptyCursorReplayBoundary( - bool inclusiveReplay, bool replayAvailable, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var requiredToken = new EventSequenceTokenV2(100); - var latestToken = new EventSequenceTokenV2(500); - var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); - queueCache.AddToCache( - [ - new TestBatchContainer(streamId.StreamId, requiredToken), - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), - ]); - queueCache.Purge(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>(replayAvailable - ? - [ - new TestBatchContainer(streamId.StreamId, requiredToken), - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), - new TestBatchContainer(streamId.StreamId, latestToken), - ] - : [new TestBatchContainer(streamId.StreamId, latestToken)])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, requiredToken, DateTime.UtcNow); - var observer = new RecordingConsumer(inclusiveReplay - ? StreamHandshakeToken.CreateStartToken(requiredToken) - : StreamHandshakeToken.CreateDeliveyToken(requiredToken)); - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); - consumer.IsRegistered = true; - Assert.False(consumer.HasDeliveryProgressError); - - var failedCursor = Substitute.For(); - var cursorError = new InvalidOperationException("Injected cursor failure before an empty-cache recovery."); - failedCursor.MoveNextWithResult().Returns(_ => throw cursorError); - consumer.SafeDisposeCursor(NullLogger.Instance); - consumer.Cursor = failedCursor; - await accessor.RunConsumerCursor(consumer); - - Assert.Contains(cursorError, observer.Errors); - Assert.Empty(observer.DeliveredBatches); - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(consumer.IsCaughtUp); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - var recoveryCursor = consumer.Cursor; - Assert.NotNull(recoveryCursor); - Assert.Same(recoveryCursor, consumer.DeliveryRecoveryCursor); - var recoveryToken = consumer.DeliveryRecoveryToken; - Assert.Equal(requiredToken, Assert.IsAssignableFrom(recoveryToken).Token); - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Equal(!replayAvailable, consumer.HasDeliveryProgressError); - Assert.Equal(replayAvailable, consumer.IsCaughtUp); - // A missing range must invalidate the recovery cursor, not silently reposition it to the new read. - Assert.Null(consumer.DeliveryRecoveryCursor); - if (replayAvailable) - { - Assert.Null(consumer.DeliveryRecoveryToken); - Assert.Equal(latestToken, consumer.LastProcessedToken); - } - else - { - Assert.Same(recoveryToken, consumer.DeliveryRecoveryToken); - Assert.Equal(inclusiveReplay ? null : requiredToken, consumer.LastProcessedToken); - } - - var deliveredTokens = observer.DeliveredBatches - .SelectMany(batch => batch is BatchContainerBatch group ? group.BatchContainers.AsEnumerable() : [batch]) - .Select(batch => batch.SequenceToken.SequenceNumber); - Assert.Equal(replayAvailable ? new[] { 100L, 200L, 500L } : [500L], deliveredTokens); - await accessor.Shutdown(); - if (inclusiveReplay && !replayAvailable) - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - else - { - Assert.Equal(replayAvailable ? latestToken : requiredToken, Assert.Single(queueCache.DeliveryProgressTokens)); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false, 1)] - [InlineData(false, false, 2)] - [InlineData(false, true, 1)] - [InlineData(false, true, 2)] - [InlineData(true, false, 1)] - [InlineData(true, false, 2)] - [InlineData(true, true, 1)] - [InlineData(true, true, 2)] - public async Task Recovery_AfterContractFailureReacquiresDisposedCursor( - bool inclusiveReplay, bool replayAvailable, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var requiredToken = new EventSequenceTokenV2(100); - var latestToken = new EventSequenceTokenV2(200); - var queueCache = new PurgeablePooledQueueCache(retainPurgeMetadata: true); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>( - [ - new TestBatchContainer(streamId.StreamId, requiredToken), - new TestBatchContainer(streamId.StreamId, latestToken), - ]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, requiredToken, DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var observer = new RecordingConsumer(inclusiveReplay - ? StreamHandshakeToken.CreateStartToken(requiredToken) - : StreamHandshakeToken.CreateDeliveyToken(requiredToken)); - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - Assert.True(await accessor.DoHandshakeWithConsumer(consumer, cacheToken: null)); - consumer.IsRegistered = true; - consumer.SafeDisposeCursor(NullLogger.Instance); - var invalidCursor = new InvalidMoveCursor(); - consumer.Cursor = invalidCursor; - - await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); - - Assert.True(invalidCursor.IsDisposed); - Assert.Null(consumer.Cursor); - Assert.True(consumer.HasDeliveryProgressError); - var recoveryToken = consumer.DeliveryRecoveryToken; - Assert.Equal(requiredToken, Assert.IsAssignableFrom(recoveryToken).Token); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - consumer.PendingHandshakes++; - await accessor.RunConsumerCursor(consumer); - Assert.Null(consumer.Cursor); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - consumer.PendingHandshakes--; - if (!replayAvailable) - { - queueCache.Purge(); - } - - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - - Assert.NotNull(consumer.Cursor); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Equal(!replayAvailable, consumer.HasDeliveryProgressError); - Assert.Equal(replayAvailable, consumer.IsCaughtUp); - if (replayAvailable) - { - Assert.Null(consumer.DeliveryRecoveryToken); - Assert.Equal(latestToken, consumer.LastProcessedToken); - var deliveredTokens = observer.DeliveredBatches - .SelectMany(batch => batch is BatchContainerBatch group ? group.BatchContainers.AsEnumerable() : [batch]) - .Select(batch => batch.SequenceToken.SequenceNumber); - Assert.Equal(inclusiveReplay ? new[] { 100L, 200L } : [200L], deliveredTokens); - } - else - { - Assert.Same(recoveryToken, consumer.DeliveryRecoveryToken); - Assert.Same(consumer.Cursor, consumer.DeliveryRecoveryCursor); - Assert.Equal(inclusiveReplay ? null : requiredToken, consumer.LastProcessedToken); - Assert.Empty(observer.DeliveredBatches); - } - - await accessor.Shutdown(); - if (inclusiveReplay && !replayAvailable) - { - Assert.Empty(queueCache.DeliveryProgressTokens); - } - else - { - Assert.Equal(replayAvailable ? latestToken : requiredToken, Assert.Single(queueCache.DeliveryProgressTokens)); - } - - await accessor.RunConsumerCursor(consumer); - Assert.Null(consumer.Cursor); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("delivery", false, false, 1)] - [InlineData("delivery", false, true, 2)] - [InlineData("delivery", true, false, 2)] - [InlineData("delivery", true, true, 1)] - [InlineData("start", false, false, 2)] - [InlineData("start", false, true, 1)] - [InlineData("start", true, false, 1)] - [InlineData("start", true, true, 2)] - [InlineData("none", false, false, 1)] - [InlineData("none", false, false, 2)] - public async Task Recovery_StaleCallbackCannotAcknowledgeReattachedCursor( - string tokenKind, bool reuseCursor, bool lateRewind, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var attemptedToken = new EventSequenceTokenV2(200); - var earlierToken = new EventSequenceTokenV2(50); - var batch = new TestBatchContainer(streamId.StreamId, attemptedToken); - var oldCursor = Substitute.For(); - oldCursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); - oldCursor.GetCurrent(out _).Returns(batch); - IQueueCacheCursor reboundCursor = reuseCursor ? oldCursor : new EmptyQueueCacheCursor(); - var queueCache = Substitute.For(); - var failEarlierRequest = true; - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - if (failEarlierRequest && call.ArgAt(1)?.SequenceNumber == 50) - { - return QueueCacheCursorResult.FromCacheMiss(new("50", "100", "200")); - } - - return QueueCacheCursorResult.FromCursor(reboundCursor); - }); - var progress = new List(); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => progress.Add(call.ArgAt(0))); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>([batch])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var observer = new RecordingConsumer(); - if (lateRewind) - { - observer.DeliveryResponses.Enqueue(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(400))); - } - - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.LastProcessedToken = new EventSequenceTokenV2(100); - consumer.LastToken = StreamHandshakeToken.CreateDeliveyToken(consumer.LastProcessedToken); - consumer.Cursor = oldCursor; - var delivery = accessor.RunConsumerCursor(consumer); - try - { - await observer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.False(delivery.IsCompleted); - Assert.Equal(100, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); - Assert.False(consumer.HasDeliveryProgressError); - if (tokenKind != "none") - { - observer.HandshakeResponse = tokenKind == "start" - ? StreamHandshakeToken.CreateStartToken(earlierToken) - : StreamHandshakeToken.CreateDeliveyToken(earlierToken); - await Reattach(); - Assert.True(consumer.HasDeliveryProgressError); - - // Rebind an exact-but-empty recovery cursor while the old delivery is still blocked. - // Some providers recycle the same cursor object, so identity alone cannot identify the callback. - failEarlierRequest = false; - await Reattach(); - Assert.Same(reboundCursor, consumer.Cursor); - Assert.Same(reboundCursor, consumer.DeliveryRecoveryCursor); - Assert.Equal(reuseCursor, ReferenceEquals(oldCursor, consumer.Cursor)); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Equal(0, consumer.PendingHandshakes); - Assert.Same(observer.HandshakeResponse, consumer.LastToken); - } - - observer.ReleaseDelivery(); - await delivery.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Same(attemptedToken, Assert.Single(observer.DeliveredTokens)); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - if (tokenKind == "none") - { - Assert.False(consumer.HasDeliveryProgressError); - Assert.True(consumer.IsCaughtUp); - Assert.Null(consumer.UnconfirmedDeliveryToken); - Assert.Equal(200, Assert.IsType(consumer.LastProcessedToken).SequenceNumber); - Assert.Equal(200, Assert.IsType( - Assert.IsType(consumer.LastToken).Token).SequenceNumber); - } - else - { - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(consumer.IsCaughtUp); - Assert.Same(observer.HandshakeResponse, consumer.LastToken); - Assert.Equal(earlierToken, consumer.DeliveryRecoveryToken?.Token); - Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); - Assert.Equal(tokenKind == "start" ? null : earlierToken, consumer.LastProcessedToken); - } - - await accessor.Shutdown(); - if (tokenKind == "start") - { - Assert.Empty(progress); - } - else - { - Assert.Equal(tokenKind == "none" ? 200 : 50, - Assert.IsType(Assert.Single(progress)).SequenceNumber); - } - } - finally - { - observer.ReleaseDelivery(); - } - - Task Reattach() => agent.RunOrQueueTask(() => agent.AddSubscriber( - consumer.SubscriptionId, streamId, GrainId.Create("test", "reattached-subscriber"), null, - TestContext.Current.CancellationToken)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Recovery_RetriesCanceledRegistrationFromOriginalCachedPosition() - { - var firstRegistration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var subscriptionId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); - var subscriptions = new HashSet - { - new(subscriptionId, streamId, GrainId.Create("test", "existing-subscriber")), - }; - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(_ => firstRegistration.Task, _ => Task.FromResult>(subscriptions)); - var backoff = Substitute.For(); - // Cancel the first attempt's retry wait, without stopping the agent. - var cancelFirstWait = true; - backoff.Next(Arg.Any()).Returns(_ => - { - if (cancelFirstWait) - { - cancelFirstWait = false; - throw new OperationCanceledException("Transient registration retry cancellation."); - } - - return TimeSpan.Zero; - }); - var queueCache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>( - [ - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), - ]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, deliveryBackoff: backoff); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var firstTask = stream.RegistrationTask; - Assert.NotNull(firstTask); - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - stream.AddConsumer(subscriptionId, streamId, observer, null, DateTime.UtcNow); - firstRegistration.SetException(new InvalidOperationException("Transient pubsub registration failure.")); - await firstTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - var streams = await accessor.GetPubSubCache(); - await Task.WhenAll(streams.Values.Select(value => value.RegistrationTask ?? Task.CompletedTask)) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await pubSub.Received(2).RegisterProducer(streamId, Arg.Any(), Arg.Any()); - Assert.Equal(new[] { 100L, 200L }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); - Assert.Equal(1, observer.HandshakeCount); - await accessor.Shutdown(); - Assert.Equal(200, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Recovery_RetriesOnlyFailedInitialAttachmentsWithoutNewMessages() - { - var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken).ReturnsForAnyArgs(_ => registration.Task); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var failedId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); - var healthyId = GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())); - var queueCache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>( - [ - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100)), - new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200)), - ]), - Task.FromResult>([])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var firstTask = stream.RegistrationTask; - Assert.NotNull(firstTask); - var healthy = new RecordingConsumer(); - healthy.ReleaseDelivery(); - stream.AddConsumer(healthyId, streamId, healthy, null, DateTime.UtcNow); - registration.SetResult(new HashSet - { - new(failedId, streamId, GrainId.Create("test", "initially-unavailable-subscriber")), - new(healthyId, streamId, GrainId.Create("test", "healthy-subscriber")), - }); - await firstTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(1, healthy.HandshakeCount); - Assert.Equal(new[] { 100L, 200L }, healthy.DeliveredTokens.Select(token => token.SequenceNumber)); - // The first attachment failed resolving its reference. Make it available for the retry. - var recovered = new RecordingConsumer(); - recovered.ReleaseDelivery(); - stream.AddConsumer(failedId, streamId, recovered, null, DateTime.UtcNow); - await accessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); - var streams = await accessor.GetPubSubCache(); - await Task.WhenAll(streams.Values.Select(value => value.RegistrationTask ?? Task.CompletedTask)) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - Assert.Equal(1, recovered.HandshakeCount); - Assert.Equal(new[] { 100L, 200L }, recovered.DeliveredTokens.Select(token => token.SequenceNumber)); - Assert.Equal(1, healthy.HandshakeCount); - Assert.Equal(new[] { 100L, 200L }, healthy.DeliveredTokens.Select(token => token.SequenceNumber)); - await accessor.Shutdown(); - Assert.Equal(200, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Recovery_RemovingConsumerReleasesOnlyItsReplayConstraint(bool faultFirstConsumer, bool removeSecondConsumer) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new RecordingQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var firstObserver = new RecordingConsumer(); - firstObserver.ReleaseDelivery(); - var first = AddConsumer(firstObserver, 100); - var second = AddConsumer(new RecordingConsumer(), 150); - await accessor.RunConsumerCursor(first); - await accessor.RunConsumerCursor(second); - Assert.True(first.HasDeliveryProgressError); - Assert.True(second.HasDeliveryProgressError); - if (faultFirstConsumer) - { - firstObserver.HandshakeResponse = new UnknownHandshakeToken(); - await agent.RunOrQueueTask(() => agent.AddSubscriber( - first.SubscriptionId, streamId, GrainId.Create("test", "faulted-subscriber"), null, - TestContext.Current.CancellationToken)); - await pubSub.Received(1).FaultSubscription(streamId, first.SubscriptionId, Arg.Any()); - } - else - { - await agent.RunOrQueueTask(() => agent.RemoveSubscriber( - first.SubscriptionId, streamId, TestContext.Current.CancellationToken)); - } - - Assert.False(stream.Contains(first.SubscriptionId)); - Assert.True(stream.Contains(second.SubscriptionId)); - Assert.True(second.HasDeliveryProgressError); - if (removeSecondConsumer) - { - await agent.RunOrQueueTask(() => agent.RemoveSubscriber( - second.SubscriptionId, streamId, TestContext.Current.CancellationToken)); - } - - await accessor.Shutdown(); - Assert.Equal(150, Assert.IsType(Assert.Single(queueCache.DeliveryProgressTokens)).SequenceNumber); - - StreamConsumerData AddConsumer(RecordingConsumer observer, long safePosition) - { - var consumer = stream.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.LastProcessedToken = new EventSequenceTokenV2(safePosition); - consumer.LastToken = StreamHandshakeToken.CreateDeliveyToken(consumer.LastProcessedToken); - var cursor = Substitute.For(); - cursor.MoveNextWithResult().Returns(_ => throw new InvalidOperationException("Injected unresolved cursor failure.")); - consumer.Cursor = cursor; - return consumer; - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(1)] - [InlineData(16)] - [InlineData(256)] - public async Task Removal_RetainsProgressWithLinearTokenComparisons(int subscriberCount) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new RecordingQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var comparisons = 0; - Action onComparison = () => comparisons++; - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>( - [new TestBatchContainer(streamId.StreamId, new CountingEventSequenceToken(subscriberCount + 1, onComparison))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var consumers = new List(); - for (var i = 0; i < subscriberCount; i++) - { - var consumer = stream.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, new RecordingConsumer(), null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.LastProcessedToken = new CountingEventSequenceToken(i + 1, onComparison); - consumers.Add(consumer); - } - - comparisons = 0; - await agent.RunOrQueueTask(() => - { - foreach (var consumer in consumers) - { - agent.RemoveSubscriber_Impl(consumer.SubscriptionId, streamId); - } - return Task.CompletedTask; - }); - - Assert.Equal(2 * subscriberCount - 1, comparisons); - Assert.Empty(await accessor.GetPubSubCache()); - await accessor.Shutdown(); - Assert.Equal(subscriberCount, Assert.Single(queueCache.DeliveryProgressTokens)!.SequenceNumber); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false)] - [InlineData(true)] - public async Task Shutdown_DoesNotReusePreviousReceiverReadBoundaryAfterReinitialize(bool readAfterReinitialize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new ScriptedQueueCache(); - var nextQueueCache = new ScriptedQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache, nextQueueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]), - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(10))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), DateTime.UtcNow); - var consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, queueCache); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(consumer.IsCaughtUp); - Assert.Equal(200, consumer.LastProcessedToken!.SequenceNumber); - await agent.RunOrQueueTask(() => agent.RemoveSubscriber( - consumer.SubscriptionId, streamId, TestContext.Current.CancellationToken)); - await accessor.Shutdown(); - - await InitializeAgent(agent); - if (readAfterReinitialize) - { - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); - consumer = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, nextQueueCache); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(consumer.IsCaughtUp); - Assert.Equal(10, consumer.LastProcessedToken!.SequenceNumber); - } - - await accessor.Shutdown(); - - Assert.Equal( - new long?[] { 200, readAfterReinitialize ? 10 : null }, - queueCache.DeliveryProgressTokens.Concat(nextQueueCache.DeliveryProgressTokens).Select(token => token?.SequenceNumber)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("none")] - [InlineData("ordered")] - [InlineData("unknown")] - public async Task Recovery_AfterReinitializeTracksOnlyCurrentReceiverReads(string nextRead) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueCache = new RecordingQueueCache(); - var nextQueueCache = new ScriptedQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache, nextQueueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, null!)]), - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]), - Task.FromResult>([new TestBatchContainer(streamId.StreamId, - nextRead == "unknown" ? null! : new EventSequenceTokenV2(10))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - await accessor.Shutdown(); - - // A later ordered read cannot repair the unknown position within the same receiver lifetime. - Assert.Empty(queueCache.DeliveryProgressTokens); - await InitializeAgent(agent); - if (nextRead != "none") - { - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(10), DateTime.UtcNow); - if (nextRead == "ordered") - { - AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, nextQueueCache); - } - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - - await accessor.Shutdown(); - - adapterCache.Received(2).CreateQueueCache(queueId); - if (nextRead == "ordered") - { - Assert.Equal(10, Assert.IsType(Assert.Single(nextQueueCache.DeliveryProgressTokens)).SequenceNumber); - } - else if (nextRead == "unknown") - { - Assert.Empty(nextQueueCache.DeliveryProgressTokens); - } - else - { - Assert.Null(Assert.Single(nextQueueCache.DeliveryProgressTokens)); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Recovery_StopsRetryingPersistentContractFailures(bool implicitSubscription, bool faultOnError) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var timeProvider = new FakeTimeProvider(); - var requiredToken = new EventSequenceTokenV2(100); - var latestToken = new EventSequenceTokenV2(200); - var queueCache = Substitute.For(); - queueCache.GetMaxAddCount().Returns(1000); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - var progress = new List(); - queueCache.When(cache => cache.UpdateDeliveryProgress(Arg.Any(), Arg.Any())) - .Do(call => progress.Add(call.ArgAt(0))); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>([new TestBatchContainer(streamId.StreamId, latestToken)])); - var failureHandler = Substitute.For(); - failureHandler.ShouldFaultSubsriptionOnError.Returns(faultOnError); - var stopped = false; - failureHandler.OnDeliveryFailure(default!, default!, default, default).ReturnsForAnyArgs(_ => - { - stopped = true; - return Task.CompletedTask; - }); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(_ => stopped ? TimeSpan.Zero : TimeSpan.FromSeconds(1)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - deliveryBackoff: backoff, failureHandler: failureHandler); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, requiredToken, timeProvider.GetUtcNow().UtcDateTime); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - observer.OnDelivery = () => observer.DeliveryException = observer.DeliveredTokens.Count == 1 - ? new InvalidOperationException("Transient failure delivering fresh traffic.") - : null; - var subscriptionId = implicitSubscription - ? SubscriptionMarker.MarkAsImplictSubscriptionId(Guid.NewGuid()) - : SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(subscriptionId), streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - consumer.IsRegistered = true; - consumer.LastProcessedToken = requiredToken; - consumer.Cursor = new InvalidMoveCursor(); - await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); - var cursors = new List(); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - if (!Equals(call.ArgAt(1), requiredToken)) - { - return QueueCacheCursorResult.FromCursor( - new ScriptedQueueCursor([new TestBatchContainer(streamId.StreamId, latestToken)], streamId.StreamId, null)); - } - - var cursor = new InvalidMoveCursor(); - cursors.Add(cursor); - return QueueCacheCursorResult.FromCursor(cursor); - }); - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Single(cursors); - for (var i = 0; i < 20; i++) - { - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Single(cursors); - - for (var i = 0; i < 10; i++) - { - timeProvider.Advance(TimeSpan.FromSeconds(1)); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - - Assert.Equal(6, cursors.Count); - Assert.All(cursors, cursor => Assert.True(cursor.IsDisposed)); - Assert.Null(consumer.PendingBatch); - var shouldFault = faultOnError && !implicitSubscription; - await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, null); - Assert.True(consumer.DeliveryRecovery!.Stopped); - await pubSub.Received(shouldFault ? 1 : 0).FaultSubscription(streamId, consumer.SubscriptionId, Arg.Any()); - for (var i = 0; i < 20; i++) - { - timeProvider.Advance(TimeSpan.FromSeconds(1)); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Equal(6, cursors.Count); - if (shouldFault) - { - Assert.Empty(observer.DeliveredTokens); - } - else - { - Assert.Equal(new[] { latestToken, latestToken }, observer.DeliveredTokens); - } - await accessor.Shutdown(); - Assert.Equal(requiredToken, Assert.Single(progress)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("contract", -1, false, false)] - [InlineData("acquisition", -1, false, false)] - [InlineData("cache-miss", -1, false, false)] - [InlineData("empty", -1, false, false)] - [InlineData("contract", 2, false, false)] - [InlineData("empty", 2, false, false)] - [InlineData("contract", -1, true, false)] - [InlineData("empty", -1, false, true)] - public async Task Recovery_BoundsIdleRetriesAndReclaimsInactiveState( - string failure, int maximumSeconds, bool holdFailureNotification, bool cancelBackoff) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var timeProvider = new FakeTimeProvider(); - var requiredToken = new EventSequenceTokenV2(100); - var latestToken = new EventSequenceTokenV2(200); - var cache = Substitute.For(); - var cachedBatches = new List(); - cache.When(value => value.AddToCache(Arg.Any>())) - .Do(call => cachedBatches.AddRange(call.Arg>())); - cache.GetMaxAddCount().Returns(1000); - cache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - var freshTraffic = true; - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(_ => Task.FromResult>(freshTraffic - ? [new TestBatchContainer(streamId.StreamId, latestToken)] - : [])); - var failureHandler = Substitute.For(); - var failureNotification = new TaskCompletionSource(); - var reportingStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - failureHandler.OnDeliveryFailure(default!, default!, default, default) - .ReturnsForAnyArgs(_ => - { - reportingStarted.SetResult(); - return holdFailureNotification ? failureNotification.Task : Task.CompletedTask; - }); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(_ => cancelBackoff - ? throw new OperationCanceledException("Injected cancellation calculating consumer backoff.") - : TimeSpan.FromSeconds(1)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions - { - MaxEventDeliveryTime = maximumSeconds < 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromSeconds(maximumSeconds), - }, - deliveryBackoff: backoff, failureHandler: failureHandler); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, requiredToken, timeProvider.GetUtcNow().UtcDateTime); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - freshTraffic = false; - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - consumer.IsRegistered = true; - consumer.PendingStartToken = requiredToken; - consumer.Cursor = new InvalidMoveCursor(); - await Assert.ThrowsAnyAsync(() => accessor.RunConsumerCursor(consumer)); - var acquisitions = 0; - var cursors = new List(); - cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(call => - { - if (consumer.DeliveryRecovery is { Stopped: true }) - { - var start = call.ArgAt(1)!; - return QueueCacheCursorResult.FromCursor(new ScriptedQueueCursor( - cachedBatches.Where(batch => batch.SequenceToken.SequenceNumber >= start.SequenceNumber).ToList(), - streamId.StreamId, null)); - } - Assert.Equal(requiredToken, call.ArgAt(1)); - acquisitions++; - if (failure == "acquisition") throw new InvalidOperationException("Injected acquisition failure."); - var cursor = Substitute.For(); - cursor.MoveNextWithResult().Returns(failure switch - { - "cache-miss" => QueueCacheCursorMoveResult.FromCacheMiss(new("100", "200", "200")), - "empty" => QueueCacheCursorMoveResult.NoData, - _ => default, - }); - cursors.Add(cursor); - return QueueCacheCursorResult.FromCursor(cursor); - }); - - var expectedAttempts = maximumSeconds < 0 ? 6 : maximumSeconds; - for (var i = 0; i < expectedAttempts - 1; i++) - { - if (cancelBackoff) - { - await accessor.RunConsumerCursor(consumer); - } - else - { - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Equal(i + 1, consumer.DeliveryRecovery!.Attempts); - for (var j = 0; !cancelBackoff && j < 10; j++) - { - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Equal(i + 1, consumer.DeliveryRecovery.Attempts); - timeProvider.Advance(TimeSpan.FromSeconds(1)); - } - - var finalAttempt = accessor.RunConsumerCursor(consumer); - if (maximumSeconds >= 0) - { - if (failure == "contract") - { - await Assert.ThrowsAnyAsync(() => finalAttempt); - } - else - { - await finalAttempt; - } - timeProvider.Advance(TimeSpan.FromSeconds(1)); - finalAttempt = accessor.RunConsumerCursor(consumer); - } - await reportingStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.True(consumer.DeliveryRecovery!.Stopped); - Assert.Null(consumer.Cursor); - Assert.Null(consumer.PendingBatch); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Null(consumer.LastProcessedToken); - Assert.All(cursors, cursor => cursor.Received(1).Dispose()); - Assert.Equal(expectedAttempts, consumer.DeliveryRecovery.Attempts); - Assert.Equal(failure == "empty" ? 1 : expectedAttempts, acquisitions); - if (holdFailureNotification) - { - Assert.False(finalAttempt.IsCompleted); - Assert.Equal(StreamConsumerDataState.Active, consumer.State); - for (var i = 0; i < 10; i++) - { - await accessor.RunConsumerCursor(consumer); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Equal(expectedAttempts, acquisitions); - freshTraffic = true; - latestToken = new EventSequenceTokenV2(300); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - latestToken = new EventSequenceTokenV2(400); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - freshTraffic = false; - Assert.Equal(300, consumer.PendingContinuationToken!.SequenceNumber); - failureNotification.SetResult(); - } - if (failure is "contract" or "acquisition" && maximumSeconds < 0) - { - await Assert.ThrowsAnyAsync(() => finalAttempt); - } - else - { - await finalAttempt; - } - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - if (holdFailureNotification) - { - Assert.Equal(new long[] { 300, 400 }, observer.DeliveredTokens.Select(token => token.SequenceNumber)); - } - await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, null); - - timeProvider.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); - await accessor.Shutdown(); - cache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, false)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Recovery_BoundsOuterRegistrationRetriesAndReleasesPins(bool failAttachment, bool cancelBackoff) - { - var registration = new TaskCompletionSource>(); - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(registration.Task); - var timeProvider = new FakeTimeProvider(); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var requiredToken = new EventSequenceTokenV2(100); - var pin = Substitute.For(); - var cache = Substitute.For(); - cache.GetMaxAddCount().Returns(1000); - var acquisitions = 0; - cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(_ => - { - acquisitions++; - if (failAttachment && acquisitions == 1) - { - return QueueCacheCursorResult.FromCursor(pin); - } - throw new InvalidOperationException("Injected registration cursor failure."); - }); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, requiredToken)]), - Task.FromResult>([])); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(_ => cancelBackoff - ? throw new OperationCanceledException("Injected cancellation calculating registration backoff.") - : TimeSpan.FromSeconds(1)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { MaxEventDeliveryTime = Timeout.InfiniteTimeSpan }, - deliveryBackoff: backoff); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - var firstAttempt = stream.RegistrationTask!; - var subscriptions = new HashSet(); - var observer = new RecordingConsumer(); - StreamConsumerData? consumer = null; - if (failAttachment) - { - consumer = stream.AddConsumer(GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - subscriptions.Add(new(consumer.SubscriptionId, streamId, GrainId.Create("test", "existing-subscriber"))); - } - registration.SetResult(subscriptions); - await firstAttempt; - for (var attempt = 1; attempt < 6; attempt++) - { - Assert.Equal(attempt, stream.RegistrationRecovery!.Attempts); - var before = acquisitions; - for (var read = 0; !cancelBackoff && read < 10; read++) - { - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - } - Assert.Equal(before, acquisitions); - if (failAttachment) - { - Assert.Same(pin, stream.RegistrationCursor); - pin.DidNotReceive().Dispose(); - } - timeProvider.Advance(TimeSpan.FromSeconds(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - await stream.RegistrationTask!; - } - - Assert.Equal(6, stream.RegistrationRecovery!.Attempts); - Task? handshake = null; - var handshakeResponse = new TaskCompletionSource(); - if (consumer is not null) - { - var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - observer.OnHandshake = () => handshakeStarted.SetResult(); - observer.HandshakeTask = handshakeResponse.Task; - handshake = accessor.DoHandshakeWithConsumer(consumer, requiredToken); - await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - } - var finalAcquisitions = acquisitions; - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(stream.RegistrationRecovery.Stopped); - Assert.Empty(await accessor.GetPubSubCache()); - Assert.Null(stream.RegistrationCursor); - Assert.Equal(0, stream.Count); - if (failAttachment) pin.Received(1).Dispose(); - if (handshake is not null) - { - cache.TryGetCacheCursor(Arg.Any(), Arg.Any()).Returns(_ => - { - acquisitions++; - return QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor()); - }); - handshakeResponse.SetResult(StreamHandshakeToken.CreateStartToken(requiredToken)); - Assert.False(await handshake); - Assert.True(consumer!.IsRemoved); - Assert.Null(consumer.Cursor); - await accessor.RunConsumerCursor(consumer); - Assert.Empty(observer.DeliveredTokens); - } - timeProvider.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Equal(finalAcquisitions, acquisitions); - await accessor.Shutdown(); - cache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false)] - [InlineData(true)] - public void Recovery_BackoffCancellationDoesNotResetOrEscapeTheBudget(bool previousDelay) - { - var clock = new FakeTimeProvider(); - var recovery = new StreamRecoveryState(clock.GetTimestamp()); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(call => previousDelay && call.Arg() == 0 - ? TimeSpan.FromSeconds(10) - : throw new OperationCanceledException("Injected backoff cancellation.")); - for (var attempt = 0; attempt < 6; attempt++) - { - Assert.False(recovery.IsExhausted(clock, TimeSpan.FromMinutes(1), 6)); - Assert.True(recovery.TryBeginAttempt(clock)); - recovery.FinishAttempt(clock, backoff, NullLogger.Instance); - Assert.Equal(attempt + 1, recovery.Attempts); - backoff.Received(1).Next(attempt); - clock.Advance(TimeSpan.FromSeconds(previousDelay && attempt == 0 ? 10 : 1)); - } - Assert.True(recovery.IsExhausted(clock, Timeout.InfiniteTimeSpan, 6)); - Assert.Equal(TimeSpan.FromSeconds(previousDelay ? 45 : 54), recovery.RemainingTime(clock, TimeSpan.FromMinutes(1))); - Assert.True(recovery.IsExhausted(clock, TimeSpan.FromSeconds(6), 100)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public void Recovery_BackoffDoesNotSuppressOtherProviderErrors() - { - var clock = new FakeTimeProvider(); - var recovery = new StreamRecoveryState(clock.GetTimestamp()); - var backoff = Substitute.For(); - var failure = new InvalidOperationException("Injected provider error."); - backoff.Next(Arg.Any()).Returns(_ => throw failure); - Assert.True(recovery.TryBeginAttempt(clock)); - Assert.Same(failure, Assert.Throws(() => recovery.FinishAttempt(clock, backoff, NullLogger.Instance))); - Assert.Equal(1, recovery.Attempts); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("none", false)] - [InlineData("none", true)] - [InlineData("drained", false)] - [InlineData("drained", true)] - [InlineData("removed", false)] - [InlineData("removed", true)] - [InlineData("expired", false)] - [InlineData("expired", true)] - public async Task Shutdown_DistinguishesNoSubscriptionsFromDrainedSubscriptions(string subscriptionState, bool readMore) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var timeProvider = new FakeTimeProvider(); - var readToken = new EventSequenceTokenV2(200); - var cache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - IList messages = [new TestBatchContainer(streamId.StreamId, readToken)]; - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(_ => Task.FromResult(messages)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, readToken, timeProvider.GetUtcNow().UtcDateTime); - var observer = new RecordingConsumer(); - observer.ReleaseDelivery(); - StreamConsumerData? consumer = null; - if (subscriptionState != "none") - { - consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - consumer.IsRegistered = true; - consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, null); - } - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - messages = []; - if (consumer is not null) - { - Assert.True(consumer.IsCaughtUp); - Assert.Equal(readToken, Assert.Single(observer.DeliveredTokens)); - } - else - { - Assert.Empty(observer.DeliveredTokens); - } - if (subscriptionState == "removed") - { - await agent.RunOrQueueTask(() => - { - agent.RemoveSubscriber_Impl(consumer!.SubscriptionId, streamId); - return Task.CompletedTask; - }); - } - else if (subscriptionState == "expired") - { - timeProvider.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); - } - if (readMore) - { - var nextToken = new EventSequenceTokenV2(400); - if (!(await accessor.GetPubSubCache()).ContainsKey(streamId)) - { - await accessor.RegisterStream(streamId, nextToken, timeProvider.GetUtcNow().UtcDateTime); - } - messages = [new TestBatchContainer(streamId.StreamId, nextToken)]; - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Equal(subscriptionState == "none" ? 0 : subscriptionState == "drained" ? 2 : 1, observer.DeliveredTokens.Count); - } - await accessor.Shutdown(); - var expected = subscriptionState == "none" ? (long?)null : readMore && subscriptionState == "drained" ? 400 : 200; - Assert.Equal(expected, Assert.Single(cache.DeliveryProgressTokens)?.SequenceNumber); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("registration", false)] - [InlineData("handshake", false)] - [InlineData("known-replay", false)] - [InlineData("known-replay", true)] - [InlineData("unknown-replay", false)] - [InlineData("unknown-replay", true)] - [InlineData("drained", false)] - public async Task Shutdown_CompletedProgressStillHonorsCurrentReplayRequirements(string requirement, bool expireAgain) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var clock = new FakeTimeProvider(); - var cache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - IList messages = [new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))]; - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => Task.FromResult(messages)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, clock); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), clock.GetUtcNow().UtcDateTime); - var original = AddRegisteredConsumer((await accessor.GetPubSubCache()).Single().Value, streamId, cache, clock.GetUtcNow().UtcDateTime); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(original.IsCaughtUp); - Assert.Equal(200, original.LastProcessedToken!.SequenceNumber); - messages = []; - clock.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); - - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(400), clock.GetUtcNow().UtcDateTime); - var stream = (await accessor.GetPubSubCache()).Single().Value; - if (requirement == "drained") - { - var current = AddRegisteredConsumer(stream, streamId, cache, clock.GetUtcNow().UtcDateTime); - messages = [new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(400))]; - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(current.IsCaughtUp); - Assert.Equal(400, current.LastProcessedToken!.SequenceNumber); - } - else if (requirement == "registration") - { - stream.RegistrationTask = Task.CompletedTask; - } - else - { - var current = stream.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, new RecordingConsumer(), null, clock.GetUtcNow().UtcDateTime); - current.IsRegistered = true; - current.LastProcessedToken = requirement == "unknown-replay" ? null : new EventSequenceTokenV2(100); - current.PendingHandshakes = requirement == "handshake" ? 1 : 0; - current.HasDeliveryProgressError = requirement != "handshake"; - current.DeliveryRecovery = new StreamRecoveryState(clock.GetTimestamp()) { Stopped = true }; - } - if (expireAgain) - { - clock.Advance(TimeSpan.FromDays(1)); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Empty(await accessor.GetPubSubCache()); - } - await accessor.Shutdown(); - if (requirement is "known-replay" or "drained") - { - Assert.Equal(requirement == "drained" ? 400 : 100, Assert.Single(cache.DeliveryProgressTokens)!.SequenceNumber); - } - else - { - Assert.Empty(cache.DeliveryProgressTokens); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_RemovalDoesNotPreserveAnUnacknowledgedReadFromAnotherStream() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var idleId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var activeId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var cache = new PurgeablePooledQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - IList messages = - [ - new TestBatchContainer(idleId.StreamId, new EventSequenceTokenV2(1)), - new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(200)), - ]; - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns(_ => Task.FromResult(messages)); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(idleId, new EventSequenceTokenV2(1), DateTime.UtcNow); - await accessor.RegisterStream(activeId, new EventSequenceTokenV2(200), DateTime.UtcNow); - var streams = await accessor.GetPubSubCache(); - var idle = AddRegisteredConsumer(streams[idleId], idleId, cache); - var active = AddRegisteredConsumer(streams[activeId], activeId, cache); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(idle.IsCaughtUp); - Assert.True(active.IsCaughtUp); - Assert.Equal(200, active.LastProcessedToken!.SequenceNumber); - - var pending = new RecordingConsumer(); - active.StreamConsumer = pending; - active.PendingHandshakes = 1; - messages = [new TestBatchContainer(activeId.StreamId, new EventSequenceTokenV2(400))]; - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - active.PendingHandshakes = 0; - var delivery = accessor.RunConsumerCursor(active); - try - { - await pending.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(StreamConsumerDataState.Active, active.State); - Assert.Equal(400, active.UnconfirmedDeliveryToken!.SequenceNumber); - await agent.RunOrQueueTask(() => agent.RemoveSubscriber( - idle.SubscriptionId, idleId, TestContext.Current.CancellationToken)); - await agent.RunOrQueueTask(() => agent.RemoveSubscriber( - active.SubscriptionId, activeId, TestContext.Current.CancellationToken)); - pending.ReleaseDelivery(); - await delivery.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(200, active.LastProcessedToken.SequenceNumber); - Assert.Empty(await accessor.GetPubSubCache()); - await accessor.Shutdown(); - Assert.Equal(200, Assert.Single(cache.DeliveryProgressTokens)!.SequenceNumber); - } - finally - { - pending.ReleaseDelivery(); - } - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Recovery_DoesNotRestartAnExpiredDeliveryWindow() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var timeProvider = new FakeTimeProvider(); - var processed = new EventSequenceTokenV2(90); - var failed = new EventSequenceTokenV2(100); - var cache = new ScriptedQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, failed)]), - Task.FromResult>([])); - var backoff = Substitute.For(); - backoff.Next(Arg.Any()).Returns(_ => throw new OperationCanceledException("End the initial callback's retry wait.")); - var failureHandler = Substitute.For(); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { MaxEventDeliveryTime = TimeSpan.FromSeconds(1) }, - deliveryBackoff: backoff, failureHandler: failureHandler); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, failed, timeProvider.GetUtcNow().UtcDateTime); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var observer = new RecordingConsumer - { - OnDelivery = () => timeProvider.Advance(TimeSpan.FromSeconds(2)), - DeliveryException = new InvalidOperationException("Injected delivery failure."), - }; - observer.ReleaseDelivery(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, timeProvider.GetUtcNow().UtcDateTime); - consumer.IsRegistered = true; - consumer.LastProcessedToken = processed; - consumer.Cursor = cache.GetCacheCursor(streamId.StreamId, processed); - await accessor.RunConsumerCursor(consumer); - Assert.True(consumer.HasDeliveryProgressError); - Assert.False(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.True(consumer.DeliveryRecovery!.Stopped); - Assert.Equal(0, consumer.DeliveryRecovery.Attempts); - Assert.Null(consumer.Cursor); - Assert.Null(consumer.PendingBatch); - Assert.Equal(failed, Assert.Single(observer.DeliveredTokens)); - await failureHandler.Received(1).OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, failed); - await failureHandler.DidNotReceive().OnDeliveryFailure(consumer.SubscriptionId, "provider", streamId.StreamId, processed); - await accessor.Shutdown(); - Assert.Equal(processed, Assert.Single(cache.DeliveryProgressTokens)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Recovery_YieldsWhenTheReplacementCursorAlsoMisses() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var cache = Substitute.For(); - cache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(new EmptyQueueCacheCursor())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(cache); - var agent = CreateAgent(pubSub, queueId, queueAdapterCache: adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(100), DateTime.UtcNow); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, new RecordingConsumer(), null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.LastProcessedToken = new EventSequenceTokenV2(100); - var moves = 0; - var cursors = new List(); - IQueueCacheCursor CreateMissingCursor() - { - if (cursors.Count > 10) throw new InvalidOperationException("Recovery did not yield."); - var cursor = Substitute.For(); - cursor.MoveNextWithResult().Returns(_ => - { - moves++; - return QueueCacheCursorMoveResult.FromCacheMiss(new("100", "200", "200")); - }); - cursors.Add(cursor); - return cursor; - } - cache.TryGetCacheCursorAtPosition(streamId.StreamId, StreamSubscriptionStartPosition.EarliestAvailable) - .Returns(_ => QueueCacheCursorResult.FromCursor(CreateMissingCursor())); - consumer.Cursor = CreateMissingCursor(); - await accessor.RunConsumerCursor(consumer); - Assert.Equal(2, moves); - Assert.Equal(2, cursors.Count); - Assert.All(cursors, cursor => cursor.Received(1).Dispose()); - Assert.Null(consumer.Cursor); - Assert.True(consumer.HasDeliveryProgressError); - Assert.Equal(1, consumer.DeliveryRecovery!.Attempts); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - await accessor.Shutdown(); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_DoesNotTreatFinishedRegistrationTaskAsRegisteredProducer() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var queueCache = new RecordingQueueCache(); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(200))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(200), DateTime.UtcNow); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var stream = (await accessor.GetPubSubCache()).Single().Value; - // A registration which exits during shutdown can finish without learning its subscribers. - stream.StreamRegistered = false; - stream.RegistrationTask = Task.CompletedTask; - - await accessor.Shutdown(); - - Assert.Empty(queueCache.DeliveryProgressTokens); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(false, 1, 100)] - [InlineData(true, 1, 42)] - [InlineData(false, 2, 100)] - [InlineData(true, 2, 42)] - public async Task Shutdown_DistinguishesEmptyCursorFromBatchWithMissingPosition(bool missingPosition, int batchSize, long expectedCheckpoint) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var cursor = Substitute.For(); - cursor.MoveNextWithResult().Returns( - missingPosition ? QueueCacheCursorMoveResult.Success : QueueCacheCursorMoveResult.NoData, - QueueCacheCursorMoveResult.NoData); - cursor.GetCurrent(out _).Returns(new TestBatchContainer(streamId.StreamId, null!)); - var queueCache = Substitute.For(); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(QueueCacheCursorResult.FromCursor(cursor)); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()).Returns( - Task.FromResult>([new TestBatchContainer(streamId.StreamId, new EventSequenceTokenV2(100))])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, new EventSequenceTokenV2(0), DateTime.UtcNow); - var observer = new RecordingConsumer(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), streamId, observer, null, DateTime.UtcNow); - consumer.IsRegistered = true; - consumer.Cursor = cursor; - consumer.IsCaughtUp = true; - consumer.LastProcessedToken = missingPosition ? new EventSequenceTokenV2(42) : null; - - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - Assert.Equal(!missingPosition, consumer.IsCaughtUp); - Assert.Empty(observer.DeliveredTokens); - await accessor.Shutdown(); - - queueCache.Received(1).UpdateDeliveryProgress( - Arg.Is(token => token.SequenceNumber == expectedCheckpoint), Arg.Any()); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_PushesEarliestDeliveryProgressTokenToCache() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>(new List())); - receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - - var queueCache = new RecordingQueueCache(); - var queueAdapterCache = Substitute.For(); - queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); - var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); - - var streamData = (await testAccessor.GetPubSubCache()).Single().Value; - Assert.Null(streamData.RegistrationTask); - queueCache.ClearDeliveryProgress(); - - var newestConsumer = streamData.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), - streamId, - streamConsumer: null!, - filterData: null, - now: DateTime.UtcNow); - newestConsumer.IsRegistered = true; - newestConsumer.LastProcessedToken = new EventSequenceTokenV2(200); - - var earliestConsumer = streamData.AddConsumer( - GuidId.GetGuidId(Guid.NewGuid()), - streamId, - streamConsumer: null!, - filterData: null, - now: DateTime.UtcNow); - earliestConsumer.IsRegistered = true; - earliestConsumer.LastProcessedToken = new EventSequenceTokenV2(95); - - await testAccessor.Shutdown(); - - Assert.Equal(earliestConsumer.LastProcessedToken, Assert.Single(queueCache.DeliveryProgressTokens)); - } - - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_PushesEarliestDeliveryProgressUsingBaseTokenPosition() - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>(new List())); - receiver.Shutdown(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - - var queueCache = new RecordingQueueCache(); - var queueAdapterCache = Substitute.For(); - queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); - - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); - var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), DateTime.UtcNow); @@ -5786,153 +3321,6 @@ public async Task Shutdown_PreservesCheckpointForIncompatibleTokens(bool provide Assert.Empty(await accessor.GetPubSubCache()); } - [TestSuite("BVT")] - [TestProvider("None")] - [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData("incompatible", false, 1)] - [InlineData("incompatible", false, 2)] - [InlineData("incompatible", true, 1)] - [InlineData("incompatible", true, 2)] - [InlineData("older", false, 1)] - [InlineData("older", false, 2)] - [InlineData("older", true, 1)] - [InlineData("older", true, 2)] - [InlineData("equal", false, 1)] - [InlineData("equal", false, 2)] - [InlineData("equal", true, 1)] - [InlineData("equal", true, 2)] - [InlineData("newer", false, 1)] - [InlineData("newer", false, 2)] - [InlineData("newer", true, 1)] - [InlineData("newer", true, 2)] - public async Task Recovery_FilteredAcknowledgmentHonorsUnconfirmedPosition( - string tokenKind, bool recovering, int batchSize) - { - var pubSub = Substitute.For(); - pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) - .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); - var timeProvider = new FakeTimeProvider(); - var now = timeProvider.GetUtcNow().UtcDateTime; - var queueId = QueueId.GetQueueId("queue", 0u, 0u); - var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); - var incompatible = tokenKind == "incompatible"; - var attemptedToken = new EventSequenceTokenV2(100); - StreamSequenceToken requiredToken = incompatible - ? new IsolatedProviderToken(50) : new EventSequenceTokenV2(50); - StreamSequenceToken progressToken = tokenKind switch - { - "incompatible" => new IsolatedProviderToken(200), - "older" => new EventSequenceTokenV2(75), - "equal" => new EventSequenceTokenV2(100), - _ => new EventSequenceTokenV2(200) - }; - var batch = Substitute.For(); - batch.StreamId.Returns(streamId.StreamId); - batch.SequenceToken.Returns(progressToken); - batch.GetEvents().Returns([Tuple.Create("payload", progressToken)]); - var filter = Substitute.For(); - filter.ShouldDeliver(streamId.StreamId, "payload", null).Returns(false); - var queueCache = Substitute.For(); - queueCache.TryGetCacheCursor(Arg.Any(), Arg.Any()) - .Returns(_ => QueueCacheCursorResult.FromCursor(CreateCursor())); - var adapterCache = Substitute.For(); - adapterCache.CreateQueueCache(queueId).Returns(queueCache); - var receiver = Substitute.For(); - receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult>([batch])); - var agent = CreateAgent(pubSub, queueId, receiver, adapterCache, timeProvider, - options: new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, streamFilter: filter); - var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; - await InitializeAgent(agent); - await accessor.RegisterStream(streamId, requiredToken, now); - Assert.True(await accessor.ReadFromQueue(queueId, receiver, 1000)); - var observer = new RecordingConsumer(); - var consumer = (await accessor.GetPubSubCache()).Single().Value.AddConsumer( - GuidId.GetGuidId(SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid())), - streamId, observer, null, now); - consumer.IsRegistered = true; - consumer.Cursor = CreateCursor(); - consumer.UnconfirmedDeliveryToken = attemptedToken; - consumer.LastToken = recovering - ? StreamHandshakeToken.CreateDeliveyToken(requiredToken) - : StreamHandshakeToken.CreateStartToken(requiredToken); - consumer.LastProcessedToken = recovering ? requiredToken : null; - consumer.PendingStartToken = recovering ? null : requiredToken; - var originalRecovery = recovering ? new StreamRecoveryState(timeProvider.GetTimestamp()) : null; - consumer.HasDeliveryProgressError = recovering; - consumer.DeliveryRecovery = originalRecovery; - consumer.DeliveryRecoveryToken = recovering ? consumer.LastToken : null; - consumer.DeliveryRecoveryCursor = recovering ? consumer.Cursor : null; - - await accessor.RunConsumerCursor(consumer); - - Assert.Empty(observer.DeliveredTokens); - Assert.Empty(observer.Errors); - Assert.Equal(StreamConsumerDataState.Inactive, consumer.State); - Assert.Equal(incompatible, consumer.HasDeliveryProgressError); - Assert.Equal(!incompatible && tokenKind != "older", consumer.IsCaughtUp); - if (incompatible || tokenKind == "older") - { - Assert.Same(attemptedToken, consumer.UnconfirmedDeliveryToken); - } - else - { - Assert.Null(consumer.UnconfirmedDeliveryToken); - } - - if (incompatible) - { - Assert.Null(consumer.LastProcessedToken); - Assert.Same(requiredToken, consumer.DeliveryRecoveryToken?.Token); - Assert.Equal(recovering, consumer.DeliveryRecoveryToken is DeliveryToken); - var recovery = Assert.IsType(consumer.DeliveryRecovery); - Assert.Equal(recovering ? 1 : 0, recovery.Attempts); - if (recovering) - { - Assert.Same(originalRecovery, recovery); - } - - // Repeated incompatible acknowledgments must not replace the recovery episode or its budget. - for (var attempt = recovery.Attempts; attempt < 6; attempt++) - { - await accessor.RunConsumerCursor(consumer); - Assert.Same(recovery, consumer.DeliveryRecovery); - Assert.Equal(attempt + 1, recovery.Attempts); - Assert.Null(consumer.LastProcessedToken); - } - - Assert.True(recovery.Stopped); - Assert.Null(consumer.Cursor); - Assert.Single(observer.Errors); - await accessor.RunConsumerCursor(consumer); - Assert.Equal(6, recovery.Attempts); - } - else - { - Assert.Same(progressToken, consumer.LastProcessedToken); - Assert.Null(consumer.DeliveryRecovery); - } - - await accessor.Shutdown(); - if (incompatible) - { - queueCache.DidNotReceive().UpdateDeliveryProgress(Arg.Any(), Arg.Any()); - } - else - { - queueCache.Received(1).UpdateDeliveryProgress(progressToken, Arg.Any()); - } - - IQueueCacheCursor CreateCursor() - { - var cursor = Substitute.For(); - cursor.MoveNextWithResult().Returns(QueueCacheCursorMoveResult.Success, QueueCacheCursorMoveResult.NoData); - cursor.GetCurrent(out _).Returns(batch); - return cursor; - } - } - private sealed class IsolatedProviderToken(long sequenceNumber) : EventSequenceTokenV2(sequenceNumber); [TestSuite("BVT")] @@ -5995,11 +3383,8 @@ public async Task Shutdown_SkipsDeliveryProgressForPendingRegistrations() [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] - [Theory, TestCategory("BVT"), TestCategory("Streaming")] - [InlineData(0)] - [InlineData(1)] - [InlineData(2)] - public async Task Shutdown_SkipsDeliveryProgressForUnregisteredOrReattachingConsumer(int pendingHandshakes) + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_SkipsDeliveryProgressForUnregisteredConsumer() { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default, TestContext.Current.CancellationToken) @@ -6041,10 +3426,6 @@ public async Task Shutdown_SkipsDeliveryProgressForUnregisteredOrReattachingCons filterData: null, now: DateTime.UtcNow); unregisteredConsumer.PendingStartToken = new EventSequenceTokenV2(50); - unregisteredConsumer.IsRegistered = pendingHandshakes > 0; - unregisteredConsumer.PendingHandshakes = pendingHandshakes; - unregisteredConsumer.IsCaughtUp = pendingHandshakes > 0; - unregisteredConsumer.LastProcessedToken = pendingHandshakes > 0 ? new EventSequenceTokenV2(50) : null; await testAccessor.Shutdown(); From cfd9fa944fba858df158bbb0b6065f9e3c8e3eb7 Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Sat, 12 Sep 2026 11:25:10 -0700 Subject: [PATCH 8/9] fix(streaming): defer checkpoints during subscriber handshakes - Reject delivery progress while any subscriber handshake is pending. - Cover pending and completed re-handshakes in both checkpoint modes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 7 +- .../PersistentStreamPullingAgentTests.cs | 79 ++++++++++++++++++- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index a0f87792f7d..b085d2ecd89 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -57,6 +57,7 @@ internal interface ITestAccessor Task ReadFromQueueWithCancellation(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount, CancellationToken cancellationToken); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); Task> GetPubSubCache(); + Task AddSubscriber(StreamConsumerData consumerData); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); Task RunConsumerCursor(StreamConsumerData consumerData); Task RunQueuePump(QueueId myQueueId, CancellationToken cancellationToken); @@ -129,6 +130,10 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); + Task ITestAccessor.AddSubscriber(StreamConsumerData consumerData) + => this.RunOrQueueTask(() => AddSubscriber_Impl( + consumerData.SubscriptionId, consumerData.StreamId, default, consumerData.FilterData, null)); + Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); @@ -1099,7 +1104,7 @@ private bool TryGetSubscriptionDeliveryProgress(StreamSequenceToken? readBoundar foreach (var consumer in streamConsumers.AllConsumers()) { - if (!consumer.IsRegistered) + if (!consumer.IsRegistered || consumer.PendingHandshakes != 0) { return false; } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 4d820c72e35..fccb2378ccf 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -1068,6 +1068,7 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); + public Func>? OnHandshake { get; set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1092,7 +1093,8 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke return Task.CompletedTask; } - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(requestedToken); + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => OnHandshake?.Invoke() ?? Task.FromResult(requestedToken); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -2574,7 +2576,7 @@ public async Task Checkpoint_DrainedSubscriptionsDoNotPinOtherStreams(bool poole [Theory, TestCategory("BVT"), TestCategory("Streaming")] [InlineData("active", 1L)] [InlineData("pending-batch", 1L)] - [InlineData("handshake", 1L)] + [InlineData("handshake", null)] [InlineData("disposed", 1L)] [InlineData("not-drained", 1L)] [InlineData("unknown-stream-read", 1L)] @@ -2631,6 +2633,79 @@ public async Task Checkpoint_UnprovenSubscriptionsUseOriginalProgress(string con } } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Checkpoint_RegisteredConsumerHandshakeBlocksOnlyUntilCompletion(bool legacyMode, bool completeHandshake) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 50), (scenario.Busy, 200)); + await scenario.Remove(scenario.Idle); + if (legacyMode) + { + scenario.Busy.SafeDisposeCursor(NullLogger.Instance); + scenario.Busy.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Busy)); + scenario.Busy.Cursor = scenario.Cache.GetCacheCursor(scenario.Busy.StreamId, new EventSequenceTokenV2(200)); + } + + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + }; + scenario.Busy.StreamConsumer = consumer; + var replayToken = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Busy); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(scenario.Busy.IsRegistered); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.False(attachment.IsCompleted); + + if (completeHandshake) + { + releaseHandshake.TrySetResult(replayToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Busy.PendingHandshakes); + Assert.Equal(50, scenario.Busy.LastProcessedToken?.SequenceNumber); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (completeHandshake) + { + Assert.Equal(50, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + else + { + Assert.Empty(scenario.Checkpoints); + Assert.False(attachment.IsCompleted); + } + } + finally + { + releaseHandshake.TrySetResult(replayToken); + consumer.ReleaseDelivery(); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.Equal(0, scenario.Busy.PendingHandshakes); + } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] [InlineData(false)] [InlineData(true)] From f19b632c7d47c027289b71b0fdd58ed6865b7c2e Mon Sep 17 00:00:00 2001 From: Jonah Fritz Date: Sat, 12 Sep 2026 12:24:57 -0700 Subject: [PATCH 9/9] fix(streaming): guard checkpoint progress across handshake races - Keep pending-handshake subscriptions out of idle cleanup. - Use legacy checkpointing when delivery overlaps handshake completion. - Cover cleanup and delivery interleavings with deterministic regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8d1da79c-4b16-4f03-be66-dfa8d0c71bf6 --- .../PersistentStreamPullingAgent.cs | 6 + .../StreamConsumerCollection.cs | 3 +- .../PersistentStreamPullingAgentTests.cs | 215 +++++++++++++++++- 3 files changed, 220 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index b085d2ecd89..ef263fad521 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -365,6 +365,12 @@ private async Task AddSubscriber_Impl( { if (await DoHandshakeWithConsumer(data, cacheToken)) { + // Delivery can start while the handshake is awaiting a response. + if (data.State == StreamConsumerDataState.Active) + { + _useLegacyDeliveryProgress = true; + } + data.LastProcessedToken = GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken); data.PendingStartToken = null; data.IsRegistered = true; diff --git a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs index 01e2671333d..391173d1089 100644 --- a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs +++ b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs @@ -88,8 +88,9 @@ public bool IsInactive(DateTime now, TimeSpan inactivityPeriod) // 1) There were no new events received for that stream in the last inactivityPeriod // 2) All consumer for that stream are currently inactive (that is, all cursors are inactive) - // meaning there is nothing for those consumers in the adapter cache. + // 3) No consumer handshake is pending. if (now - lastActivityTime < inactivityPeriod) return false; - return !queueData.Values.Any(data => data.State.Equals(StreamConsumerDataState.Active)); + return !queueData.Values.Any(data => data.PendingHandshakes != 0 || data.State.Equals(StreamConsumerDataState.Active)); } } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index fccb2378ccf..dd3f1d773b7 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -1069,6 +1069,7 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); public Func>? OnHandshake { get; set; } + public Func>? OnDelivery { get; set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1081,6 +1082,11 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu DeliveredTokens.Add(item.SequenceToken); DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); + if (OnDelivery is not null) + { + return await OnDelivery(item); + } + await releaseDelivery.Task; return null; } @@ -2706,6 +2712,206 @@ await Assert.ThrowsAnyAsync( Assert.Equal(0, scenario.Busy.PendingHandshakes); } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("none", 50)] + [InlineData("completed", 50)] + [InlineData("overlapping", 1)] + public async Task Checkpoint_HandshakeAccountsForDeliveryStartedWhileAwaitingResponse( + string deliveryTiming, long expectedCheckpoint) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 50), (scenario.Busy, 100)); + using var replayPin = scenario.Cache.GetCacheCursor(scenario.Busy.StreamId, new EventSequenceTokenV2(50)); + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseNewDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var replayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseReplay = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + OnDelivery = batch => + { + if (batch.SequenceToken.SequenceNumber == 200) + { + return releaseNewDelivery.Task; + } + + Assert.Equal(100, batch.SequenceToken.SequenceNumber); + replayStarted.TrySetResult(); + return releaseReplay.Task; + }, + }; + scenario.Busy.StreamConsumer = consumer; + var replayToken = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Busy); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(100, scenario.Busy.LastProcessedToken?.SequenceNumber); + + if (deliveryTiming != "none") + { + if (deliveryTiming == "completed") + { + releaseNewDelivery.TrySetResult(null); + } + + await scenario.Read((scenario.Busy, 200)); + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal( + deliveryTiming == "overlapping" ? StreamConsumerDataState.Active : StreamConsumerDataState.Inactive, + scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(200, Assert.Single(consumer.DeliveredTokens).SequenceNumber); + } + + releaseHandshake.TrySetResult(replayToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Busy.PendingHandshakes); + Assert.Equal(50, scenario.Busy.LastProcessedToken?.SequenceNumber); + + releaseNewDelivery.TrySetResult(null); + await replayStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(deliveryTiming == "overlapping" ? 200 : 50, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.Equal(StreamConsumerDataState.Active, scenario.Busy.State); + Assert.Equal( + deliveryTiming == "none" ? new long[] { 100 } : new long[] { 200, 100 }, + consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.Empty(consumer.Errors); + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(expectedCheckpoint, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + finally + { + releaseHandshake.TrySetResult(replayToken); + releaseNewDelivery.TrySetResult(null); + releaseReplay.TrySetResult(null); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, "pending")] + [InlineData(false, "replaying")] + [InlineData(false, "drained")] + [InlineData(true, "pending")] + [InlineData(true, "replaying")] + [InlineData(true, "drained")] + public async Task Checkpoint_IdleCleanupPreservesHandshakeUntilCompletion(bool legacyMode, string outcome) + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var scenario = await CreateCheckpointScenario(timeProvider: timeProvider); + await scenario.Read((scenario.Idle, 50), (scenario.Idle, 100), (scenario.Busy, 200)); + // Isolate subscription cleanup from the cache's independent eviction policy. + using var replayPin = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(50)); + if (legacyMode) + { + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + scenario.Idle.Cursor = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(100)); + } + + var streams = await scenario.Accessor.GetPubSubCache(); + var idleStream = streams[scenario.Idle.StreamId]; + var busyStream = streams[scenario.Busy.StreamId]; + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + }; + scenario.Idle.StreamConsumer = consumer; + var handshakeToken = StreamHandshakeToken.CreateDeliveyToken( + new EventSequenceTokenV2(outcome == "drained" ? 100 : 50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Idle); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Idle.State); + Assert.Equal(1, scenario.Idle.PendingHandshakes); + Assert.Equal(100, scenario.Idle.LastProcessedToken?.SequenceNumber); + var cursor = scenario.Idle.Cursor; + + var inactivityPeriod = new StreamPullingAgentOptions().StreamInactivityPeriod; + timeProvider.Advance(inactivityPeriod + TimeSpan.FromTicks(1)); + busyStream.RefreshActivity(timeProvider.GetUtcNow().UtcDateTime); + await scenario.Read((scenario.Busy, 201)); + + streams = await scenario.Accessor.GetPubSubCache(); + Assert.Equal(2, streams.Count); + Assert.Same(idleStream, streams[scenario.Idle.StreamId]); + Assert.Same(scenario.Idle, Assert.Single(idleStream.AllConsumers())); + Assert.Same(cursor, scenario.Idle.Cursor); + Assert.Equal(1, scenario.Idle.PendingHandshakes); + Assert.False(attachment.IsCompleted); + Assert.Equal(201, scenario.Busy.LastProcessedToken?.SequenceNumber); + + if (outcome != "pending") + { + releaseHandshake.TrySetResult(handshakeToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Idle.PendingHandshakes); + Assert.Empty(consumer.Errors); + if (outcome == "replaying") + { + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(50, scenario.Idle.LastProcessedToken?.SequenceNumber); + } + else + { + Assert.Equal(100, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Idle.State); + timeProvider.Advance(inactivityPeriod + TimeSpan.FromTicks(1)); + busyStream.RefreshActivity(timeProvider.GetUtcNow().UtcDateTime); + await scenario.Read((scenario.Busy, 202)); + + streams = await scenario.Accessor.GetPubSubCache(); + Assert.Equal(scenario.Busy.StreamId, Assert.Single(streams).Key); + Assert.Empty(idleStream.AllConsumers()); + Assert.Null(scenario.Idle.Cursor); + } + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (outcome == "pending") + { + Assert.Empty(scenario.Checkpoints); + Assert.False(attachment.IsCompleted); + } + else + { + Assert.Equal(outcome == "drained" ? 202 : 50, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + } + finally + { + releaseHandshake.TrySetResult(handshakeToken); + consumer.ReleaseDelivery(); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.Equal(0, scenario.Idle.PendingHandshakes); + } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] [InlineData(false)] [InlineData(true)] @@ -2965,7 +3171,8 @@ await Assert.ThrowsAnyAsync( } private static async Task CreateCheckpointScenario( - bool pooled = false, int batchSize = 1, bool filtered = false, RecordingQueueCache? emptyCache = null) + bool pooled = false, int batchSize = 1, bool filtered = false, RecordingQueueCache? emptyCache = null, + TimeProvider? timeProvider = null) { IQueueCache cache; List checkpoints; @@ -2992,7 +3199,7 @@ private static async Task CreateCheckpointScenario( var (accessor, _, idleStream) = await CreateInitializedAgentWithStream( idleId, new EventSequenceTokenV2(1), cache, new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, - filtered ? Substitute.For() : null); + filtered ? Substitute.For() : null, timeProvider); await accessor.RegisterStream(busyId, new EventSequenceTokenV2(2), DateTime.UtcNow); var busyStream = (await accessor.GetPubSubCache())[busyId]; @@ -3057,7 +3264,8 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => StreamSequenceToken registrationToken, IQueueCache queueCache, StreamPullingAgentOptions options, - IStreamFilter? filter = null) + IStreamFilter? filter = null, + TimeProvider? timeProvider = null) { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default) @@ -3070,6 +3278,7 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache, + timeProvider: timeProvider, options: options, filter: filter); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent;