diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index e441c795b19..ef263fad521 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -46,6 +46,8 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; + private StreamSequenceToken? _lastReadToken; + private bool _useLegacyDeliveryProgress; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -55,6 +57,7 @@ internal interface ITestAccessor Task ReadFromQueueWithCancellation(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount, CancellationToken cancellationToken); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); Task> GetPubSubCache(); + Task AddSubscriber(StreamConsumerData consumerData); Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); Task RunConsumerCursor(StreamConsumerData consumerData); Task RunQueuePump(QueueId myQueueId, CancellationToken cancellationToken); @@ -127,6 +130,10 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); + Task ITestAccessor.AddSubscriber(StreamConsumerData consumerData) + => this.RunOrQueueTask(() => AddSubscriber_Impl( + consumerData.SubscriptionId, consumerData.StreamId, default, consumerData.FilterData, null)); + Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); @@ -155,6 +162,8 @@ public Task Initialize(CancellationToken cancellationToken) LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; + _lastReadToken = null; + _useLegacyDeliveryProgress = false; lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -345,14 +354,34 @@ private async Task AddSubscriber_Impl( StreamingEvents.EmitSubscriptionAdded(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); } - if (await DoHandshakeWithConsumer(data, cacheToken)) + data.IsCaughtUp = false; + if (data.PendingHandshakes != 0 || data.State == StreamConsumerDataState.Active) + { + _useLegacyDeliveryProgress = true; + } + + data.PendingHandshakes++; + try + { + if (await DoHandshakeWithConsumer(data, cacheToken)) + { + // Delivery can start while the handshake is awaiting a response. + if (data.State == StreamConsumerDataState.Active) + { + _useLegacyDeliveryProgress = true; + } + + data.LastProcessedToken = GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken); + data.PendingStartToken = null; + data.IsRegistered = true; + StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); + if (data.State == StreamConsumerDataState.Inactive) + RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + } + } + finally { - data.LastProcessedToken = GetInitialDeliveryProgress(data.LastToken, data.LastProcessedToken); - data.PendingStartToken = null; - data.IsRegistered = true; - StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); - if (data.State == StreamConsumerDataState.Inactive) - RunConsumerCursor(data).Ignore(); // Start delivering events if not actively doing so + data.PendingHandshakes--; } } @@ -455,6 +484,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception exception) { + _useLegacyDeliveryProgress = true; exceptionOccured = exception; } if (exceptionOccured != null) @@ -496,6 +526,7 @@ QueueCacheCursorResultKind.CacheMiss when cacheToken is not null } catch (Exception) { + _useLegacyDeliveryProgress = true; consumerData.Cursor = GetCacheCursorOrThrow(consumerData.StreamId, null); } } @@ -747,6 +778,15 @@ public void RemoveSubscriber_Impl(GuidId subscriptionId, QualifiedStreamId strea if (!pubSubCache.TryGetValue(streamId, out var streamData)) return; + if (!_useLegacyDeliveryProgress && (!streamData.StreamRegistered || streamData.RegistrationTask is not null + || streamData.TryGetConsumer(subscriptionId, out var consumer) + && (!HasCaughtUpDeliveryProgress(consumer, streamData.LastReadToken) + || _lastReadToken is null + || !TryCompareQueueProgress(consumer.LastProcessedToken!, _lastReadToken, out _)))) + { + _useLegacyDeliveryProgress = true; + } + // remove consumer bool removed = streamData.RemoveConsumer(subscriptionId, logger); if (removed) @@ -823,6 +863,7 @@ private async Task PumpQueue(QueueId queueId, CancellationToken cancellationToke bool ReadLoopRetryExceptionFilter(Exception e, int retryCounter) { + _useLegacyDeliveryProgress = true; LogErrorRetrying(retryCounter, new(queueId), e); return !cancellationToken.IsCancellationRequested && !IsShutdown; } @@ -902,37 +943,90 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? // Receivers built against older Orleans versions can still return null. if (multiBatch is null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback. - queueCache?.AddToCache(multiBatch); - numMessages += multiBatch.Count; - if (_streamInstruments?.PersistentStreamReadMessages.Enabled is true) + var accountedFor = false; + try + { + queueCache?.AddToCache(multiBatch); + numMessages += multiBatch.Count; + if (_streamInstruments?.PersistentStreamReadMessages.Enabled is true) + { + tags = StreamInstrumentsTagUtils.InitializeTags(myQueueId, streamProviderName); + _streamInstruments.PersistentStreamReadMessages.Add(multiBatch.Count, tags.Value); + } + + LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); + + foreach (var group in + multiBatch + .Where(m => m is not null) + .GroupBy(container => container.StreamId)) + { + var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); + StreamSequenceToken startToken = group.First().SequenceToken; + if (pubSubCache.TryGetValue(streamId, out var streamData)) + { + streamData.RefreshActivity(now); + StartInactiveCursors(streamData, startToken, CancellationToken.None); + } + else + { + // Run registration in the background so that cold-stream pubsub + // calls do not stall message delivery for other streams on the same queue. + RegisterStream(streamId, startToken, now, CancellationToken.None); + } + + if (pubSubCache.TryGetValue(streamId, out streamData)) + { + streamData.LastReadToken = group.Last().SequenceToken; + } + } + + accountedFor = true; + } + finally { - tags = StreamInstrumentsTagUtils.InitializeTags(myQueueId, streamProviderName); - _streamInstruments.PersistentStreamReadMessages.Add(multiBatch.Count, tags.Value); + if (!accountedFor) + { + _useLegacyDeliveryProgress = true; + } } - LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); + RecordReadProgress(multiBatch); + return !IsShutdown && !cancellationToken.IsCancellationRequested; + } + + private void RecordReadProgress(IList batches) + { + if (queueCache is null || _useLegacyDeliveryProgress) return; - foreach (var group in - multiBatch - .Where(m => m is not null) - .GroupBy(container => container.StreamId)) + foreach (var batch in batches) { - var streamId = new QualifiedStreamId(queueAdapter.Name, group.Key); - StreamSequenceToken startToken = group.First().SequenceToken; - if (pubSubCache.TryGetValue(streamId, out var streamData)) + if (batch?.SequenceToken is not { } token) { - streamData.RefreshActivity(now); - StartInactiveCursors(streamData, startToken, CancellationToken.None); + _useLegacyDeliveryProgress = true; + LogWarningUnknownReadProgress(new(QueueId)); + return; + } + + if (_lastReadToken is null) + { + _lastReadToken = token; + } + else if (!TryCompareQueueProgress(token, _lastReadToken, out var comparison)) + { + return; + } + else if (comparison < 0) + { + _useLegacyDeliveryProgress = true; + LogWarningUnknownReadProgress(new(QueueId)); + return; } else { - // Run registration in the background so that cold-stream pubsub - // calls do not stall message delivery for other streams on the same queue. - RegisterStream(streamId, startToken, now, CancellationToken.None); + _lastReadToken = token; } } - - return !IsShutdown && !cancellationToken.IsCancellationRequested; } private void CleanupPubSubCache(DateTime now) @@ -956,6 +1050,14 @@ private void CleanupPubSubCache(DateTime now) { if (pubSubCache.Remove(streamId, out var streamData)) { + if (!_useLegacyDeliveryProgress && (!streamData.StreamRegistered || streamData.RegistrationTask is not null + || streamData.AllConsumers().Any(consumer => + !HasCaughtUpDeliveryProgress(consumer, streamData.LastReadToken) + || _lastReadToken is null + || !TryCompareQueueProgress(consumer.LastProcessedToken!, _lastReadToken, out _)))) + { + _useLegacyDeliveryProgress = true; + } streamData.DisposeAll(logger); StreamingEvents.EmitStreamInactive(streamProviderName, streamId.StreamId, options.StreamInactivityPeriod, Silo); } @@ -977,8 +1079,27 @@ private void NotifyDeliveryProgress() } private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) + { + var canAdvancePastDrainedSubscriptions = !_useLegacyDeliveryProgress + && _lastReadToken is not null + && pubSubCache.Values.All(static stream => stream.StreamRegistered && stream.RegistrationTask is null + && stream.AllConsumers().All(static consumer => consumer.PendingHandshakes == 0)); + + if (canAdvancePastDrainedSubscriptions + && TryGetSubscriptionDeliveryProgress(_lastReadToken, out earliest) + && !_useLegacyDeliveryProgress) + { + return true; + } + + // Opting out restores the original calculation; it does not freeze normal delivery progress. + return TryGetSubscriptionDeliveryProgress(null, out earliest); + } + + private bool TryGetSubscriptionDeliveryProgress(StreamSequenceToken? readBoundary, out StreamSequenceToken? earliest) { earliest = null; + var hasSubscriptions = false; foreach (var streamConsumers in pubSubCache.Values) { @@ -989,7 +1110,7 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) foreach (var consumer in streamConsumers.AllConsumers()) { - if (!consumer.IsRegistered) + if (!consumer.IsRegistered || consumer.PendingHandshakes != 0) { return false; } @@ -1000,6 +1121,20 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return false; } + hasSubscriptions = true; + if (readBoundary is not null) + { + if (!TryCompareQueueProgress(current, readBoundary, out _)) + { + return false; + } + + if (HasCaughtUpDeliveryProgress(consumer, streamConsumers.LastReadToken)) + { + continue; + } + } + if (earliest is null) { earliest = current; @@ -1023,9 +1158,56 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) } } + if (readBoundary is not null && hasSubscriptions) + { + if (earliest is null) + { + earliest = readBoundary; + } + else + { + if (!TryCompareQueueProgress(readBoundary, earliest, out var comparison)) + { + return false; + } + + if (comparison < 0) + { + earliest = readBoundary; + } + } + } + return true; } + private bool HasCaughtUpDeliveryProgress(StreamConsumerData consumer, StreamSequenceToken? streamReadBoundary) + => consumer.IsCaughtUp + && consumer.State == StreamConsumerDataState.Inactive + && consumer.PendingHandshakes == 0 + && consumer.PendingBatch is null + && consumer.Cursor is not null + && consumer.LastProcessedToken is { } progress + && streamReadBoundary is not null + && TryCompareQueueProgress(progress, streamReadBoundary, out var comparison) + && comparison >= 0; + + private bool TryCompareQueueProgress(StreamSequenceToken token, StreamSequenceToken other, out int comparison) + { + try + { + comparison = EventSequenceTokenCompatibility.Compare(token, other); + return true; + } + catch (ArgumentOutOfRangeException exception) when (exception.ParamName == "other") + { + _useLegacyDeliveryProgress = true; + LogWarningIncompatibleQueueProgress(new(QueueId), token, other, exception); + comparison = default; + return false; + } + } + private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => EventSequenceTokenCompatibility.Compare(current, other) < 0; @@ -1106,6 +1288,7 @@ async Task RegisterStreamAsync() void RemoveCanceledRegistration() { + _useLegacyDeliveryProgress = true; if (pubSubCache.TryGetValue(streamId, out var cachedStreamData) && ReferenceEquals(cachedStreamData, streamData)) { @@ -1117,6 +1300,7 @@ void RemoveCanceledRegistration() void FailRegistration(Exception exception) { + _useLegacyDeliveryProgress = true; LogWarningFailedToRegisterStream(streamId, exception); StreamingEvents.EmitPullingAgentStreamRegistrationFailed(streamProviderName, Silo, QueueId, streamId.StreamId, exception); @@ -1143,6 +1327,7 @@ async Task SubscribeWithIsolation(PubSubSubscriptionState item) } catch (Exception exception) { + _useLegacyDeliveryProgress = true; LogWarningFailedToAddSubscription(item.Stream, exception); } } @@ -1188,6 +1373,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati consumerData.Cursor is null) return; consumerData.State = StreamConsumerDataState.Active; + consumerData.IsCaughtUp = false; var deliveredAny = false; while (!IsShutdown && !cancellationToken.IsCancellationRequested && consumerData.Cursor is not null) { @@ -1203,6 +1389,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati nextBatch = GetBatchForConsumer(consumerData); if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) { + _useLegacyDeliveryProgress = true; consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetCacheMissRecoveryCursor(consumerData); continue; @@ -1215,6 +1402,15 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati if (!nextBatch.HasProgress) { + if (nextBatch.CursorResult.Kind == QueueCacheCursorMoveResultKind.NoData) + { + consumerData.IsCaughtUp = consumerData.LastProcessedToken is not null; + } + else + { + _useLegacyDeliveryProgress = true; + } + // Only emit cursor-drained when we transitioned from delivering to empty, // not on every empty poll. if (deliveredAny) @@ -1229,6 +1425,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (Exception exc) { + _useLegacyDeliveryProgress = true; exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); consumerData.Cursor = GetRecoveryCursor(consumerData); @@ -1272,6 +1469,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati cancellationToken: cancellationToken); if (newToken is not null) { + _useLegacyDeliveryProgress = true; consumerData.LastToken = newToken; IQueueCacheCursor newCursor; IBatchContainer? pendingBatch = null; @@ -1344,6 +1542,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (Exception exc) { + _useLegacyDeliveryProgress = true; if (batchCursor is not null && nextBatch.Batch is not null) { batchCursor.RecordDeliveryFailure(nextBatch.Batch); @@ -1378,10 +1577,12 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData, Cancellati } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + _useLegacyDeliveryProgress = true; consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { + _useLegacyDeliveryProgress = true; // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception LogErrorRunConsumerCursor(exc); consumerData.State = StreamConsumerDataState.Inactive; @@ -1698,6 +1899,15 @@ private readonly struct QueueIdLogRecord(QueueId queueId) public override string ToString() => queueId.ToStringWithHashCode(); } + [LoggerMessage(Level = LogLevel.Warning, + Message = "Cannot establish the read boundary for queue {QueueId}; using subscription-based delivery progress.")] + private partial void LogWarningUnknownReadProgress(QueueIdLogRecord queueId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Incompatible tokens {Token} and {Other} for queue {QueueId}; using subscription-based delivery progress.")] + private partial void LogWarningIncompatibleQueueProgress( + QueueIdLogRecord queueId, StreamSequenceToken token, StreamSequenceToken other, Exception exception); + [LoggerMessage( Level = LogLevel.Information, EventId = (int)ErrorCode.PersistentStreamPullingAgent_01, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index e4c82054321..2b96fd1c4f3 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -46,6 +46,11 @@ internal sealed class StreamConsumerData [NonSerialized] public StreamSequenceToken? LastProcessedToken; + [NonSerialized] + public bool IsCaughtUp; + [NonSerialized] + public int PendingHandshakes; + public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { SubscriptionId = subscriptionId; @@ -56,6 +61,7 @@ public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, ISt internal void SafeDisposeCursor(ILogger logger) { + IsCaughtUp = false; PendingBatch = null; if (Cursor is { } cursor) { diff --git a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs index e77c8f21a3a..391173d1089 100644 --- a/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs +++ b/src/Orleans.Streaming/PersistentStreams/StreamConsumerCollection.cs @@ -23,6 +23,9 @@ internal sealed class StreamConsumerCollection [NonSerialized] public Task? RegistrationTask; + [NonSerialized] + public StreamSequenceToken? LastReadToken; + public StreamConsumerCollection(DateTime now) { queueData = new Dictionary(); @@ -85,8 +88,9 @@ public bool IsInactive(DateTime now, TimeSpan inactivityPeriod) // 1) There were no new events received for that stream in the last inactivityPeriod // 2) All consumer for that stream are currently inactive (that is, all cursors are inactive) - // meaning there is nothing for those consumers in the adapter cache. + // 3) No consumer handshake is pending. if (now - lastActivityTime < inactivityPeriod) return false; - return !queueData.Values.Any(data => data.State.Equals(StreamConsumerDataState.Active)); + return !queueData.Values.Any(data => data.PendingHandshakes != 0 || data.State.Equals(StreamConsumerDataState.Active)); } } } diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs index cd27e0da14f..c436a4eecd0 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs @@ -103,6 +103,7 @@ QueueCacheCursorResult TryGetCacheCursorAtPosition( /// /// /// The earliest last processed sequence token across registered subscriptions. + /// When caught-up progress is known, drained subscriptions can instead advance to the queue's read boundary. /// A value indicates that there are no active subscriptions. /// The token is only valid for the duration of the call and must not be stored. /// diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 60aece238a5..dd3f1d773b7 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -399,7 +399,8 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterReceiver? receiver = null, IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, - StreamPullingAgentOptions? options = null) + StreamPullingAgentOptions? options = null, + IStreamFilter? filter = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -436,7 +437,7 @@ private static PersistentStreamPullingAgent CreateAgent( SystemTargetGrainId.Create(SystemTargetGrainId.CreateGrainType("persistent-stream-pulling-agent-test"), siloAddress), "provider", pubSub!, - new NoOpStreamFilter(), + filter ?? new NoOpStreamFilter(), queueId, options ?? new StreamPullingAgentOptions(), queueAdapter, @@ -448,6 +449,15 @@ private static PersistentStreamPullingAgent CreateAgent( shared); } + private sealed class RecordingSimpleQueueCache() + : SimpleQueueCache(256, NullLogger.Instance), IQueueCache + { + public List DeliveryProgressTokens { get; } = []; + + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => DeliveryProgressTokens.Add(earliestSubscriptionToken); + } + private sealed class RecordingQueueCache : IQueueCache { public int DeliveryProgressCallCount { get; private set; } @@ -878,6 +888,7 @@ public void RecordDeliveryFailure() private sealed class PurgeablePooledQueueCache : IQueueCache { private readonly PooledQueueCache cache; + public List DeliveryProgressTokens { get; } = []; public PurgeablePooledQueueCache(bool retainPurgeMetadata = false) { @@ -942,6 +953,7 @@ private QueueCacheCursorResult WrapCursorResult(QueueCacheCur public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { + DeliveryProgressTokens.Add(earliestSubscriptionToken); } public void Purge() @@ -1056,6 +1068,8 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu public List DeliveredTokens { get; } = new(); public List DeliveredHandshakeTokens { get; } = new(); public List Errors { get; } = new(); + public Func>? OnHandshake { get; set; } + public Func>? OnDelivery { get; set; } public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -1068,6 +1082,11 @@ private sealed class RecordingConsumer(StreamHandshakeToken? requestedToken = nu DeliveredTokens.Add(item.SequenceToken); DeliveredHandshakeTokens.Add(handshakeToken); Delivered.TrySetResult(true); + if (OnDelivery is not null) + { + return await OnDelivery(item); + } + await releaseDelivery.Task; return null; } @@ -1080,7 +1099,8 @@ public Task ErrorInStream(GuidId subscriptionId, Exception exc, CancellationToke return Task.CompletedTask; } - public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) => Task.FromResult(requestedToken); + public Task GetSequenceToken(GuidId subscriptionId, CancellationToken cancellationToken) + => OnHandshake?.Invoke() ?? Task.FromResult(requestedToken); public void ReleaseDelivery() => releaseDelivery.TrySetResult(true); } @@ -2526,6 +2546,713 @@ await testAccessor.ReadFromQueue( Assert.Equal(attemptedToken, Assert.Single(queueCache.DeliveryProgressTokens)); } + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, 1, false)] + [InlineData(false, 10, false)] + [InlineData(true, 1, false)] + [InlineData(true, 10, false)] + [InlineData(false, 1, true)] + [InlineData(false, 10, true)] + [InlineData(true, 1, true)] + [InlineData(true, 10, true)] + public async Task Checkpoint_DrainedSubscriptionsDoNotPinOtherStreams(bool pooled, int batchSize, bool filtered) + { + await using var scenario = await CreateCheckpointScenario(pooled, batchSize, filtered); + var batches = Enumerable.Range(1, 200) + .Select(sequence => (IBatchContainer)new TestBatchContainer( + sequence == 1 ? scenario.Idle.StreamId.StreamId : scenario.Busy.StreamId.StreamId, + new EventSequenceTokenV2(sequence))) + .ToList(); + + await scenario.Read(batches); + + Assert.True(scenario.Idle.IsCaughtUp); + Assert.True(scenario.Busy.IsCaughtUp); + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + if (filtered) + { + Assert.Empty(((ImmediateRecordingConsumer)scenario.Idle.StreamConsumer).DeliveredTokens); + Assert.Empty(((ImmediateRecordingConsumer)scenario.Busy.StreamConsumer).DeliveredTokens); + } + + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("active", 1L)] + [InlineData("pending-batch", 1L)] + [InlineData("handshake", null)] + [InlineData("disposed", 1L)] + [InlineData("not-drained", 1L)] + [InlineData("unknown-stream-read", 1L)] + [InlineData("unregistered-producer", 1L)] + [InlineData("unregistered-consumer", null)] + [InlineData("unknown-progress", null)] + public async Task Checkpoint_UnprovenSubscriptionsUseOriginalProgress(string condition, long? expected) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + var streams = await scenario.Accessor.GetPubSubCache(); + switch (condition) + { + case "active": + scenario.Idle.State = StreamConsumerDataState.Active; + break; + case "pending-batch": + scenario.Idle.PendingBatch = new TestBatchContainer(scenario.Idle.StreamId.StreamId, new EventSequenceTokenV2(1)); + break; + case "handshake": + scenario.Idle.PendingHandshakes = 1; + break; + case "disposed": + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + break; + case "not-drained": + scenario.Idle.IsCaughtUp = false; + break; + case "unknown-stream-read": + streams[scenario.Idle.StreamId].LastReadToken = null; + break; + case "unregistered-producer": + streams[scenario.Idle.StreamId].StreamRegistered = false; + break; + case "unregistered-consumer": + scenario.Idle.IsRegistered = false; + break; + case "unknown-progress": + scenario.Idle.LastProcessedToken = null; + break; + default: + throw new InvalidOperationException(condition); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown(); + if (expected.HasValue) + { + Assert.Equal(expected.Value, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + else + { + Assert.Empty(scenario.Checkpoints); + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Checkpoint_RegisteredConsumerHandshakeBlocksOnlyUntilCompletion(bool legacyMode, bool completeHandshake) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 50), (scenario.Busy, 200)); + await scenario.Remove(scenario.Idle); + if (legacyMode) + { + scenario.Busy.SafeDisposeCursor(NullLogger.Instance); + scenario.Busy.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Busy)); + scenario.Busy.Cursor = scenario.Cache.GetCacheCursor(scenario.Busy.StreamId, new EventSequenceTokenV2(200)); + } + + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + }; + scenario.Busy.StreamConsumer = consumer; + var replayToken = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Busy); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(scenario.Busy.IsRegistered); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.False(attachment.IsCompleted); + + if (completeHandshake) + { + releaseHandshake.TrySetResult(replayToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Busy.PendingHandshakes); + Assert.Equal(50, scenario.Busy.LastProcessedToken?.SequenceNumber); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (completeHandshake) + { + Assert.Equal(50, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + else + { + Assert.Empty(scenario.Checkpoints); + Assert.False(attachment.IsCompleted); + } + } + finally + { + releaseHandshake.TrySetResult(replayToken); + consumer.ReleaseDelivery(); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.Equal(0, scenario.Busy.PendingHandshakes); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("none", 50)] + [InlineData("completed", 50)] + [InlineData("overlapping", 1)] + public async Task Checkpoint_HandshakeAccountsForDeliveryStartedWhileAwaitingResponse( + string deliveryTiming, long expectedCheckpoint) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 50), (scenario.Busy, 100)); + using var replayPin = scenario.Cache.GetCacheCursor(scenario.Busy.StreamId, new EventSequenceTokenV2(50)); + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseNewDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var replayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseReplay = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + OnDelivery = batch => + { + if (batch.SequenceToken.SequenceNumber == 200) + { + return releaseNewDelivery.Task; + } + + Assert.Equal(100, batch.SequenceToken.SequenceNumber); + replayStarted.TrySetResult(); + return releaseReplay.Task; + }, + }; + scenario.Busy.StreamConsumer = consumer; + var replayToken = StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Busy); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(100, scenario.Busy.LastProcessedToken?.SequenceNumber); + + if (deliveryTiming != "none") + { + if (deliveryTiming == "completed") + { + releaseNewDelivery.TrySetResult(null); + } + + await scenario.Read((scenario.Busy, 200)); + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal( + deliveryTiming == "overlapping" ? StreamConsumerDataState.Active : StreamConsumerDataState.Inactive, + scenario.Busy.State); + Assert.Equal(1, scenario.Busy.PendingHandshakes); + Assert.Equal(200, Assert.Single(consumer.DeliveredTokens).SequenceNumber); + } + + releaseHandshake.TrySetResult(replayToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Busy.PendingHandshakes); + Assert.Equal(50, scenario.Busy.LastProcessedToken?.SequenceNumber); + + releaseNewDelivery.TrySetResult(null); + await replayStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(deliveryTiming == "overlapping" ? 200 : 50, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.Equal(StreamConsumerDataState.Active, scenario.Busy.State); + Assert.Equal( + deliveryTiming == "none" ? new long[] { 100 } : new long[] { 200, 100 }, + consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + Assert.Empty(consumer.Errors); + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(expectedCheckpoint, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + finally + { + releaseHandshake.TrySetResult(replayToken); + releaseNewDelivery.TrySetResult(null); + releaseReplay.TrySetResult(null); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, "pending")] + [InlineData(false, "replaying")] + [InlineData(false, "drained")] + [InlineData(true, "pending")] + [InlineData(true, "replaying")] + [InlineData(true, "drained")] + public async Task Checkpoint_IdleCleanupPreservesHandshakeUntilCompletion(bool legacyMode, string outcome) + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var scenario = await CreateCheckpointScenario(timeProvider: timeProvider); + await scenario.Read((scenario.Idle, 50), (scenario.Idle, 100), (scenario.Busy, 200)); + // Isolate subscription cleanup from the cache's independent eviction policy. + using var replayPin = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(50)); + if (legacyMode) + { + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + scenario.Idle.Cursor = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(100)); + } + + var streams = await scenario.Accessor.GetPubSubCache(); + var idleStream = streams[scenario.Idle.StreamId]; + var busyStream = streams[scenario.Busy.StreamId]; + var handshakeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandshake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumer = new RecordingConsumer + { + OnHandshake = () => + { + handshakeStarted.TrySetResult(); + return releaseHandshake.Task; + }, + }; + scenario.Idle.StreamConsumer = consumer; + var handshakeToken = StreamHandshakeToken.CreateDeliveyToken( + new EventSequenceTokenV2(outcome == "drained" ? 100 : 50)); + var attachment = scenario.Accessor.AddSubscriber(scenario.Idle); + + try + { + await handshakeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Idle.State); + Assert.Equal(1, scenario.Idle.PendingHandshakes); + Assert.Equal(100, scenario.Idle.LastProcessedToken?.SequenceNumber); + var cursor = scenario.Idle.Cursor; + + var inactivityPeriod = new StreamPullingAgentOptions().StreamInactivityPeriod; + timeProvider.Advance(inactivityPeriod + TimeSpan.FromTicks(1)); + busyStream.RefreshActivity(timeProvider.GetUtcNow().UtcDateTime); + await scenario.Read((scenario.Busy, 201)); + + streams = await scenario.Accessor.GetPubSubCache(); + Assert.Equal(2, streams.Count); + Assert.Same(idleStream, streams[scenario.Idle.StreamId]); + Assert.Same(scenario.Idle, Assert.Single(idleStream.AllConsumers())); + Assert.Same(cursor, scenario.Idle.Cursor); + Assert.Equal(1, scenario.Idle.PendingHandshakes); + Assert.False(attachment.IsCompleted); + Assert.Equal(201, scenario.Busy.LastProcessedToken?.SequenceNumber); + + if (outcome != "pending") + { + releaseHandshake.TrySetResult(handshakeToken); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.Idle.PendingHandshakes); + Assert.Empty(consumer.Errors); + if (outcome == "replaying") + { + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(50, scenario.Idle.LastProcessedToken?.SequenceNumber); + } + else + { + Assert.Equal(100, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + Assert.Equal(StreamConsumerDataState.Inactive, scenario.Idle.State); + timeProvider.Advance(inactivityPeriod + TimeSpan.FromTicks(1)); + busyStream.RefreshActivity(timeProvider.GetUtcNow().UtcDateTime); + await scenario.Read((scenario.Busy, 202)); + + streams = await scenario.Accessor.GetPubSubCache(); + Assert.Equal(scenario.Busy.StreamId, Assert.Single(streams).Key); + Assert.Empty(idleStream.AllConsumers()); + Assert.Null(scenario.Idle.Cursor); + } + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (outcome == "pending") + { + Assert.Empty(scenario.Checkpoints); + Assert.False(attachment.IsCompleted); + } + else + { + Assert.Equal(outcome == "drained" ? 202 : 50, Assert.Single(scenario.Checkpoints)?.SequenceNumber); + } + } + finally + { + releaseHandshake.TrySetResult(handshakeToken); + consumer.ReleaseDelivery(); + await attachment.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.Equal(0, scenario.Idle.PendingHandshakes); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_NewReadsMustBeAcknowledgedAfterDraining(bool pooled) + { + await using var scenario = await CreateCheckpointScenario(pooled); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + + Assert.Equal(3, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(200, scenario.Busy.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_EmptyCursorCannotHideAnUnacknowledgedRead(bool pooled) + { + await using var scenario = await CreateCheckpointScenario(pooled); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + if (scenario.Cache is PurgeablePooledQueueCache pooledCache) + { + pooledCache.Purge(); + scenario.Idle.Cursor = pooledCache.GetCacheCursor(scenario.Idle.StreamId, scenario.Idle.LastProcessedToken); + } + else + { + scenario.Idle.Cursor = new EmptyQueueCacheCursor(); + } + + scenario.Idle.State = StreamConsumerDataState.Inactive; + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + await scenario.AssertCheckpoint(1); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false, 0)] + [InlineData(false, 1)] + [InlineData(true, 0)] + [InlineData(true, 1)] + public async Task Checkpoint_InitialResumeCanEstablishCaughtUpProgress(bool pooled, long acknowledged) + { + await using var scenario = await CreateCheckpointScenario(pooled); + scenario.Idle.IsRegistered = false; + var consumer = new RecordingConsumer(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(acknowledged))); + consumer.ReleaseDelivery(); + scenario.Idle.StreamConsumer = consumer; + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + + Assert.True(await scenario.Accessor.DoHandshakeWithConsumer(scenario.Idle, new EventSequenceTokenV2(1))); + scenario.Idle.LastProcessedToken = PersistentStreamPullingAgent.GetInitialDeliveryProgress(scenario.Idle.LastToken); + scenario.Idle.IsRegistered = true; + scenario.Idle.PendingStartToken = null; + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(1, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.Equal(acknowledged == 0 ? 1 : 0, consumer.DeliveredTokens.Count); + await scenario.AssertCheckpoint(200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_CursorFailureDoesNotFreezeLaterOriginalProgress(bool failCursor) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + if (failCursor) + { + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + } + + await scenario.Read((scenario.Idle, 3), (scenario.Busy, 200)); + scenario.Idle.Cursor = scenario.Cache.GetCacheCursor(scenario.Idle.StreamId, new EventSequenceTokenV2(3)); + await scenario.Accessor.RunConsumerCursor(scenario.Idle); + + Assert.Equal(3, scenario.Idle.LastProcessedToken?.SequenceNumber); + Assert.True(scenario.Idle.IsCaughtUp); + await scenario.AssertCheckpoint(failCursor ? 3 : 200); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_InFlightDeliveryConstrainsProgressUntilAcknowledged(bool acknowledge) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + var consumer = new RecordingConsumer(); + scenario.Busy.StreamConsumer = consumer; + scenario.Busy.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Busy, 200)); + scenario.Busy.State = StreamConsumerDataState.Inactive; + var delivery = scenario.Accessor.RunConsumerCursor(scenario.Busy); + + try + { + await consumer.Delivered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (acknowledge) + { + consumer.ReleaseDelivery(); + await delivery; + } + + await scenario.AssertCheckpoint(acknowledge ? 200 : 2); + } + finally + { + consumer.ReleaseDelivery(); + await delivery; + } + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("undelivered")] + [InlineData("incompatible")] + [InlineData("pending-registration")] + public async Task Checkpoint_UncertainRemovalUsesRemainingOriginalProgress(string condition) + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 2)); + scenario.Idle.State = StreamConsumerDataState.Active; + await scenario.Read((scenario.Idle, 200)); + + if (condition == "incompatible") + { + scenario.Idle.State = StreamConsumerDataState.Inactive; + scenario.Idle.LastProcessedToken = new IsolatedProviderToken(200); + } + else if (condition == "pending-registration") + { + var streams = await scenario.Accessor.GetPubSubCache(); + streams[scenario.Idle.StreamId].RegistrationTask = new TaskCompletionSource().Task; + } + + await scenario.Remove(scenario.Idle); + await scenario.AssertCheckpoint(2); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_NoSubscriptionsDoNotManufactureProgress(bool previouslySubscribed) + { + await using var scenario = await CreateCheckpointScenario(); + if (!previouslySubscribed) + { + var streams = await scenario.Accessor.GetPubSubCache(); + streams[scenario.Idle.StreamId].RemoveConsumer(scenario.Idle.SubscriptionId, NullLogger.Instance); + streams[scenario.Busy.StreamId].RemoveConsumer(scenario.Busy.SubscriptionId, NullLogger.Instance); + } + + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + if (previouslySubscribed) + { + await scenario.Remove(scenario.Idle); + await scenario.Remove(scenario.Busy); + } + + scenario.Checkpoints.Clear(); + await scenario.Accessor.Shutdown(); + Assert.Null(Assert.Single(scenario.Checkpoints)); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData("ordered", 300L)] + [InlineData("unordered", 1L)] + [InlineData("unordered-batch", 1L)] + [InlineData("incompatible", 1L)] + [InlineData("missing-token", 1L)] + [InlineData("null-batch", 1L)] + public async Task Checkpoint_UncertainReadUsesOriginalProgress(string condition, long expected) + { + await using var scenario = await CreateCheckpointScenario(emptyCache: new RecordingQueueCache()); + scenario.Idle.LastProcessedToken = new EventSequenceTokenV2(1); + scenario.Busy.LastProcessedToken = new EventSequenceTokenV2(200); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + + var other = new QualifiedStreamId("provider", StreamId.Create("other", Guid.NewGuid())); + await scenario.Accessor.RegisterStream(other, new EventSequenceTokenV2(201), DateTime.UtcNow); + StreamSequenceToken? token = condition switch + { + "unordered" => new EventSequenceTokenV2(100), + "incompatible" => new IsolatedProviderToken(300), + "missing-token" => null, + _ => new EventSequenceTokenV2(300), + }; + IList batches = condition == "null-batch" + ? [null!] + : [new TestBatchContainer(other.StreamId, token!)]; + if (condition == "unordered-batch") + { + batches.Add(new TestBatchContainer(other.StreamId, new EventSequenceTokenV2(250))); + } + + await scenario.Read(batches); + await scenario.AssertCheckpoint(expected); + } + + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Checkpoint_IncompatibleReadBoundaryStillAllowsOriginalSubscriberToken() + { + await using var scenario = await CreateCheckpointScenario(); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + await scenario.Remove(scenario.Busy); + scenario.Idle.LastProcessedToken = new IsolatedProviderToken(10); + + await scenario.AssertCheckpoint(10); + Assert.IsType(Assert.Single(scenario.Checkpoints)); + } + + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(false)] + [InlineData(true)] + public async Task Checkpoint_ReinitializeClearsReadBoundaryAndOptimizationOptOut(bool readAgain) + { + await using var scenario = await CreateCheckpointScenario(emptyCache: new RecordingQueueCache()); + scenario.Idle.LastProcessedToken = new EventSequenceTokenV2(1); + scenario.Busy.LastProcessedToken = new EventSequenceTokenV2(200); + await scenario.Read((scenario.Idle, 1), (scenario.Busy, 200)); + scenario.Idle.SafeDisposeCursor(NullLogger.Instance); + scenario.Idle.Cursor = new InvalidMoveCursor(); + await Assert.ThrowsAnyAsync( + () => scenario.Accessor.RunConsumerCursor(scenario.Idle)); + await scenario.Accessor.Shutdown(); + await InitializeAgent((PersistentStreamPullingAgent)scenario.Accessor); + + await scenario.Accessor.RegisterStream(scenario.Idle.StreamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + await scenario.Accessor.RegisterStream(scenario.Busy.StreamId, new EventSequenceTokenV2(200), DateTime.UtcNow); + var streams = await scenario.Accessor.GetPubSubCache(); + var restarted = scenario with + { + Idle = AddCheckpointConsumer(streams[scenario.Idle.StreamId], scenario.Idle.StreamId, scenario.Cache), + Busy = AddCheckpointConsumer(streams[scenario.Busy.StreamId], scenario.Busy.StreamId, scenario.Cache), + }; + restarted.Idle.LastProcessedToken = streams[scenario.Idle.StreamId].LastReadToken = new EventSequenceTokenV2(1); + restarted.Busy.LastProcessedToken = streams[scenario.Busy.StreamId].LastReadToken = new EventSequenceTokenV2(200); + restarted.Idle.IsCaughtUp = restarted.Busy.IsCaughtUp = true; + + if (readAgain) + { + await restarted.Read((restarted.Idle, 1), (restarted.Busy, 200)); + } + + await restarted.AssertCheckpoint(readAgain ? 200 : 1); + } + + private static async Task CreateCheckpointScenario( + bool pooled = false, int batchSize = 1, bool filtered = false, RecordingQueueCache? emptyCache = null, + TimeProvider? timeProvider = null) + { + IQueueCache cache; + List checkpoints; + if (emptyCache is not null) + { + cache = emptyCache; + checkpoints = emptyCache.DeliveryProgressTokens; + } + else if (pooled) + { + var pooledCache = new PurgeablePooledQueueCache(); + cache = pooledCache; + checkpoints = pooledCache.DeliveryProgressTokens; + } + else + { + var simpleCache = new RecordingSimpleQueueCache(); + cache = simpleCache; + checkpoints = simpleCache.DeliveryProgressTokens; + } + + var idleId = new QualifiedStreamId("provider", StreamId.Create("idle", Guid.NewGuid())); + var busyId = new QualifiedStreamId("provider", StreamId.Create("busy", Guid.NewGuid())); + var (accessor, _, idleStream) = await CreateInitializedAgentWithStream( + idleId, new EventSequenceTokenV2(1), cache, + new StreamPullingAgentOptions { BatchContainerBatchSize = batchSize }, + filtered ? Substitute.For() : null, timeProvider); + await accessor.RegisterStream(busyId, new EventSequenceTokenV2(2), DateTime.UtcNow); + var busyStream = (await accessor.GetPubSubCache())[busyId]; + + return new(accessor, cache, checkpoints, + AddCheckpointConsumer(idleStream, idleId, cache), AddCheckpointConsumer(busyStream, busyId, cache)); + } + + private static StreamConsumerData AddCheckpointConsumer( + StreamConsumerCollection stream, QualifiedStreamId id, IQueueCache cache) + { + var consumer = stream.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), id, new ImmediateRecordingConsumer(), null, DateTime.UtcNow); + consumer.IsRegistered = true; + consumer.Cursor = cache.GetCacheCursor(id, null); + return consumer; + } + + private sealed record CheckpointScenario( + PersistentStreamPullingAgent.ITestAccessor Accessor, + IQueueCache Cache, + List Checkpoints, + StreamConsumerData Idle, + StreamConsumerData Busy) : IAsyncDisposable + { + public Task Read(params (StreamConsumerData Consumer, long Sequence)[] messages) + => Read(messages.Select(message => (IBatchContainer)new TestBatchContainer( + message.Consumer.StreamId.StreamId, new EventSequenceTokenV2(message.Sequence))).ToList()); + + public async Task Read(IList messages) + { + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(messages)); + await Accessor.ReadFromQueue(QueueId.GetQueueId("queue", 0u, 0u), receiver, 1000); + } + + public Task Remove(StreamConsumerData consumer) + { + var agent = (PersistentStreamPullingAgent)Accessor; + return agent.RunOrQueueTask(() => agent.RemoveSubscriber( + consumer.SubscriptionId, consumer.StreamId, TestContext.Current.CancellationToken)); + } + + public async Task AssertCheckpoint(long expected) + { + Checkpoints.Clear(); + await Accessor.Shutdown(); + Assert.Equal(expected, Assert.Single(Checkpoints)?.SequenceNumber); + } + + public ValueTask DisposeAsync() => new(Accessor.Shutdown()); + } + private static Task InitializeAgent(PersistentStreamPullingAgent agent) => agent.RunOrQueueTask(() => agent.Initialize(TestContext.Current.CancellationToken)); @@ -2536,7 +3263,9 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => QualifiedStreamId streamId, StreamSequenceToken registrationToken, IQueueCache queueCache, - StreamPullingAgentOptions options) + StreamPullingAgentOptions options, + IStreamFilter? filter = null, + TimeProvider? timeProvider = null) { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default) @@ -2549,7 +3278,9 @@ private static Task InitializeAgent(PersistentStreamPullingAgent agent) => pubSub, QueueId.GetQueueId("queue", 0u, 0u), queueAdapterCache: queueAdapterCache, - options: options); + timeProvider: timeProvider, + options: options, + filter: filter); var accessor = (PersistentStreamPullingAgent.ITestAccessor)agent; await InitializeAgent(agent); await accessor.RegisterStream(streamId, registrationToken, DateTime.UtcNow);