diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs index 0f023d86359..d878aad8e27 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs @@ -256,7 +256,7 @@ private async Task LoadCore(CancellationToken cancellationToken) } if (!item.TryGetValue(CheckpointAttribute, out var checkpoint) - || string.IsNullOrEmpty(checkpoint.S) + || checkpoint.S is null || !item.TryGetValue(VersionAttribute, out var version) || !long.TryParse(version.N, NumberStyles.None, CultureInfo.InvariantCulture, out _version) || _version <= 0) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 56ba751fb24..4297dd0a647 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -4,74 +4,80 @@ using Orleans.Configuration; using Orleans.Streams; -namespace Orleans.Streaming.Kinesis +namespace Orleans.Streaming.Kinesis; + +/// +/// Persists stream queue checkpoints in DynamoDB. +/// +internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer { - /// - /// Persists stream queue checkpoints in DynamoDB. - /// - internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer + private readonly GrainStreamQueueCheckpointer _inner; + + internal DynamoDBStreamQueueCheckpointer( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options) { - private readonly GrainStreamQueueCheckpointer _inner; + _inner = new GrainStreamQueueCheckpointer( + new StreamCheckpointStoreAdapter(store), + new GrainStreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = options.PersistInterval, + }); + } - internal DynamoDBStreamQueueCheckpointer( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options) - { - _inner = new GrainStreamQueueCheckpointer( - new StreamCheckpointStoreAdapter(store), - new GrainStreamQueueCheckpointerOptions - { - CheckpointComparer = StreamCheckpointComparers.Numeric, - PersistInterval = options.PersistInterval, - }); - } + /// + public bool CheckpointExists => _inner.CheckpointExists; - /// - public bool CheckpointExists => _inner.CheckpointExists; + internal static async Task> Create( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options) + => await Create(store, options, CancellationToken.None); - internal static async Task> Create( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options) - => await Create(store, options, CancellationToken.None); + internal static async Task> Create( + IDynamoDBStreamCheckpointStore store, + DynamoDBStreamQueueCheckpointerOptions options, + CancellationToken cancellationToken) + { + var result = new DynamoDBStreamQueueCheckpointer(store, options); + _ = await result.Load(cancellationToken); + return result; + } - internal static async Task> Create( - IDynamoDBStreamCheckpointStore store, - DynamoDBStreamQueueCheckpointerOptions options, - CancellationToken cancellationToken) - { - var result = new DynamoDBStreamQueueCheckpointer(store, options); - _ = await result.Load(cancellationToken); - return result; - } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + + /// + public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Load() => Load(CancellationToken.None); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); - /// - public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); - /// - [Obsolete("Use the overload which accepts a CancellationToken.")] - public void Update(string offset, DateTime utcNow) - => Update(offset, utcNow, CancellationToken.None); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); - /// - public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) - => _inner.Update(offset, utcNow, cancellationToken); + /// + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); - /// - public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + /// + public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); - private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain - { - public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); + private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain + { + public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); - public ValueTask Update( - string checkpoint, - string expectedCheckpoint, - CancellationToken cancellationToken) - => store.Update(checkpoint, expectedCheckpoint, cancellationToken); - } + public ValueTask Update( + string checkpoint, + string expectedCheckpoint, + CancellationToken cancellationToken) + => store.Update(checkpoint, expectedCheckpoint, cancellationToken); } } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs index ff76c59de58..04dc841f450 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs @@ -77,6 +77,7 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public async Task> Create(string partition) => await Create(partition, CancellationToken.None); diff --git a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs index 63a904a0cbc..e62c2461618 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs @@ -12,5 +12,6 @@ internal enum OrleansEventHubErrorCode FailedPartitionRead = ServiceBus + 1, RetryReceiverInit = ServiceBus + 2, + FailedPartitionRecovery = ServiceBus + 3, } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs index be76b2e5df1..183645aa2a2 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs @@ -1,52 +1,49 @@ -using Orleans.Streams; using System; +using System.Threading; using System.Threading.Tasks; +using Orleans.Streams; + +namespace Orleans.Streaming.EventHubs.Testing; -namespace Orleans.Streaming.EventHubs.Testing +public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory { - /// - /// Creates checkpointers which keep no persistent checkpoint state. - /// - public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory + public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); + + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled>(cancellationToken) + : Task.FromResult>(NoOpCheckpointer.Instance); +} +/// +/// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, +/// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. +/// So a checkpointer with no ops should be enough. +/// +public class NoOpCheckpointer : IStreamQueueCheckpointer +{ + public static NoOpCheckpointer Instance = new NoOpCheckpointer(); + + public bool CheckpointExists => false; + public Task Load() { - /// - /// Gets the shared factory instance. - /// - public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory(); - - /// - public Task> Create(string partition) - { - return Task.FromResult>(NoOpCheckpointer.Instance); - } + return Task.FromResult(EventHubConstants.StartOfStream); } - /// - /// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider, - /// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there. - /// So a checkpointer with no ops should be enough. - /// - public class NoOpCheckpointer : IStreamQueueCheckpointer + + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Task.CompletedTask; + + public Task Reset(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + + public void Update(string offset, DateTime utcNow) { - /// - /// Gets the shared checkpointer instance. - /// - public static NoOpCheckpointer Instance = new NoOpCheckpointer(); - - /// - public bool CheckpointExists => true; - - /// - public Task Load() - { - return Task.FromResult(EventHubConstants.StartOfStream); - } - /// - /// Ignores a checkpoint update because generated-event streams do not persist checkpoints. - /// - /// The ignored offset. - /// The ignored update time. - public void Update(string offset, DateTime utcNow) - { - } } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 7c69c12b365..12407b99099 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -6,220 +6,497 @@ using Orleans.Configuration; using Orleans.Streaming.EventHubs; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Persists stream queue checkpoints using Azure Table Storage. +/// +public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer { + private readonly AzureTableDataManager _dataManager; + private readonly TimeSpan _persistInterval; + private readonly IComparer? _checkpointComparer; + private readonly object _lock = new(); + + private StreamQueueCheckpointEntity _entity; + private Task _inProgressSave = Task.CompletedTask; + private DateTime? _throttleSavesUntilUtc; + private string _latestCheckpoint = string.Empty; + private string _persistedCheckpoint = string.Empty; + private int _pendingResetCount; + private long _updateGeneration; + + private AzureTableStreamQueueCheckpointer( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + IComparer? defaultComparer = null, + string? partitionKeyPrefix = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName); + ArgumentException.ThrowIfNullOrWhiteSpace(partition); + ArgumentNullException.ThrowIfNull(loggerFactory); + if (options.PersistInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(options), + options.PersistInterval, + $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); + } + + _persistInterval = options.PersistInterval; + _checkpointComparer = options.CheckpointComparer ?? defaultComparer; + _dataManager = new AzureTableDataManager( + options, + loggerFactory.CreateLogger()); + _entity = StreamQueueCheckpointEntity.Create( + partitionKeyPrefix ?? options.PartitionKeyPrefix, + streamProviderName, + serviceId, + partition); + LogCreatingCheckpointer( + loggerFactory.CreateLogger(), + partition, + streamProviderName, + serviceId); + } + + /// + public bool CheckpointExists + { + get + { + lock (_lock) + { + return !string.IsNullOrEmpty(_latestCheckpoint); + } + } + } + /// - /// Persists stream queue checkpoints using Azure Table Storage. + /// Creates and initializes an Azure Table stream queue checkpointer. /// - public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer + [Obsolete("Use the overload which accepts a CancellationToken.")] + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory) + => Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + + /// + /// Creates and initializes an Azure Table stream queue checkpointer. + /// + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) + { + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: cancellationToken); + } + + internal static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + IComparer? defaultComparer, + string? partitionKeyPrefix = null, + CancellationToken cancellationToken = default) + { + var checkpointer = new AzureTableStreamQueueCheckpointer( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer, + partitionKeyPrefix); + await checkpointer._dataManager.InitTableAsync(cancellationToken); + return checkpointer; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + + /// + public async Task Load(CancellationToken cancellationToken) { - private readonly AzureTableDataManager _dataManager; - private readonly TimeSpan _persistInterval; - private readonly IComparer? _checkpointComparer; - private readonly object _lock = new(); - - private StreamQueueCheckpointEntity _entity; - private Task _inProgressSave = Task.CompletedTask; - private DateTime? _throttleSavesUntilUtc; - private string _latestCheckpoint = string.Empty; - private string _persistedCheckpoint = string.Empty; - - private AzureTableStreamQueueCheckpointer( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory, - IComparer? defaultComparer = null, - string? partitionKeyPrefix = null) - { - ArgumentNullException.ThrowIfNull(options); - ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName); - ArgumentException.ThrowIfNullOrWhiteSpace(partition); - ArgumentNullException.ThrowIfNull(loggerFactory); - if (options.PersistInterval <= TimeSpan.Zero) - { - throw new ArgumentOutOfRangeException( - nameof(options), - options.PersistInterval, - $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); - } - - _persistInterval = options.PersistInterval; - _checkpointComparer = options.CheckpointComparer ?? defaultComparer; - _dataManager = new AzureTableDataManager( - options, - loggerFactory.CreateLogger()); - _entity = StreamQueueCheckpointEntity.Create( - partitionKeyPrefix ?? options.PartitionKeyPrefix, - streamProviderName, - serviceId, - partition); - LogCreatingCheckpointer( - loggerFactory.CreateLogger(), - partition, - streamProviderName, - serviceId); - } - - /// - public bool CheckpointExists - { - get + var result = await _dataManager.ReadSingleTableEntryAsync( + _entity.PartitionKey, + _entity.RowKey, + cancellationToken); + var checkpoint = result.Entity?.Offset ?? string.Empty; + lock (_lock) + { + if (result.Entity is not null) { + _entity = result.Entity; + } + + _latestCheckpoint = checkpoint; + _persistedCheckpoint = checkpoint; + } + + return checkpoint; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) + { + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + lock (_lock) + { + _pendingResetCount++; + _throttleSavesUntilUtc = DateTime.MaxValue; + var inProgressSave = _inProgressSave; + _inProgressSave = completion.Task; + RunReset( + completion, + inProgressSave, + _inProgressSave, + _latestCheckpoint, + _updateGeneration, + cancellationToken).Ignore(); + completion.Task.Ignore(); + } + + return completion.Task.WaitAsync(cancellationToken); + } + + private async Task RunReset( + TaskCompletionSource completion, + Task inProgressSave, + Task resetTask, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await ResetCore( + inProgressSave, + enqueuedCheckpoint, + updateGeneration, + cancellationToken); + + while (true) + { + string checkpoint; lock (_lock) { - return !string.IsNullOrEmpty(_latestCheckpoint); + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + completion.TrySetResult(); + return; + } + + checkpoint = _latestCheckpoint; } + + await Save(checkpoint, cancellationToken); } } - - /// - /// Creates and initializes an Azure Table stream queue checkpointer. - /// - public static Task> Create( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory) + catch (OperationCanceledException exception) { - return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: true); } - - internal static async Task> Create( - AzureTableStreamCheckpointerOptions options, - string streamProviderName, - string partition, - string serviceId, - ILoggerFactory loggerFactory, - IComparer? defaultComparer, - string? partitionKeyPrefix = null) + catch (Exception exception) { - var checkpointer = new AzureTableStreamQueueCheckpointer( - options, - streamProviderName, - partition, - serviceId, - loggerFactory, - defaultComparer, - partitionKeyPrefix); - await checkpointer._dataManager.InitTableAsync(); - return checkpointer; + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: false); } + } - /// - public async Task Load() + private async Task CompleteFailedReset( + TaskCompletionSource completion, + Task resetTask, + Exception resetFailure, + bool canceled) + { + try { - var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); - var checkpoint = result.Entity?.Offset ?? string.Empty; - lock (_lock) + while (true) { - if (result.Entity is not null) + string checkpoint; + lock (_lock) { - _entity = result.Entity; + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + if (canceled) + { + completion.TrySetCanceled( + ((OperationCanceledException)resetFailure).CancellationToken); + } + else + { + completion.TrySetException(resetFailure); + } + + return; + } + + checkpoint = _latestCheckpoint; } - _latestCheckpoint = checkpoint; - _persistedCheckpoint = checkpoint; + await Save(checkpoint, CancellationToken.None); } - - return checkpoint; } - - /// - public void Update(string offset, DateTime utcNow) + catch (Exception persistenceFailure) { - ArgumentNullException.ThrowIfNull(offset); - lock (_lock) { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_checkpointComparer is { } comparer - && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) - { - return; - } + CompleteReset(); + completion.TrySetException( + new AggregateException(resetFailure, persistenceFailure)); + } + } + } + + private void CompleteReset() + { + _pendingResetCount--; + if (_pendingResetCount == 0) + { + _throttleSavesUntilUtc = null; + } + } + + /// + public void Update(string offset, DateTime utcNow) + { + ArgumentNullException.ThrowIfNull(offset); - _latestCheckpoint = offset; - if (_throttleSavesUntilUtc.HasValue - && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) + lock (_lock) + { + if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal)) + { + if (string.Equals(_persistedCheckpoint, offset, StringComparison.Ordinal) + || !_inProgressSave.IsCompleted + || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) { return; } _throttleSavesUntilUtc = utcNow + _persistInterval; - _inProgressSave = Save(offset); + _inProgressSave = Save(offset, CancellationToken.None); _inProgressSave.Ignore(); + return; } + + if (_checkpointComparer is { } comparer + && !string.IsNullOrEmpty(_latestCheckpoint) + && comparer.Compare(offset, _latestCheckpoint) <= 0) + { + return; + } + + _latestCheckpoint = offset; + _updateGeneration++; + _entity.Offset = offset; + if (!_inProgressSave.IsCompleted + || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) + { + return; + } + + _throttleSavesUntilUtc = utcNow + _persistInterval; + _inProgressSave = Save(offset, CancellationToken.None); + _inProgressSave.Ignore(); } + } - /// - public async Task FlushAsync(CancellationToken cancellationToken) + /// + public async Task FlushAsync(CancellationToken cancellationToken) + { + var retryingSave = false; + while (true) { - var retryingSave = false; - while (true) + Task inProgressSave; + lock (_lock) { - Task inProgressSave; - lock (_lock) - { - inProgressSave = _inProgressSave; - } + inProgressSave = _inProgressSave; + } - if (retryingSave) + if (retryingSave) + { + await inProgressSave.WaitAsync(cancellationToken); + } + else + { + try { await inProgressSave.WaitAsync(cancellationToken); } - else + catch (Exception) when (!cancellationToken.IsCancellationRequested) { - try - { - await inProgressSave.WaitAsync(cancellationToken); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) - { - } - - cancellationToken.ThrowIfCancellationRequested(); } - lock (_lock) - { - if (!ReferenceEquals(inProgressSave, _inProgressSave)) - { - retryingSave = false; - continue; - } + cancellationToken.ThrowIfCancellationRequested(); + } - if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) - { - return; - } + lock (_lock) + { + if (!ReferenceEquals(inProgressSave, _inProgressSave)) + { + retryingSave = false; + continue; + } - _inProgressSave = Save(_latestCheckpoint); - retryingSave = true; + if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + return; } + + _inProgressSave = Save(_latestCheckpoint, cancellationToken); + _inProgressSave.Ignore(); + retryingSave = true; } } + } - private async Task Save(string checkpoint) + private async Task Save(string checkpoint, CancellationToken cancellationToken) + { + var entity = CreateWriteEntity(checkpoint); + await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); + lock (_lock) { _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity); + _persistedCheckpoint = checkpoint; + } + } + + private async Task ResetCore( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await inProgressSave; + } + catch + { + } + + cancellationToken.ThrowIfCancellationRequested(); + + string rollbackCheckpoint; + lock (_lock) + { + if (_updateGeneration <= updateGeneration) + { + rollbackCheckpoint = _latestCheckpoint; + _latestCheckpoint = string.Empty; + } + else + { + rollbackCheckpoint = enqueuedCheckpoint; + } + } + + try + { + var entity = CreateWriteEntity(string.Empty); + await _dataManager.UpsertTableEntryAsync(entity, cancellationToken); lock (_lock) { - _persistedCheckpoint = checkpoint; + _persistedCheckpoint = string.Empty; + if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _entity.Offset = string.Empty; + } } } + catch + { + lock (_lock) + { + RestoreLatestCheckpoint(rollbackCheckpoint); + } + + throw; + } + } + + private void RestoreLatestCheckpoint(string latestCheckpoint) + { + if (_checkpointComparer is { } comparer + && !string.IsNullOrEmpty(_latestCheckpoint) + && !string.IsNullOrEmpty(latestCheckpoint)) + { + if (comparer.Compare(_latestCheckpoint, latestCheckpoint) < 0) + { + _latestCheckpoint = latestCheckpoint; + } + + return; + } + + if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _latestCheckpoint = latestCheckpoint; + } + } - [LoggerMessage( - Level = LogLevel.Information, - Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] - private static partial void LogCreatingCheckpointer( - ILogger logger, - string partition, - string streamProviderName, - string serviceId); + private StreamQueueCheckpointEntity CreateWriteEntity(string checkpoint) + { + lock (_lock) + { + return new StreamQueueCheckpointEntity + { + PartitionKey = _entity.PartitionKey, + RowKey = _entity.RowKey, + Offset = checkpoint, + }; + } } + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")] + private static partial void LogCreatingCheckpointer( + ILogger logger, + string partition, + string streamProviderName, + string serviceId); } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs index 6aae03ee714..69f24caf8fe 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -6,56 +7,63 @@ using Orleans.Configuration; using Orleans.Configuration.Overrides; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Creates Azure Table stream queue checkpointers. +/// +public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory { + private readonly string _providerName; + private readonly AzureTableStreamCheckpointerOptions _options; + private readonly ClusterOptions _clusterOptions; + private readonly ILoggerFactory _loggerFactory; + /// - /// Creates Azure Table stream queue checkpointers. + /// Initializes a new instance. /// - public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory + public AzureTableStreamQueueCheckpointerFactory( + string providerName, + AzureTableStreamCheckpointerOptions options, + IOptions clusterOptions, + ILoggerFactory loggerFactory) { - private readonly string _providerName; - private readonly AzureTableStreamCheckpointerOptions _options; - private readonly ClusterOptions _clusterOptions; - private readonly ILoggerFactory _loggerFactory; + _providerName = providerName; + _options = options; + _clusterOptions = clusterOptions.Value; + _loggerFactory = loggerFactory; + } - /// - /// Initializes a new instance. - /// - public AzureTableStreamQueueCheckpointerFactory( - string providerName, - AzureTableStreamCheckpointerOptions options, - IOptions clusterOptions, - ILoggerFactory loggerFactory) - { - _providerName = providerName; - _options = options; - _clusterOptions = clusterOptions.Value; - _loggerFactory = loggerFactory; - } + /// + /// Creates a factory from a service provider. + /// + public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) + { + var options = services.GetOptionsByName(providerName); + var clusterOptions = services.GetProviderClusterOptions(providerName); + return ActivatorUtilities.CreateInstance( + services, + providerName, + options, + clusterOptions); + } - /// - /// Creates a factory from a service provider. - /// - public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) - { - var options = services.GetOptionsByName(providerName); - var clusterOptions = services.GetProviderClusterOptions(providerName); - return ActivatorUtilities.CreateInstance( - services, - providerName, - options, - clusterOptions); - } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> Create(string partition) + => Create(partition, CancellationToken.None); - /// - public Task> Create(string partition) - { - return AzureTableStreamQueueCheckpointer.Create( - _options, - _providerName, - partition, - _clusterOptions.ServiceId.ToString(), - _loggerFactory); - } + /// + public Task> Create( + string partition, + CancellationToken cancellationToken) + { + return AzureTableStreamQueueCheckpointer.Create( + _options, + _providerName, + partition, + _clusterOptions.ServiceId.ToString(), + _loggerFactory, + cancellationToken); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs index 3489a9a8d12..8642c99857e 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs @@ -52,6 +52,7 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly IQueueAdapterReceiverMonitor monitor; private readonly LoadSheddingOptions loadSheddingOptions; private readonly IEnvironmentStatisticsProvider environmentStatisticsProvider; + private readonly object cacheLock = new(); private IEventHubQueueCache? cache; private IEventHubReceiver? receiver; @@ -59,17 +60,32 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa private readonly Func eventHubReceiverFactory; private IStreamQueueCheckpointer? checkpointer; - private AggregatedQueueFlowController flowController = null!; + private AggregatedQueueFlowController flowController = new(MaxMessagesPerRead); + private bool receiverUsesCheckpoint; + private IEventHubQueueCache? recoveryCache; + private Dictionary? recoveredCursorProgress; + private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance); + private HashSet? recoveryPendingCursors; + private Task? recoveryTask; + private readonly SemaphoreSlim initializationLock = new(1, 1); + private readonly SemaphoreSlim receiverCloseLock = new(1, 1); + private readonly SemaphoreSlim shutdownLock = new(1, 1); + private IEventHubReceiver? receiverPendingClose; + private Task? receiverCloseTask; // Receiver life cycle private int receiverState = ReceiverShutdown; private const int ReceiverShutdown = 0; private const int ReceiverRunning = 1; + private const int ReceiverShutdownCompleted = 2; public int GetMaxAddCount() { - return this.flowController.GetMaxAddCount(); + lock (this.cacheLock) + { + return this.flowController.GetMaxAddCount(); + } } public EventHubAdapterReceiver(EventHubPartitionSettings settings, @@ -112,18 +128,39 @@ public EventHubAdapterReceiver(EventHubPartitionSettings settings, this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory; } - public async Task Initialize(TimeSpan timeout) + public Task Initialize(TimeSpan timeout) + => Initialize(timeout, CancellationToken.None); + + public async Task Initialize( + TimeSpan timeout, + CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCancellation.CancelAfter(timeout); LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); + try + { + await this.shutdownLock.WaitAsync(timeoutCancellation.Token); + try + { + lock (this.cacheLock) + { + this.receiverState = ReceiverRunning; + } + } + finally + { + this.shutdownLock.Release(); + } - // if receiver was already running, do nothing - if (ReceiverRunning == Interlocked.Exchange(ref this.receiverState, ReceiverRunning)) + await EnsureInitialized(timeoutCancellation.Token); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - return; + cancellationToken.ThrowIfCancellationRequested(); + throw; } - - using var cancellation = new CancellationTokenSource(timeout); - await Initialize(cancellation.Token); } /// @@ -131,37 +168,170 @@ public async Task Initialize(TimeSpan timeout) /// it will be retried when messages are requested /// /// - private async Task Initialize(CancellationToken cancellationToken) + private async Task EnsureInitialized(CancellationToken cancellationToken) + { + while (true) + { + if (Volatile.Read(ref this.receiverPendingClose) is not null) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + } + + Task? recoveryTask; + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning || this.receiver is not null) + { + return; + } + + recoveryTask = this.recoveryTask; + } + + if (recoveryTask is not null) + { + await recoveryTask.WaitAsync(cancellationToken); + continue; + } + + await this.initializationLock.WaitAsync(cancellationToken); + try + { + bool hasPendingClose; + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning || this.receiver is not null) + { + return; + } + + recoveryTask = this.recoveryTask; + hasPendingClose = this.receiverPendingClose is not null; + } + + if (hasPendingClose) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + continue; + } + + if (recoveryTask is null) + { + await InitializeCore(cancellationToken); + return; + } + } + finally + { + this.initializationLock.Release(); + } + + await recoveryTask.WaitAsync(cancellationToken); + } + } + + private async Task InitializeCore(CancellationToken cancellationToken) { var watch = Stopwatch.StartNew(); + IEventHubQueueCache? cache = null; + IEventHubReceiver? receiver = null; + var installed = false; + var failures = new List(); try { - this.checkpointer = await this.checkpointerFactory( + var checkpointer = await this.checkpointerFactory( this.settings.Partition, cancellationToken); - if (this.cache != null) - { - this.cache.Dispose(); - this.cache = null; - } - this.cache = this.cacheFactory(this.settings.Partition, this.checkpointer, this.loggerFactory); - this.flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) { this.cache, LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) }; - string offset = await this.checkpointer.Load(cancellationToken); - if (!this.checkpointer.CheckpointExists) + string offset = await checkpointer.Load(cancellationToken); + var receiverUsesCheckpoint = checkpointer.CheckpointExists; + if (!receiverUsesCheckpoint) { offset = EventHubConstants.StartOfStream; } - this.receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger); - watch.Stop(); - this.monitor?.TrackInitialization(true, watch.Elapsed, null); + cache = this.cacheFactory(this.settings.Partition, checkpointer, this.loggerFactory); + var flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) + { + cache, + LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) + }; + receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger); + cancellationToken.ThrowIfCancellationRequested(); + + lock (this.cacheLock) + { + if (this.receiverState == ReceiverRunning) + { + this.cache?.Dispose(); + this.checkpointer = checkpointer; + this.cache = cache; + this.flowController = flowController; + this.receiverUsesCheckpoint = receiverUsesCheckpoint; + this.receiver = receiver; + if (this.recoveredCursorProgress is not null) + { + this.recoveryCache = cache; + } + + installed = true; + } + } } catch (Exception ex) { - watch.Stop(); - this.monitor?.TrackInitialization(false, watch.Elapsed, ex); - throw; + failures.Add(ex); + } + + if (!installed) + { + try + { + cache?.Dispose(); + } + catch (Exception exception) + { + failures.Add(exception); + } + + try + { + if (receiver is not null) + { + lock (this.cacheLock) + { + if (this.receiverPendingClose is not null + && !ReferenceEquals(this.receiverPendingClose, receiver)) + { + throw new InvalidOperationException( + "A previous Event Hub receiver is still pending cleanup."); + } + + this.receiverPendingClose = receiver; + this.receiverCloseTask = null; + } + + await ClosePendingReceiverWithTimeout(); + } + } + catch (Exception exception) + { + failures.Add(exception); + } } + + watch.Stop(); + if (failures.Count == 0) + { + this.monitor?.TrackInitialization(true, watch.Elapsed, null); + return; + } + + var failure = failures.Count == 1 + ? failures[0] + : new AggregateException(failures); + this.monitor?.TrackInitialization(false, watch.Elapsed, failure); + ExceptionDispatchInfo.Capture(failure).Throw(); + throw new UnreachableException(); } [Obsolete("Use the overload which accepts a CancellationToken.")] @@ -173,29 +343,48 @@ public async Task> GetQueueMessagesAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (this.receiverState == ReceiverShutdown || maxCount <= 0) + if (this.receiverState != ReceiverRunning || maxCount <= 0) { return new List(); } + var recoveryTask = Volatile.Read(ref this.recoveryTask); + if (recoveryTask is not null) + { + await recoveryTask.WaitAsync(cancellationToken); + } + + if (Volatile.Read(ref this.receiverPendingClose) is not null) + { + await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken); + } + // if receiver initialization failed, retry if (this.receiver == null) { LogWarningRetryingInitializationOfEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); - await Initialize(cancellationToken); + await EnsureInitialized(cancellationToken); if (this.receiver == null) { // should not get here, should throw instead, but just incase. return new List(); } } + + var receiver = Volatile.Read(ref this.receiver); + if (Volatile.Read(ref this.receiverState) != ReceiverRunning || receiver is null) + { + return new List(); + } + + var cache = Volatile.Read(ref this.cache); var watch = Stopwatch.StartNew(); List? messages; try { // Receivers built against older Orleans versions can still return null. - messages = (await this.receiver.ReceiveAsync( + messages = (await receiver.ReceiveAsync( maxCount, ReceiveTimeout, cancellationToken))?.ToList(); @@ -208,6 +397,33 @@ public async Task> GetQueueMessagesAsync( watch.Stop(); this.monitor?.TrackRead(false, watch.Elapsed, ex); LogWarningFailedToReadFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition, ex); + + bool shouldRecover; + lock (this.cacheLock) + { + shouldRecover = this.receiverState == ReceiverRunning + && ReferenceEquals(receiver, this.receiver) + && this.receiverUsesCheckpoint; + } + + if (shouldRecover && IsInvalidOffsetException(ex)) + { + try + { + await ResetReceiver(receiver, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception recoveryException) + { + LogWarningFailedToRecoverFromInvalidCheckpoint( + this.settings.Hub.EventHubName, + this.settings.Partition, + recoveryException); + } + } throw; } @@ -226,7 +442,19 @@ public async Task> GetQueueMessagesAsync( this.monitor?.TrackMessagesReceived(messages.Count, oldestMessageEnqueueTime, newestMessageEnqueueTime); - List messageStreamPositions = this.cache!.Add(messages, dequeueTimeUtc); + List messageStreamPositions; + lock (this.cacheLock) + { + if (!ReferenceEquals(receiver, this.receiver) + || !ReferenceEquals(cache, this.cache) + || cache is null) + { + return batches; + } + + messageStreamPositions = cache.Add(messages, dequeueTimeUtc); + } + foreach (var streamPosition in messageStreamPositions) { batches.Add(new StreamActivityNotificationBatch(streamPosition)); @@ -234,6 +462,216 @@ public async Task> GetQueueMessagesAsync( return batches; } + // The SDK translates this service rejection to ArgumentException, preserving the service description. + private static bool IsInvalidOffsetException(Exception exception) + => exception is ArgumentException + && exception.Message.StartsWith("The supplied offset", StringComparison.OrdinalIgnoreCase) + && exception.Message.Contains(" is invalid.", StringComparison.OrdinalIgnoreCase); + + private async Task ResetReceiver( + IEventHubReceiver failedReceiver, + CancellationToken cancellationToken) + { + Task? existingRecovery; + lock (this.cacheLock) + { + existingRecovery = this.recoveryTask; + } + + if (existingRecovery is not null) + { + await existingRecovery.WaitAsync(cancellationToken); + return; + } + + await this.initializationLock.WaitAsync(cancellationToken); + var releaseInitializationLock = true; + try + { + IStreamQueueCheckpointer checkpointer; + IEventHubQueueCache? cache; + TaskCompletionSource recoveryCompletion; + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning + || !ReferenceEquals(failedReceiver, this.receiver)) + { + return; + } + + recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + recoveryCompletion.Task.Ignore(); + this.recoveryTask = recoveryCompletion.Task; + checkpointer = this.checkpointer!; + this.recoveryCache = null; + this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance); + this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance); + cache = this.cache; + this.cache = null; + this.flowController = new(MaxMessagesPerRead); + this.receiverPendingClose = Interlocked.Exchange(ref this.receiver, null); + this.receiverCloseTask = null; + } + + RunRecovery(checkpointer, cache, recoveryCompletion).Ignore(); + releaseInitializationLock = false; + cancellationToken.ThrowIfCancellationRequested(); + await recoveryCompletion.Task.WaitAsync(cancellationToken); + } + finally + { + if (releaseInitializationLock) + { + this.initializationLock.Release(); + } + } + } + + private async Task RunRecovery( + IStreamQueueCheckpointer checkpointer, + IEventHubQueueCache? cache, + TaskCompletionSource recoveryCompletion) + { + var exceptions = new List(); + try + { + try + { + await checkpointer.Reset(CancellationToken.None); + lock (this.cacheLock) + { + this.receiverUsesCheckpoint = false; + } + + } + catch (Exception exception) + { + exceptions.Add(exception); + } + + try + { + cache?.Dispose(); + } + catch (Exception exception) + { + exceptions.Add(exception); + } + + try + { + await ClosePendingReceiverWithTimeout(); + } + catch (Exception exception) + { + exceptions.Add(exception); + } + + bool isRunning; + lock (this.cacheLock) + { + isRunning = this.receiverState == ReceiverRunning; + } + + if (exceptions.Count == 0 && isRunning) + { + try + { + using var initializationCancellation = new CancellationTokenSource(ReceiveTimeout); + await InitializeCore(initializationCancellation.Token); + } + catch (Exception exception) + { + exceptions.Add(exception); + } + } + } + catch (Exception exception) + { + exceptions.Add(exception); + } + finally + { + if (exceptions.Count == 0) + { + recoveryCompletion.TrySetResult(); + } + else if (exceptions.Count == 1) + { + recoveryCompletion.TrySetException(exceptions[0]); + } + else + { + recoveryCompletion.TrySetException(new AggregateException(exceptions)); + } + + lock (this.cacheLock) + { + if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task)) + { + this.recoveryTask = null; + } + } + + this.initializationLock.Release(); + } + } + + private async Task ClosePendingReceiverWithTimeout() + { + using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout); + await ClosePendingReceiver(cleanupCancellation.Token); + } + + private async Task ClosePendingReceiver(CancellationToken cancellationToken) + { + await this.receiverCloseLock.WaitAsync(cancellationToken); + try + { + IEventHubReceiver? receiver; + Task closeTask; + lock (this.cacheLock) + { + receiver = this.receiverPendingClose; + if (receiver is null) + { + return; + } + + if (this.receiverCloseTask is { IsCompletedSuccessfully: true }) + { + this.receiverPendingClose = null; + this.receiverCloseTask = null; + return; + } + + var existingCloseTask = this.receiverCloseTask; + if (existingCloseTask is null || existingCloseTask.IsCompleted) + { + existingCloseTask = receiver.CloseAsync(cancellationToken); + existingCloseTask.Ignore(); + this.receiverCloseTask = existingCloseTask; + } + + closeTask = existingCloseTask; + } + + await closeTask.WaitAsync(cancellationToken); + lock (this.cacheLock) + { + if (ReferenceEquals(this.receiverPendingClose, receiver)) + { + this.receiverPendingClose = null; + this.receiverCloseTask = null; + } + } + } + finally + { + this.receiverCloseLock.Release(); + } + } + public void AddToCache(IList messages) { // do nothing, we add data directly into cache. No need for agent involvement @@ -243,10 +681,21 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList { purgedItems = null; - //if not under pressure, signal the cache to do a time based purge - //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache - if (!this.IsUnderPressure()) - this.cache!.SignalPurge(); + lock (this.cacheLock) + { + var cache = this.cache; + if (cache is null) + { + return false; + } + + //if not under pressure, signal the cache to do a time based purge + //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache + if (!this.IsUnderPressure()) + { + cache.SignalPurge(); + } + } return false; } @@ -254,27 +703,71 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList [Obsolete("Use IQueueCache.TryGetCacheCursor instead.")] public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) { - return new Cursor(this.cache!, streamId, token); + lock (this.cacheLock) + { + var cache = this.cache + ?? throw new InvalidOperationException("The Event Hub receiver is recovering."); +#pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. + var cursor = cache.GetCursor(streamId, token); +#pragma warning restore CS0618 + return new Cursor( + this, + streamId, + cache, + cursor, + ReferenceEquals(cache, this.recoveryCache) ? token : null); + } } QueueCacheCursorResult IQueueCache.TryGetCacheCursor( StreamId streamId, StreamSequenceToken? token) { - return WrapCursorResult(this.cache!.TryGetCursor(streamId, token)); + lock (this.cacheLock) + { + var cache = this.cache; + if (cache is null) + { + return QueueCacheCursorResult.NotSupported; + } + + return WrapCursorResult( + streamId, + cache, + cache.TryGetCursor(streamId, token), + ReferenceEquals(cache, this.recoveryCache) ? token : null); + } } QueueCacheCursorResult IQueueCache.TryGetCacheCursorAtPosition( StreamId streamId, StreamSubscriptionStartPosition startPosition) { - return WrapCursorResult(this.cache!.TryGetCursorAtPosition(streamId, startPosition)); + lock (this.cacheLock) + { + var cache = this.cache; + if (cache is null) + { + return QueueCacheCursorResult.NotSupported; + } + + return WrapCursorResult( + streamId, + cache, + cache.TryGetCursorAtPosition(streamId, startPosition), + resumeToken: null); + } } - private QueueCacheCursorResult WrapCursorResult(QueueCacheCursorResult result) + private QueueCacheCursorResult WrapCursorResult( + StreamId streamId, + IEventHubQueueCache cache, + QueueCacheCursorResult result, + StreamSequenceToken? resumeToken) => result.Kind switch { - QueueCacheCursorResultKind.Success => QueueCacheCursorResult.FromCursor(new Cursor(this.cache!, result.Cursor!)), + QueueCacheCursorResultKind.Success => QueueCacheCursorResult.FromCursor( + new Cursor(this, streamId, cache, result.Cursor!, resumeToken)), QueueCacheCursorResultKind.CacheMiss => QueueCacheCursorResult.FromCacheMiss(result.CacheMiss!.Value), QueueCacheCursorResultKind.NotSupported => QueueCacheCursorResult.NotSupported, _ => throw new InvalidOperationException("The cursor result is not initialized."), @@ -295,23 +788,103 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke if (earliestSubscriptionToken is IEventHubPartitionLocation location && long.TryParse(location.EventHubOffset, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) { - this.checkpointer?.Update(location.EventHubOffset, utcNow, CancellationToken.None); + lock (this.cacheLock) + { + if (this.receiverState != ReceiverRunning + || this.receiver is null + || this.cache is null) + { + return; + } + + if (this.recoveredCursorProgress is { } recoveredProgress) + { + if (this.recoveryPendingCursors is { Count: > 0 } + || !recoveredProgress.Values.Any( + progress => ReferenceEquals(progress.DeliveredToken, earliestSubscriptionToken) + || ReferenceEquals(progress.ResumeToken, earliestSubscriptionToken))) + { + return; + } + + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + } + + this.checkpointer?.Update( + location.EventHubOffset, + utcNow, + CancellationToken.None); + } + } + } + + private void RecordRecoveredDeliveryToken( + Cursor cursor, + IEventHubQueueCache? cursorCache, + StreamSequenceToken deliveredToken, + StreamSequenceToken? resumeToken) + { + if (this.recoveredCursorProgress is not { } cursorProgress + || !ReferenceEquals(cursorCache, this.recoveryCache)) + { + return; + } + + cursorProgress.TryGetValue(cursor, out var previousProgress); + cursorProgress[cursor] = new( + deliveredToken, + previousProgress.ResumeToken ?? resumeToken); + this.recoveryPendingCursors?.Remove(cursor); + } + + private void ClearRecoveredDeliveryToken(Cursor cursor) + { + this.recoveredCursorProgress?.Remove(cursor); + } + + private readonly record struct RecoveredCursorProgress( + StreamSequenceToken DeliveredToken, + StreamSequenceToken? ResumeToken); + + private void UnregisterCursor(Cursor cursor) + { + lock (this.cacheLock) + { + this.ClearRecoveredDeliveryToken(cursor); + this.recoveryPendingCursors?.Remove(cursor); + this.cursors.Remove(cursor); } } public async Task Shutdown(TimeSpan timeout) { + using var shutdownCancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var shutdownCancellationToken = shutdownCancellation?.Token ?? CancellationToken.None; + await this.shutdownLock.WaitAsync(shutdownCancellationToken); var watch = Stopwatch.StartNew(); + var initializationLockHeld = false; try { - // if receiver was already shutdown, do nothing - if (ReceiverShutdown == Interlocked.Exchange(ref this.receiverState, ReceiverShutdown)) + lock (this.cacheLock) { - return; + if (this.receiverState == ReceiverShutdownCompleted) + { + return; + } + + this.receiverState = ReceiverShutdown; } LogInfoStoppingReadingFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition); + // Recovery owns this lock until reset and detached-resource cleanup finish, even if its pull was canceled. + await this.initializationLock.WaitAsync(shutdownCancellationToken); + initializationLockHeld = true; + var shutdownExceptions = new List(); try @@ -320,8 +893,7 @@ public async Task Shutdown(TimeSpan timeout) // so the latest processed offset is persisted and not replayed on restart. if (this.checkpointer != null) { - using var flushCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - await this.checkpointer.FlushAsync(flushCancellation?.Token ?? CancellationToken.None); + await this.checkpointer.FlushAsync(shutdownCancellationToken); } } catch (Exception ex) @@ -329,31 +901,26 @@ public async Task Shutdown(TimeSpan timeout) shutdownExceptions.Add(ex); } - // clear cache and receiver - IEventHubQueueCache? localCache = Interlocked.Exchange(ref this.cache, null); - + // clear receiver var localReceiver = Interlocked.Exchange(ref this.receiver, null); - - // start closing receiver - Task closeTask = Task.CompletedTask; - using var closeCancellation = timeout == Timeout.InfiniteTimeSpan ? null : new CancellationTokenSource(timeout); - var closeCancellationToken = closeCancellation?.Token ?? CancellationToken.None; - try + lock (this.cacheLock) { - if (localReceiver != null) + if (localReceiver is not null) { - closeTask = localReceiver.CloseAsync(closeCancellationToken); + this.receiverPendingClose = localReceiver; + this.receiverCloseTask = null; } } - catch (Exception ex) - { - shutdownExceptions.Add(ex); - } // dispose of cache try { - localCache?.Dispose(); + lock (this.cacheLock) + { + var localCache = this.cache; + this.cache = null; + localCache?.Dispose(); + } } catch (Exception ex) { @@ -363,7 +930,10 @@ public async Task Shutdown(TimeSpan timeout) // finish return receiver closing task try { - await closeTask.WaitAsync(closeCancellationToken); + using var cleanupCancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + await ClosePendingReceiver(cleanupCancellation?.Token ?? CancellationToken.None); } catch (Exception ex) { @@ -371,6 +941,16 @@ public async Task Shutdown(TimeSpan timeout) } ThrowIfAny(shutdownExceptions); + lock (this.cacheLock) + { + this.checkpointer = null; + this.recoveryCache = null; + this.recoveredCursorProgress = null; + this.recoveryPendingCursors = null; + this.cursors.Clear(); + this.flowController = new(MaxMessagesPerRead); + this.receiverState = ReceiverShutdownCompleted; + } watch.Stop(); this.monitor?.TrackShutdown(true, watch.Elapsed, null); @@ -381,6 +961,15 @@ public async Task Shutdown(TimeSpan timeout) this.monitor?.TrackShutdown(false, watch.Elapsed, ex); throw; } + finally + { + if (initializationLockHeld) + { + this.initializationLock.Release(); + } + + this.shutdownLock.Release(); + } static void ThrowIfAny(List exceptions) { @@ -435,64 +1024,251 @@ public StreamActivityNotificationBatch(StreamPosition position) private class Cursor : IQueueCacheCursor { - private readonly IEventHubQueueCache cache; - private readonly object cursor; + private readonly EventHubAdapterReceiver owner; + private readonly StreamId streamId; + private IEventHubQueueCache? cache; + private object? cursor; private IBatchContainer? current; + private StreamSequenceToken? recoveryResumeToken; + private QueueCacheCursorMoveResult? pendingMoveResult; - public Cursor(IEventHubQueueCache cache, StreamId streamId, StreamSequenceToken? token) + public Cursor(EventHubAdapterReceiver owner, StreamId streamId, StreamSequenceToken? token) { - this.cache = cache; + this.owner = owner; + this.streamId = streamId; + lock (owner.cacheLock) + { + this.cache = owner.cache!; #pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. - this.cursor = cache.GetCursor(streamId, token); + this.cursor = this.cache.GetCursor(streamId, token); #pragma warning restore CS0618 + if (ReferenceEquals(this.cache, owner.recoveryCache)) + { + this.recoveryResumeToken = token; + } + + owner.cursors.Add(this); + } } - public Cursor(IEventHubQueueCache cache, object cursor) + public Cursor( + EventHubAdapterReceiver owner, + StreamId streamId, + IEventHubQueueCache cache, + object cursor, + StreamSequenceToken? resumeToken) { + this.owner = owner; + this.streamId = streamId; this.cache = cache; this.cursor = cursor; + this.recoveryResumeToken = resumeToken; + owner.cursors.Add(this); } public void Dispose() { + this.owner.UnregisterCursor(this); } public IBatchContainer? GetCurrent(out Exception? exception) { - exception = null; - return this.current; + lock (this.owner.cacheLock) + { + exception = null; + if (!ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + } + + return this.current; + } } [Obsolete("Use MoveNextWithResult instead.")] public bool MoveNext() { + lock (this.owner.cacheLock) + { + if (this.pendingMoveResult is { } pendingResult) + { + this.pendingMoveResult = null; + this.current = null; + if (pendingResult.CacheMiss is { } cacheMiss) + { + Invalidate(); + throw cacheMiss.ToException(); + } + + if (pendingResult.Kind == QueueCacheCursorMoveResultKind.NoData) + { + return false; + } + + throw new InvalidOperationException("The cursor move result is not initialized."); + } + + if (this.cache is null || this.cursor is null || !ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + return false; + } + + try + { #pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior. - if (!this.cache.TryGetNextMessage(this.cursor, out var next)) + if (!this.cache.TryGetNextMessage(this.cursor, out var next)) #pragma warning restore CS0618 - { - this.current = null; - return false; - } + { + this.current = null; + return false; + } - this.current = next; - return true; + SetCurrent(next); + return true; + } + catch + { + Invalidate(); + throw; + } + } } public QueueCacheCursorMoveResult MoveNextWithResult() { - var result = this.cache.TryGetNextMessageWithResult(this.cursor, out var next); - this.current = result.Kind == QueueCacheCursorMoveResultKind.Success ? next : null; - return result; + lock (this.owner.cacheLock) + { + if (this.pendingMoveResult is { } pendingResult) + { + this.pendingMoveResult = null; + this.current = null; + if (pendingResult.Kind == QueueCacheCursorMoveResultKind.CacheMiss) + { + Invalidate(); + } + + return pendingResult; + } + + if (this.cache is null || this.cursor is null || !ReferenceEquals(this.cache, this.owner.cache)) + { + this.current = null; + return QueueCacheCursorMoveResult.NoData; + } + + QueueCacheCursorMoveResult result; + IBatchContainer? next; + try + { + result = this.cache.TryGetNextMessageWithResult(this.cursor, out next); + } + catch + { + Invalidate(); + throw; + } + + switch (result.Kind) + { + case QueueCacheCursorMoveResultKind.Success: + SetCurrent(next ?? throw new InvalidOperationException( + "A successful cursor move did not produce a current item.")); + break; + case QueueCacheCursorMoveResultKind.CacheMiss: + this.current = null; + Invalidate(); + break; + case QueueCacheCursorMoveResultKind.NoData: + this.current = null; + break; + default: + throw new InvalidOperationException("The cursor move result is not initialized."); + } + + return result; + } } - public void Refresh(StreamSequenceToken? token) + public void Refresh(StreamSequenceToken token) { - this.cache.Refresh(this.cursor, token); + lock (this.owner.cacheLock) + { + var cache = this.owner.cache; + if (cache is null) + { + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = null; + this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; + this.pendingMoveResult = null; + return; + } + + if (!ReferenceEquals(this.cache, cache)) + { + var result = cache.TryGetCursor(this.streamId, token); + switch (result.Kind) + { + case QueueCacheCursorResultKind.Success: + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = cache; + this.cursor = result.Cursor; + this.current = null; + this.recoveryResumeToken = + ReferenceEquals(cache, this.owner.recoveryCache) ? token : null; + this.pendingMoveResult = null; + break; + case QueueCacheCursorResultKind.CacheMiss: + this.owner.ClearRecoveredDeliveryToken(this); + this.cache = cache; + this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; + this.pendingMoveResult = QueueCacheCursorMoveResult.FromCacheMiss(result.CacheMiss!.Value); + break; + default: + throw new InvalidOperationException("The cursor result is not initialized."); + } + + return; + } + + if (this.cursor is null + && this.pendingMoveResult is { Kind: QueueCacheCursorMoveResultKind.CacheMiss }) + { + return; + } + + cache.Refresh(this.cursor!, token); + } } public void RecordDeliveryFailure() { } + + private void SetCurrent(IBatchContainer next) + { + this.current = next; + this.owner.RecordRecoveredDeliveryToken( + this, + this.cache, + next.SequenceToken, + this.recoveryResumeToken); + this.recoveryResumeToken = null; + } + + private void Invalidate() + { + this.owner.UnregisterCursor(this); + this.cache = null; + this.cursor = null; + this.current = null; + this.recoveryResumeToken = null; + this.pendingMoveResult = null; + } } [LoggerMessage( @@ -509,7 +1285,7 @@ public void RecordDeliveryFailure() [LoggerMessage( Level = LogLevel.Warning, - EventId = (int)OrleansEventHubErrorCode.FailedPartitionRead, + EventId = (int)OrleansEventHubErrorCode.RetryReceiverInit, Message = "Retrying initialization of EventHub partition {EventHubName}-{Partition}." )] private partial void LogWarningRetryingInitializationOfEventHubPartition(string eventHubName, string partition); @@ -520,5 +1296,12 @@ public void RecordDeliveryFailure() Message = "Failed to read from EventHub partition {EventHubName}-{Partition}" )] private partial void LogWarningFailedToReadFromEventHubPartition(string eventHubName, string partition, Exception exception); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = (int)OrleansEventHubErrorCode.FailedPartitionRecovery, + Message = "Failed to recover EventHub partition {EventHubName}-{Partition} from an invalid checkpoint. The original read failure will be rethrown." + )] + private partial void LogWarningFailedToRecoverFromInvalidCheckpoint(string eventHubName, string partition, Exception exception); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 1c13fac1227..5ad093703b4 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -1,140 +1,162 @@ using System; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Streams; -using Orleans.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Orleans.Configuration; using Orleans.Configuration.Overrides; +using Orleans.Streams; + +namespace Orleans.Streaming.EventHubs; + +public class EventHubCheckpointerFactory : IStreamQueueCheckpointerFactory +{ + private readonly ILoggerFactory _loggerFactory; + private readonly string _providerName; + private readonly AzureTableStreamCheckpointerOptions _options; + private readonly ClusterOptions _clusterOptions; + + public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpointerOptions options, IOptions clusterOptions, ILoggerFactory loggerFactory) + { + _options = options; + _clusterOptions = clusterOptions.Value; + _loggerFactory = loggerFactory; + _providerName = providerName; + } + + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) + { + return EventHubCheckpointer.Create( + _options, + _providerName, + partition, + _clusterOptions.ServiceId.ToString(), + _loggerFactory, + cancellationToken); + } -namespace Orleans.Streaming.EventHubs + public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) + { + var options = services.GetOptionsByName(providerName); + IOptions clusterOptions = services.GetProviderClusterOptions(providerName); + return ActivatorUtilities.CreateInstance(services, providerName, options, clusterOptions); + } +} + +/// +/// This class stores EventHub partition checkpoint information in Azure Table Storage. +/// +public class EventHubCheckpointer : IStreamQueueCheckpointer { + private readonly IStreamQueueCheckpointer _inner; + + /// + /// Indicates if a checkpoint exists + /// + public bool CheckpointExists => _inner.CheckpointExists; + + /// + /// Factory function that creates and initializes the checkpointer + /// + /// + /// + /// + /// + /// + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) + => await Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + /// - /// Creates Event Hub partition checkpointers backed by Azure Table Storage. + /// Factory function that creates and initializes the checkpointer. /// - public class EventHubCheckpointerFactory : IStreamQueueCheckpointerFactory + public static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) { - private readonly ILoggerFactory loggerFactory; - private readonly string providerName; - private readonly AzureTableStreamCheckpointerOptions options; - private readonly ClusterOptions clusterOptions; - - /// - /// Initializes a new instance of the class. - /// - /// The stream provider name. - /// The Azure Table Storage checkpointer options. - /// The cluster options. - /// The logger factory. - public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpointerOptions options, IOptions clusterOptions, ILoggerFactory loggerFactory) - { - this.options = options; - this.clusterOptions = clusterOptions.Value; - this.loggerFactory = loggerFactory; - this.providerName = providerName; - } - - /// - public Task> Create(string partition) - { - return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); - } - - /// - /// Creates an Event Hub partition checkpointer factory. - /// - /// The service provider. - /// The stream provider name. - /// The checkpointer factory. - public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) - { - var options = services.GetOptionsByName(providerName); - IOptions clusterOptions = services.GetProviderClusterOptions(providerName); - return ActivatorUtilities.CreateInstance(services, providerName, options, clusterOptions); - } + var inner = await AzureTableStreamQueueCheckpointer.Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + StreamCheckpointComparers.Numeric, + StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix, + cancellationToken); + return new EventHubCheckpointer(inner); } + private EventHubCheckpointer(IStreamQueueCheckpointer inner) + { + _inner = inner; + } + + /// + /// Loads a checkpoint + /// + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + /// - /// This class stores EventHub partition checkpoint information in Azure Table Storage. + /// Loads a checkpoint. /// - public class EventHubCheckpointer : IStreamQueueCheckpointer + /// The cancellation token. + /// The checkpoint. + public async Task Load(CancellationToken cancellationToken) { - private readonly IStreamQueueCheckpointer _inner; - - /// - public bool CheckpointExists => _inner.CheckpointExists; - - /// - /// Creates and initializes an Event Hub partition checkpointer. - /// - /// The Azure Table Storage checkpointer options. - /// The stream provider name. - /// The Event Hub partition identifier. - /// The Orleans service identifier. - /// The logger factory. - /// A task which resolves to the initialized checkpointer. - public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) - { - var inner = await AzureTableStreamQueueCheckpointer.Create( - options, - streamProviderName, - partition, - serviceId, - loggerFactory, - StreamCheckpointComparers.Numeric, - StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); - return new EventHubCheckpointer(inner); - } - - private EventHubCheckpointer(IStreamQueueCheckpointer inner) - { - _inner = inner; - } - - /// - /// Loads the checkpoint. - /// - /// A task which resolves to the checkpoint offset. - [Obsolete("Use the overload which accepts a CancellationToken.")] - public Task Load() => Load(CancellationToken.None); - - /// - /// Loads a checkpoint. - /// - /// The cancellation token. - /// The checkpoint. - public async Task Load(CancellationToken cancellationToken) - { - var checkpoint = await _inner.Load(cancellationToken); - return CheckpointExists ? checkpoint : EventHubConstants.StartOfStream; - } - - /// - /// Updates the checkpoint. This is a best effort. It does not always update the checkpoint. - /// The latest offset is always tracked in memory so that can persist it on shutdown. - /// - /// The checkpoint offset. - /// The current UTC time. - [Obsolete("Use the overload which accepts a CancellationToken.")] - public void Update(string offset, DateTime utcNow) - => Update(offset, utcNow, CancellationToken.None); - - /// - /// Updates the checkpoint. - /// - /// The checkpoint offset. - /// The current UTC time. - /// The cancellation token. - public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) - => _inner.Update(offset, utcNow, cancellationToken); - - /// - /// Flushes any pending checkpoint to persistent storage. - /// Awaits any in-progress save, then persists the latest offset if it has advanced beyond the last saved value. - /// - /// The cancellation token. - /// A task which represents the flush operation. - public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + var checkpoint = await _inner.Load(cancellationToken); + return CheckpointExists ? checkpoint : EventHubConstants.StartOfStream; } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken); + + /// + /// Updates the checkpoint. This is a best effort. It does not always update the checkpoint. + /// The latest offset is always tracked in memory so that can persist it on shutdown. + /// + /// + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); + + /// + /// Updates the checkpoint. + /// + /// The checkpoint offset. + /// The current UTC time. + /// The cancellation token. + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); + + /// + /// Flushes any pending checkpoint to persistent storage. + /// Awaits any in-progress save, then persists the latest offset if it has advanced beyond the last saved value. + /// + /// The cancellation token. + public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs index bc12faf9568..e8006b583af 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubStreamOptions.cs @@ -255,8 +255,9 @@ public class EventHubReceiverOptions /// public int? PrefetchCount { get; set; } /// - /// In cases where no checkpoint is found, this indicates if service should read from the most recent data, or from the beginning of a partition. + /// Selects the receiver position when a checkpoint is unavailable, including after recovery from an invalid persisted offset. /// + /// starts at the partition tail and reads newly enqueued events; starts at the earliest retained event. public bool StartFromNow { get; set; } = DEFAULT_START_FROM_NOW; private const bool DEFAULT_START_FROM_NOW = true; } diff --git a/src/Azure/Orleans.Streaming.EventHubs/README.md b/src/Azure/Orleans.Streaming.EventHubs/README.md index 4f2c4ab0578..7179ba21605 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/README.md +++ b/src/Azure/Orleans.Streaming.EventHubs/README.md @@ -74,6 +74,18 @@ Set `GrainStreamQueueCheckpointerOptions.StorageProviderName` to use another reg Both `UseGrainCheckpointer` and `UseAzureTableCheckpointer` extend `ISiloPersistentStreamConfigurator`, so they can also be used with other persistent stream providers. Event Hubs configures numeric checkpoint ordering by default; other providers can set the corresponding `CheckpointComparer` option for their checkpoint format. +### Recovering from an invalid checkpoint + +When Event Hubs rejects a persisted partition offset with its known invalid-offset argument error, the provider clears that partition's checkpoint, closes and recreates its receiver, and resumes on the next pull using the configured no-checkpoint position. Recovery recognizes an `ArgumentException` whose service message starts with `The supplied offset` and contains ` is invalid.`. Azure.Messaging.EventHubs 5.12.2 exposes this rejection through the service message, so recognition depends on that message format. + +`EventHubReceiverOptions.StartFromNow` selects the recovery position. Its default value, `true`, starts at the partition tail and reads newly enqueued events, skipping the retained backlog. Set it to `false` to replay from the earliest retained event, which can duplicate delivery. Choose the policy according to the application's recovery requirements and keep consumers idempotent. + +The built-in Azure Table and grain-backed checkpointers support this reset contract. A custom `IStreamQueueCheckpointer` participates in recovery by implementing `Reset(CancellationToken)`; the default implementation surfaces unsupported recovery with `NotSupportedException`. + +Existing Orleans subscription cursors rebind to the replacement partition cache, so subscriptions continue automatically. Initial receiver creation and reads observe the pulling agent's cancellation token. Once recovery detaches a failed receiver, checkpoint reset runs to completion as a barrier before replacement initialization. Canceling a pull cancels its wait for that recovery; the recovery continues with independent cleanup and replacement-initialization timeouts. + +Receiver shutdown stops initialization and recovery from installing new resources, drains admitted initialization and recovery, then flushes the checkpoint and closes the current lifetime's resources. Its timeout bounds the drain and flush; receiver close has an independent cleanup timeout. Successful shutdown completes cleanup for that lifetime, and a subsequent initialization starts a new lifetime with its own shutdown. + ## Example - Using Event Hub Streams in a Grain ```csharp diff --git a/src/Azure/Shared/Storage/AzureTableDataManager.cs b/src/Azure/Shared/Storage/AzureTableDataManager.cs index da64f019586..1c6cd3b8722 100644 --- a/src/Azure/Shared/Storage/AzureTableDataManager.cs +++ b/src/Azure/Shared/Storage/AzureTableDataManager.cs @@ -195,7 +195,16 @@ public async Task CreateTableEntryAsync(T data, CancellationToken cancel /// /// Data to be inserted or replaced in the table. /// Value promise with new Etag for this data entry after completing this storage operation. - public async Task UpsertTableEntryAsync(T data) + public Task UpsertTableEntryAsync(T data) + => UpsertTableEntryAsync(data, CancellationToken.None); + + /// + /// Inserts a data entry in the Azure table: creates a new one if does not exist or overwrites an existing version. + /// + /// Data to be inserted or replaced in the table. + /// The cancellation token. + /// Value promise with new Etag for this data entry after completing this storage operation. + public async Task UpsertTableEntryAsync(T data, CancellationToken cancellationToken) { const string operation = "UpsertTableEntry"; var startTime = DateTime.UtcNow; @@ -204,7 +213,7 @@ public async Task UpsertTableEntryAsync(T data) { try { - var opResult = await Table.UpsertEntityAsync(data); + var opResult = await Table.UpsertEntityAsync(data, cancellationToken: cancellationToken); return opResult.Headers.ETag.GetValueOrDefault().ToString(); } catch (Exception exc) diff --git a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs index e1ae927ae3e..98e4a7ec21d 100644 --- a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs @@ -173,6 +173,14 @@ internal static string GetConfiguredStorageProviderName(ReadOnlySpan grain /// public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) + => _inner.Reset(cancellationToken); + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) diff --git a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs index 2cc967877c2..66587ac1846 100644 --- a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs @@ -17,6 +17,8 @@ public sealed class StreamQueueCheckpointer : IStreamQueueCheckpointer private StreamCheckpointStoreState _persistedState = new(string.Empty, string.Empty); private Task _inProgressSave = Task.CompletedTask; private DateTime? _throttleSavesUntilUtc; + private int _pendingResetCount; + private long _updateGeneration; /// /// Initializes a new instance of the class. @@ -68,6 +70,143 @@ public async Task Load(CancellationToken cancellationToken) return state.Checkpoint; } + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Reset() => Reset(CancellationToken.None); + + /// + public Task Reset(CancellationToken cancellationToken) + { + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + lock (_lock) + { + _pendingResetCount++; + _throttleSavesUntilUtc = DateTime.MaxValue; + var inProgressSave = _inProgressSave; + _inProgressSave = completion.Task; + RunReset( + completion, + inProgressSave, + _inProgressSave, + _latestCheckpoint, + _updateGeneration, + cancellationToken).Ignore(); + completion.Task.Ignore(); + } + + return completion.Task.WaitAsync(cancellationToken); + } + + private async Task RunReset( + TaskCompletionSource completion, + Task inProgressSave, + Task resetTask, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await ResetCore( + inProgressSave, + enqueuedCheckpoint, + updateGeneration, + cancellationToken); + + while (true) + { + string checkpoint; + lock (_lock) + { + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedState.Checkpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + completion.TrySetResult(); + return; + } + + checkpoint = _latestCheckpoint; + } + + await Save(checkpoint, cancellationToken); + } + } + catch (OperationCanceledException exception) + { + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: true); + } + catch (Exception exception) + { + await CompleteFailedReset( + completion, + resetTask, + exception, + canceled: false); + } + } + + private async Task CompleteFailedReset( + TaskCompletionSource completion, + Task resetTask, + Exception resetFailure, + bool canceled) + { + try + { + while (true) + { + string checkpoint; + lock (_lock) + { + if (!ReferenceEquals(resetTask, _inProgressSave) + || string.Equals(_persistedState.Checkpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + CompleteReset(); + if (canceled) + { + completion.TrySetCanceled( + ((OperationCanceledException)resetFailure).CancellationToken); + } + else + { + completion.TrySetException(resetFailure); + } + + return; + } + + checkpoint = _latestCheckpoint; + } + + await Save(checkpoint, CancellationToken.None); + } + } + catch (Exception persistenceFailure) + { + lock (_lock) + { + CompleteReset(); + completion.TrySetException( + new AggregateException(resetFailure, persistenceFailure)); + } + } + } + + private void CompleteReset() + { + _pendingResetCount--; + if (_pendingResetCount == 0) + { + _throttleSavesUntilUtc = null; + } + } + /// [Obsolete("Use the overload which accepts a CancellationToken.")] public void Update(string offset, DateTime utcNow) @@ -104,6 +243,7 @@ public void Update(string offset, DateTime utcNow, CancellationToken cancellatio } _latestCheckpoint = offset; + _updateGeneration++; if (_throttleSavesUntilUtc.HasValue && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) { @@ -159,6 +299,7 @@ public async Task FlushAsync(CancellationToken cancellationToken) } _inProgressSave = Save(_latestCheckpoint, cancellationToken); + _inProgressSave.Ignore(); retryingSave = true; } } @@ -205,6 +346,107 @@ private async Task Save(string checkpoint, CancellationToken cancellationToken) } } + private async Task ResetCore( + Task inProgressSave, + string enqueuedCheckpoint, + long updateGeneration, + CancellationToken cancellationToken) + { + try + { + await inProgressSave; + } + catch + { + } + + cancellationToken.ThrowIfCancellationRequested(); + + string rollbackCheckpoint; + string expectedVersion; + string persistedVersion; + lock (_lock) + { + persistedVersion = expectedVersion = _persistedState.Version; + if (_updateGeneration <= updateGeneration) + { + rollbackCheckpoint = _latestCheckpoint; + _latestCheckpoint = string.Empty; + } + else + { + rollbackCheckpoint = enqueuedCheckpoint; + } + } + + try + { + while (true) + { + var persistedState = await _store.Update(string.Empty, expectedVersion, cancellationToken); + lock (_lock) + { + _persistedState = persistedState; + } + + if (string.IsNullOrEmpty(persistedState.Checkpoint)) + { + return; + } + + expectedVersion = persistedState.Version; + } + } + catch + { + lock (_lock) + { + RestoreLatestCheckpoint(rollbackCheckpoint, persistedVersion); + } + + throw; + } + } + + private void RestoreLatestCheckpoint(string latestCheckpoint, string persistedVersion) + { + if (string.Equals(_persistedState.Version, persistedVersion, StringComparison.Ordinal)) + { + if (_options.CheckpointComparer is { } checkpointComparer) + { + if (Compare(checkpointComparer, _latestCheckpoint, latestCheckpoint) < 0) + { + _latestCheckpoint = latestCheckpoint; + } + } + else if (string.IsNullOrEmpty(_latestCheckpoint)) + { + _latestCheckpoint = latestCheckpoint; + } + + return; + } + + if (_options.CheckpointComparer is not { } comparer) + { + _latestCheckpoint = _persistedState.Checkpoint; + return; + } + + var candidate = _latestCheckpoint; + if (Compare(comparer, candidate, latestCheckpoint) < 0) + { + candidate = latestCheckpoint; + } + + if (Compare(comparer, candidate, _persistedState.Checkpoint) < 0) + { + candidate = _persistedState.Checkpoint; + } + + _latestCheckpoint = candidate; + } + private static int Compare(IComparer comparer, string left, string right) { if (string.IsNullOrEmpty(left)) diff --git a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs index be0c7c013c6..f8c686cbb90 100644 --- a/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/PersistentStreams/IStreamQueueCheckpointer.cs @@ -2,86 +2,109 @@ using System.Threading; using System.Threading.Tasks; -namespace Orleans.Streams +namespace Orleans.Streams; + +/// +/// Factory for creating instances. +/// +public interface IStreamQueueCheckpointerFactory { /// - /// Factory for creating instances. + /// Creates a stream checkpointer for the specified partition. /// - public interface IStreamQueueCheckpointerFactory - { - /// - /// Creates a stream checkpointer for the specified partition. - /// - /// The partition. - /// The stream checkpointer. - [Obsolete("Use the overload which accepts a CancellationToken.")] - Task> Create(string partition); + /// The partition. + /// The stream checkpointer. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task> Create(string partition); - /// - /// Creates a stream checkpointer for the specified partition. - /// - /// The partition. - /// The cancellation token. - /// The stream checkpointer. + /// + /// Creates a stream checkpointer for the specified partition. + /// + /// The partition. + /// The cancellation token. + /// The stream checkpointer. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task> Create(string partition, CancellationToken cancellationToken) - => Create(partition); + Task> Create(string partition, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled>(cancellationToken) + : Create(partition); #pragma warning restore CS0618 - } +} +/// +/// Functionality for checkpointing a stream. +/// +/// The checkpoint type. +public interface IStreamQueueCheckpointer +{ /// - /// Functionality for checkpointing a stream. + /// Gets a value indicating whether a checkpoint exists. /// - /// The checkpoint type. - public interface IStreamQueueCheckpointer - { - /// - /// Gets a value indicating whether a checkpoint exists. - /// - /// if checkpoint exists; otherwise, . - bool CheckpointExists { get; } + /// if checkpoint exists; otherwise, . + bool CheckpointExists { get; } - /// - /// Loads the checkpoint. - /// - /// The checkpoint. - [Obsolete("Use the overload which accepts a CancellationToken.")] - Task Load(); + /// + /// Loads the checkpoint. + /// + /// The checkpoint. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task Load(); - /// - /// Loads the checkpoint. - /// - /// The cancellation token. - /// The checkpoint. + /// + /// Loads the checkpoint. + /// + /// The cancellation token. + /// The checkpoint. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - Task Load(CancellationToken cancellationToken) => Load(); + Task Load(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Load(); #pragma warning restore CS0618 - /// - /// Updates the checkpoint. - /// - /// The offset. - /// The current UTC time. - [Obsolete("Use the overload which accepts a CancellationToken.")] - void Update(TCheckpoint offset, DateTime utcNow); + /// + /// Clears the persisted checkpoint. + /// + /// A representing the operation. + [Obsolete("Use the overload which accepts a CancellationToken.")] + Task Reset() => throw new NotSupportedException("This checkpointer does not support resetting checkpoints."); - /// - /// Updates the checkpoint. - /// - /// The offset. - /// The current UTC time. - /// The cancellation token. + /// + /// Clears the persisted checkpoint. + /// + /// The cancellation token. + /// A representing the operation. #pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. - void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationToken) - => Update(offset, utcNow); + Task Reset(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Reset(); #pragma warning restore CS0618 - /// - /// Flushes any pending checkpoint to persistent storage, ensuring the latest offset is durably saved. - /// Called during shutdown or rebalancing to prevent message replay on restart. - /// - /// The cancellation token. - /// A representing the flush operation. - Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + /// + /// Updates the checkpoint. + /// + /// The offset. + /// The current UTC time. + [Obsolete("Use the overload which accepts a CancellationToken.")] + void Update(TCheckpoint offset, DateTime utcNow); + + /// + /// Updates the checkpoint. + /// + /// The offset. + /// The current UTC time. + /// The cancellation token. +#pragma warning disable CS0618 // Required for compatibility with providers which only implement the legacy overload. + void Update(TCheckpoint offset, DateTime utcNow, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Update(offset, utcNow); } +#pragma warning restore CS0618 + + /// + /// Flushes any pending checkpoint to persistent storage, ensuring the latest offset is durably saved. + /// Called during shutdown or rebalancing to prevent message replay on restart. + /// + /// The cancellation token. + /// A representing the flush operation. + Task FlushAsync(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; } diff --git a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs index 9ef8844adb3..ecf77c70234 100644 --- a/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs +++ b/src/api/AWS/Orleans.Streaming.Kinesis/Orleans.Streaming.Kinesis.cs @@ -89,6 +89,7 @@ public DynamoDBStreamQueueCheckpointerFactory(string providerName, Configuration public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index ab34b90eaa2..ccd44301ef6 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -368,6 +368,9 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -377,6 +380,11 @@ internal EventHubCheckpointer() { } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] @@ -387,6 +395,9 @@ public partial class EventHubCheckpointerFactory : Streams.IStreamQueueCheckpoin { public EventHubCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } @@ -745,12 +756,20 @@ public partial class NoOpCheckpointer : Streams.IStreamQueueCheckpointer public System.Threading.Tasks.Task Load() { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow) { } } public partial class NoOpCheckpointerFactory : Streams.IStreamQueueCheckpointerFactory { public static NoOpCheckpointerFactory Instance; + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } } @@ -778,12 +797,23 @@ internal AzureTableStreamQueueCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } + public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow) { } } @@ -791,6 +821,9 @@ public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueChec { public AzureTableStreamQueueCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index c8567217755..363c4530276 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -1557,6 +1557,11 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain) { } public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] @@ -1854,6 +1859,9 @@ public partial interface IStreamQueueCheckpointer [System.Obsolete("Use the overload which accepts a CancellationToken.")] System.Threading.Tasks.Task Load(); System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken); + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + System.Threading.Tasks.Task Reset(); + System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken); void Update(TCheckpoint offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken); [System.Obsolete("Use the overload which accepts a CancellationToken.")] void Update(TCheckpoint offset, System.DateTime utcNow); @@ -2330,6 +2338,11 @@ public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpoi public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Reset() { throw null; } + + public System.Threading.Tasks.Task Reset(System.Threading.CancellationToken cancellationToken) { throw null; } + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] diff --git a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs index 31382d25b66..d110b6df080 100644 --- a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs +++ b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerStressTests.cs @@ -149,7 +149,7 @@ private async Task WriteAlot_Async( dataObject.PartitionKey = partitionKey; dataObject.RowKey = rowKey; dataObject.StringData = rowKey; - var promise = manager.UpsertTableEntryAsync(dataObject); + var promise = manager.UpsertTableEntryAsync(dataObject, cancellationToken); promises.Add(promise); if ((i % batchSize) == 0 && i > 0) { diff --git a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs index 2521be419d1..65e3f72b2b2 100644 --- a/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs +++ b/test/Extensions/Orleans.Azure.Tests/AzureTableDataManagerTests.cs @@ -64,7 +64,7 @@ public async Task AzureTableDataManager_CreateTableEntryAsync() public async Task AzureTableDataManager_UpsertTableEntryAsync() { var data = GenerateNewData(); - await manager.UpsertTableEntryAsync(data); + await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var tuple = await manager.ReadSingleTableEntryAsync( data.PartitionKey, data.RowKey, @@ -74,7 +74,7 @@ public async Task AzureTableDataManager_UpsertTableEntryAsync() var data2 = data.Clone(); data2.StringData = "NewData"; - await manager.UpsertTableEntryAsync(data2); + await manager.UpsertTableEntryAsync(data2, TestContext.Current.CancellationToken); tuple = await manager.ReadSingleTableEntryAsync( data2.PartitionKey, data2.RowKey, @@ -103,7 +103,7 @@ public async Task AzureTableDataManager_UpdateTableEntryAsync() Assert.Equal(TableErrorCode.ResourceNotFound.ToString(), restStatus); } - await manager.UpsertTableEntryAsync(data); + await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var tuple = await manager.ReadSingleTableEntryAsync( data.PartitionKey, data.RowKey, @@ -164,7 +164,7 @@ public async Task AzureTableDataManager_DeleteTableAsync() Assert.Equal(HttpStatusCode.NotFound, httpStatusCode); } - string eTag1 = await manager.UpsertTableEntryAsync(data); + string eTag1 = await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); await manager.DeleteTableEntryAsync(data, eTag1); try @@ -207,7 +207,7 @@ public async Task AzureTableDataManager_MergeTableAsync() Assert.Equal(TableErrorCode.ResourceNotFound.ToString(), restStatus); } - string eTag1 = await manager.UpsertTableEntryAsync(data); + string eTag1 = await manager.UpsertTableEntryAsync(data, TestContext.Current.CancellationToken); var data2 = data.Clone(); data2.StringData = "NewData"; await manager.MergeTableEntryAsync(data2, eTag1, TestContext.Current.CancellationToken); diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index 97846df0868..52b7e208f74 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -25,16 +25,25 @@ public class EventHubCheckpointerTests /// private class TestCheckpointer : IStreamQueueCheckpointer { - public bool CheckpointExists { get; init; } = true; + public bool CheckpointExists { get; set; } = true; public string LoadedOffset { get; init; } = EventHubConstants.StartOfStream; public string? LastOffset { get; private set; } public int UpdateCount { get; private set; } public string? FlushedOffset { get; private set; } public int FlushCount { get; private set; } + public int ResetCount { get; private set; } public Task Load() => Task.FromResult(LoadedOffset); - public void Update(string offset, DateTime utcNow) + public virtual Task Reset(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + CheckpointExists = false; + ResetCount++; + return Task.CompletedTask; + } + + public virtual void Update(string offset, DateTime utcNow) { if (LastOffset is not null && long.Parse(offset, CultureInfo.InvariantCulture) <= long.Parse(LastOffset, CultureInfo.InvariantCulture)) @@ -64,6 +73,104 @@ public override Task FlushAsync(CancellationToken cancellationToken) } } + private sealed class FailOnceFlushCheckpointer : TestCheckpointer + { + public int FlushAttempts { get; private set; } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + FlushAttempts++; + return FlushAttempts == 1 + ? Task.FromException(new InvalidOperationException("Flush failed")) + : base.FlushAsync(cancellationToken); + } + } + + private sealed class BlockingFlushCheckpointer : TestCheckpointer + { + public TaskCompletionSource FlushStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseFlush { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override async Task FlushAsync(CancellationToken cancellationToken) + { + FlushStarted.TrySetResult(); + await ReleaseFlush.Task.WaitAsync(cancellationToken); + await base.FlushAsync(cancellationToken); + } + } + + private sealed class CanceledFlushCheckpointer : TestCheckpointer + { + public override async Task FlushAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + } + + private sealed class FailingResetCheckpointer : TestCheckpointer + { + public override Task Reset(CancellationToken cancellationToken) + { + _ = base.Reset(cancellationToken); + throw new InvalidOperationException("Reset failed"); + } + } + + private sealed class BlockingResetCheckpointer : TestCheckpointer + { + public CancellationToken ResetCancellationToken { get; private set; } + + public TaskCompletionSource ResetStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseReset { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override async Task Reset(CancellationToken cancellationToken) + { + ResetCancellationToken = cancellationToken; + await base.Reset(cancellationToken); + ResetStarted.TrySetResult(); + await ReleaseReset.Task.WaitAsync(cancellationToken); + } + } + + private sealed class CancelingResetCheckpointer(CancellationTokenSource cancellation) : TestCheckpointer + { + public override async Task Reset(CancellationToken cancellationToken) + { + await base.Reset(cancellationToken); + cancellation.Cancel(); + } + } + + private sealed class BlockingUpdateCheckpointer : TestCheckpointer + { + public TaskCompletionSource UpdateStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseUpdate { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ResetStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public override void Update(string offset, DateTime utcNow) + { + UpdateStarted.TrySetResult(); + ReleaseUpdate.Task.GetAwaiter().GetResult(); + base.Update(offset, utcNow); + } + + public override Task Reset(CancellationToken cancellationToken) + { + ResetStarted.TrySetResult(); + return base.Reset(cancellationToken); + } + } + private class TestEventHubQueueCache : IEventHubQueueCache { private readonly IStreamQueueCheckpointer? checkpointer; @@ -79,23 +186,36 @@ public TestEventHubQueueCache(IStreamQueueCheckpointer? checkpointer = n public object Cursor { get; } = new(); public object? RefreshedCursor { get; private set; } public StreamSequenceToken? RefreshToken { get; private set; } + public int RefreshCount { get; private set; } public Exception? CursorException { get; set; } - public QueueCacheMissException? MoveNextException { get; set; } + public Exception? MoveNextException { get; set; } public IBatchContainer? NoDataMessage { get; set; } + public int GetCursorCount { get; private set; } + public IBatchContainer? NextMessage { get; set; } + public List PositionsToReturn { get; init; } = []; public int GetMaxAddCount() => 1_000; public List Add(List message, DateTime dequeueTimeUtc) { AddCount++; - return []; + return PositionsToReturn; } public object GetCursor(StreamId streamId, StreamSequenceToken? sequenceToken) - => CursorException is { } exception ? throw exception : Cursor; + { + if (CursorException is { } exception) + { + throw exception; + } + + GetCursorCount++; + return Cursor; + } public void Refresh(object cursor, StreamSequenceToken? sequenceToken) { + RefreshCount++; RefreshedCursor = cursor; RefreshToken = sequenceToken; } @@ -107,6 +227,13 @@ public bool TryGetNextMessage(object cursorObj, out IBatchContainer message) throw exception; } + if (NextMessage is { } next) + { + NextMessage = null; + message = next; + return true; + } + message = NoDataMessage!; return false; } @@ -132,10 +259,11 @@ public void Dispose() private sealed class TestEventHubReceiver : IEventHubReceiver { public int CloseCount { get; private set; } + public IEnumerable Messages { get; init; } = []; public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) { - return Task.FromResult>([]); + return Task.FromResult(Messages); } public Task CloseAsync() @@ -145,6 +273,16 @@ public Task CloseAsync() } } + private sealed class TestBatchContainer : IBatchContainer + { + public StreamId StreamId { get; } = StreamId.Create("namespace", Guid.NewGuid()); + public StreamSequenceToken SequenceToken { get; init; } = MakeToken(1); + + public IEnumerable> GetEvents() => []; + + public bool ImportRequestContext() => false; + } + [Fact, TestCategory("BVT")] public void GetCursorPreservesLegacyEventHubCacheBehavior() { @@ -282,6 +420,20 @@ QueueCacheCursorResult IEventHubQueueCache.TryGetCursor( } } + private sealed class SequencedAcquisitionEventHubQueueCache : TestEventHubQueueCache, IEventHubQueueCache + { + public Queue> Results { get; } = new(); + public int RequestCount { get; private set; } + + QueueCacheCursorResult IEventHubQueueCache.TryGetCursor( + StreamId streamId, + StreamSequenceToken? token) + { + RequestCount++; + return Results.Dequeue(); + } + } + private sealed class NullReturningEventHubReceiver : IEventHubReceiver { public int ReceiveCount { get; private set; } @@ -296,6 +448,101 @@ public Task> ReceiveAsync(int maxCount, TimeSpan waitTime public Task CloseAsync() => Task.CompletedTask; } + private sealed class InvalidOffsetEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken CloseCancellationToken { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + { + ReceiveStarted.TrySetResult(); + throw new ArgumentException("The supplied offset is invalid."); + } + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + CloseCancellationToken = cancellationToken; + return Task.CompletedTask; + } + } + + private sealed class InvalidArgumentEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + public Exception Failure { get; init; } = new ArgumentException("The receiver rejected an unrelated argument."); + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => throw Failure; + + public Task CloseAsync() + { + CloseCount++; + return Task.CompletedTask; + } + } + + private sealed class RetryingCloseEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => throw new ArgumentException("The supplied offset is invalid."); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + return CloseCount == 1 + ? Task.FromException(new InvalidOperationException("Close failed")) + : Task.CompletedTask; + } + } + + private sealed class BlockingReadEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseReceive { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int CloseCount { get; private set; } + + public async Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + { + ReceiveStarted.TrySetResult(); + await ReleaseReceive.Task; + return [new EventData()]; + } + + public Task CloseAsync() + { + CloseCount++; + return Task.CompletedTask; + } + } + + private sealed class BlockingInvalidOffsetEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReceiveStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseReceive { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + { + ReceiveStarted.TrySetResult(); + await ReleaseReceive.Task; + throw new ArgumentException("The supplied offset is invalid."); + } + + public Task CloseAsync() => Task.CompletedTask; + } + private sealed class CancellableEventHubReceiver : IEventHubReceiver { public TaskCompletionSource ReceiveStarted { get; } = new( @@ -335,11 +582,58 @@ public Task CloseAsync(CancellationToken cancellationToken) } } + private sealed class NonCancelableCloseEventHubReceiver : IEventHubReceiver + { + public TaskCompletionSource ReleaseClose { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int CloseCount { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + return ReleaseClose.Task; + } + } + + private sealed class CleanupTrackingEventHubReceiver : IEventHubReceiver + { + public int CloseCount { get; private set; } + public CancellationToken CloseCancellationToken { get; private set; } + + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => CloseAsync(CancellationToken.None); + + public Task CloseAsync(CancellationToken cancellationToken) + { + CloseCount++; + CloseCancellationToken = cancellationToken; + return Task.CompletedTask; + } + } + private static EventHubSequenceToken MakeToken(long offset, long sequenceNumber = 0) { return new EventHubSequenceToken(offset.ToString(), sequenceNumber, 0); } + private static IQueueCacheCursor AssertCursor( + EventHubAdapterReceiver receiver, + StreamId streamId, + StreamSequenceToken token) + { + var result = ((IQueueCache)receiver).TryGetCacheCursor(streamId, token); + Assert.Equal(QueueCacheCursorResultKind.Success, result.Kind); + Assert.NotNull(result.Cursor); + return result.Cursor; + } + private static void UpdateDeliveryProgress(EventHubAdapterReceiver receiver, StreamSequenceToken token) { receiver.UpdateDeliveryProgress(token, DateTime.UtcNow); @@ -354,7 +648,11 @@ private static async Task CreateReceiver( TestCheckpointer checkpointer, TestEventHubQueueCache? cache = null, IEventHubReceiver? eventHubReceiver = null, - Action? onReceiverCreated = null) + Action? onReceiverCreated = null, + Func? receiverFactory = null, + Func? cacheFactory = null, + Func>>? checkpointerFactory = null, + bool initialize = true) { var settings = new EventHubPartitionSettings { @@ -370,8 +668,9 @@ private static async Task CreateReceiver( var receiver = new EventHubAdapterReceiver( settings, - cacheFactory: (_, createdCheckpointer, _) => cache ?? new TestEventHubQueueCache(createdCheckpointer), - checkpointerFactory: _ => Task.FromResult>(checkpointer), + cacheFactory: (_, createdCheckpointer, _) => cacheFactory?.Invoke() ?? cache ?? new TestEventHubQueueCache(createdCheckpointer), + checkpointerFactory: (_, cancellationToken) => checkpointerFactory?.Invoke(cancellationToken) + ?? Task.FromResult>(checkpointer), loggerFactory: Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance, monitor: new Orleans.Streaming.EventHubs.DefaultEventHubReceiverMonitor( new EventHubReceiverMonitorDimensions @@ -385,45 +684,866 @@ private static async Task CreateReceiver( eventHubReceiverFactory: (_, offset, _) => { onReceiverCreated?.Invoke(offset); - return eventHubReceiver ?? new TestEventHubReceiver(); + return receiverFactory?.Invoke(offset) ?? eventHubReceiver ?? new TestEventHubReceiver(); }); - await receiver.Initialize(TimeSpan.FromSeconds(5)); + if (initialize) + { + await receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + } + + return receiver; + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetMaxAddCount_BeforeInitialize_UsesSafeDefault() + { + var receiver = await CreateReceiver(new TestCheckpointer(), initialize: false); + + Assert.Equal(EventHubAdapterReceiver.MaxMessagesPerRead, receiver.GetMaxAddCount()); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenCheckpointDoesNotExist_UsesStartOfStream() + { + string? receiverOffset = null; + var checkpointer = new TestCheckpointer + { + CheckpointExists = false, + LoadedOffset = string.Empty, + }; + + _ = await CreateReceiver( + checkpointer, + onReceiverCreated: offset => receiverOffset = offset); + + Assert.Equal(EventHubConstants.StartOfStream, receiverOffset); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ConcurrentCallersShareInitialization() + { + var checkpointer = new TestCheckpointer(); + var factoryStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var factoryCalls = 0; + var receiver = await CreateReceiver( + checkpointer, + checkpointerFactory: async cancellationToken => + { + Interlocked.Increment(ref factoryCalls); + factoryStarted.TrySetResult(); + await releaseFactory.Task.WaitAsync(cancellationToken); + return checkpointer; + }, + initialize: false); + + var first = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + await factoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.False(second.IsCompleted); + releaseFactory.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, factoryCalls); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ThroughInterface_ForwardsCancellationToken() + { + var checkpointer = new TestCheckpointer(); + var initializationStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var receiver = await CreateReceiver( + checkpointer, + checkpointerFactory: async cancellationToken => + { + initializationStarted.TrySetResult(cancellationToken); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return checkpointer; + }, + initialize: false); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + + var initialization = ((IQueueAdapterReceiver)receiver).Initialize( + TimeSpan.FromSeconds(30), + cancellation.Token); + var forwardedToken = await initializationStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.True(forwardedToken.IsCancellationRequested); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenReceiverCreationFails_DisposesCreatedCache() + { + var cache = new TestEventHubQueueCache(); + var expected = new InvalidOperationException("Receiver creation failed"); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + receiverFactory: _ => throw expected, + initialize: false); + + var actual = await Assert.ThrowsAsync( + () => receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken)); + + Assert.Same(expected, actual); + Assert.Equal(1, cache.DisposeCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenCanceledAfterResourceCreation_UsesIndependentCleanupToken() + { + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new CleanupTrackingEventHubReceiver(); + using var cancellation = new CancellationTokenSource(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + receiverFactory: _ => + { + cancellation.Cancel(); + return eventHubReceiver; + }, + initialize: false); + + var exception = await Assert.ThrowsAnyAsync( + () => InvokeInitialize(receiver, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(eventHubReceiver.CloseCancellationToken.IsCancellationRequested); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WhenCleanupFails_RetriesCloseBeforeCreatingReplacement() + { + using var cancellation = new CancellationTokenSource(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var receiverCreations = 0; + var receiver = await CreateReceiver( + new TestCheckpointer(), + receiverFactory: _ => + { + if (Interlocked.Increment(ref receiverCreations) == 1) + { + cancellation.Cancel(); + return failedReceiver; + } + + return replacementReceiver; + }, + initialize: false); + + await Assert.ThrowsAsync( + () => InvokeInitialize(receiver, cancellation.Token)); + + Assert.Equal(1, failedReceiver.CloseCount); + Assert.Equal(1, receiverCreations); + + await receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.Equal(2, failedReceiver.CloseCount); + Assert.Equal(2, receiverCreations); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_WaitingCallerDrainsFailedCleanupBeforeCreatingReplacement() + { + var checkpointer = new TestCheckpointer(); + var firstCancellation = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var receiverFactoryStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + using var releaseReceiverFactory = new ManualResetEventSlim(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var checkpointerFactoryCalls = 0; + var receiverCreations = 0; + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: _ => + { + if (Interlocked.Increment(ref receiverCreations) == 1) + { + receiverFactoryStarted.TrySetResult(); + releaseReceiverFactory.Wait(TestContext.Current.CancellationToken); + return failedReceiver; + } + + return replacementReceiver; + }, + checkpointerFactory: cancellationToken => + { + if (Interlocked.Increment(ref checkpointerFactoryCalls) == 1) + { + _ = cancellationToken.Register(() => firstCancellation.TrySetResult()); + } + + return Task.FromResult>(checkpointer); + }, + initialize: false); + + var first = Task.Run( + () => receiver.Initialize( + TimeSpan.FromMilliseconds(50), + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + await receiverFactoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + await firstCancellation.Task.WaitAsync(TestContext.Current.CancellationToken); + releaseReceiverFactory.Set(); + + await Assert.ThrowsAsync(() => first); + await second; + + Assert.Equal(2, failedReceiver.CloseCount); + Assert.Equal(2, receiverCreations); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_TreatsNullReceiverResultAsEmpty() + { + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new NullReturningEventHubReceiver(); + var receiver = await CreateReceiver(new TestCheckpointer(), cache, eventHubReceiver); + + var messages = await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + + Assert.Empty(messages); + Assert.Equal(1, eventHubReceiver.ReceiveCount); + Assert.Equal(0, cache.AddCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenCheckpointOffsetIsInvalid_ResetsAndRecreatesReceiver() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var initialBatch = new TestBatchContainer(); + var initialCache = new TestEventHubQueueCache + { + NextMessage = initialBatch, + }; + var recoveredToken = MakeToken(10); + var recoveredBatch = new TestBatchContainer + { + SequenceToken = recoveredToken, + }; + var replacementCache = new TestEventHubQueueCache + { + NextMessage = recoveredBatch, + PositionsToReturn = + [ + new StreamPosition( + StreamId.Create("namespace", Guid.NewGuid()), + recoveredToken), + ], + }; + var replacementReceiver = new TestEventHubReceiver + { + Messages = [new EventData()], + }; + var caches = new Queue([initialCache, replacementCache]); + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + onReceiverCreated: offsets.Add, + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver, + cacheFactory: caches.Dequeue); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var abandonedCursor = AssertCursor(receiver, streamId, MakeToken(1)); + initialCache.MoveNextException = new InvalidOperationException("Cache cursor failed"); + Assert.Throws(() => abandonedCursor.MoveNextWithResult()); + initialCache.MoveNextException = null; + + var cursor = AssertCursor(receiver, streamId, MakeToken(1)); + var pendingCursor = AssertCursor(receiver, StreamId.Create("namespace", Guid.NewGuid()), MakeToken(1)); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(initialBatch, cursor.GetCurrent(out _)); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(1, checkpointer.ResetCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(1, initialCache.DisposeCount); + + receiver.UpdateDeliveryProgress(MakeToken(123), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + Assert.Single(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(["123", EventHubConstants.StartOfStream], offsets); + + var refreshToken = MakeToken(2); + cursor.Refresh(refreshToken); + Assert.Equal(1, replacementCache.GetCursorCount); + Assert.Null(cursor.GetCurrent(out _)); + + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(recoveredBatch, cursor.GetCurrent(out _)); + receiver.UpdateDeliveryProgress(recoveredToken, DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + pendingCursor.Dispose(); + receiver.UpdateDeliveryProgress(recoveredToken, DateTime.UtcNow); + Assert.Equal("10", checkpointer.LastOffset); + Assert.Equal(1, checkpointer.UpdateCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorGetCurrent_WhenRecoveryDetachesCache_DiscardsSelectedBatch() + { + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var initialBatch = new TestBatchContainer(); + var initialCache = new TestEventHubQueueCache + { + NextMessage = initialBatch, + }; + var caches = new Queue( + [initialCache, new TestEventHubQueueCache()]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); + using var cursor = AssertCursor( + receiver, + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + Assert.Same(initialBatch, cursor.GetCurrent(out _)); + + var recovery = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + + Assert.Null(cursor.GetCurrent(out _)); + + checkpointer.ReleaseReset.TrySetResult(); + await Assert.ThrowsAsync(() => recovery); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenShutdownDetachesFailedRead_DoesNotReset() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var eventHubReceiver = new BlockingInvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: eventHubReceiver); + + var read = receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + await eventHubReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + eventHubReceiver.ReleaseReceive.TrySetResult(); + + await Assert.ThrowsAsync(() => read); + Assert.Equal(0, checkpointer.ResetCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenArgumentFailureIsNotAnInvalidOffset_DoesNotReset() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var eventHubReceiver = new InvalidArgumentEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: eventHubReceiver, + onReceiverCreated: offsets.Add); + + var exception = await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal("The receiver rejected an unrelated argument.", exception.Message); + Assert.Equal(0, checkpointer.ResetCount); + Assert.Equal(0, eventHubReceiver.CloseCount); + Assert.Equal(["123"], offsets); + } + + [TestSuite("BVT")] + [Theory, TestCategory("BVT")] + [InlineData(true, true, "The supplied offset '13953' is invalid. The last offset in the system is '13952' TrackingId:example\nFor troubleshooting information, see https://aka.ms/azsdk/net/eventhubs/exceptions/troubleshoot", true)] + [InlineData(false, true, "The supplied offset '13953' is invalid.", false)] + [InlineData(true, false, "The supplied offset '13953' is invalid.", false)] + [InlineData(true, true, "The supplied offset must be numeric.", false)] + [InlineData(true, true, "The consumer group is invalid.", false)] + public async Task GetQueueMessagesAsync_ClassifiesKnownServiceOffsetError( + bool checkpointExists, bool argumentException, string message, bool recover) + { + var checkpointer = new TestCheckpointer { CheckpointExists = checkpointExists, LoadedOffset = "13953" }; + var cache = new TestEventHubQueueCache(); + var failure = argumentException ? (Exception)new ArgumentException(message) : new InvalidOperationException(message); + var failedReceiver = new InvalidArgumentEventHubReceiver { Failure = failure }; + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => offsets.Count == 0 ? cache : new TestEventHubQueueCache(), + onReceiverCreated: offsets.Add, + receiverFactory: _ => offsets.Count == 1 ? failedReceiver : new TestEventHubReceiver()); + + Assert.Same(failure, await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken))); + Assert.Equal(recover ? 1 : 0, checkpointer.ResetCount); + Assert.Equal(recover ? 1 : 0, failedReceiver.CloseCount); + Assert.Equal(recover ? 1 : 0, cache.DisposeCount); + Assert.Equal( + recover ? ["13953", EventHubConstants.StartOfStream] : new[] { checkpointExists ? "13953" : EventHubConstants.StartOfStream }, + offsets); + } + + [TestSuite("BVT")] + [Theory] + [InlineData(false)] + [InlineData(true)] + [TestCategory("BVT")] + public async Task GetQueueMessagesAsync_AcceptsResumeTokenResolvedByReplacementCache( + bool useLegacyCursor) + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var recoveredToken = MakeToken(10); + var replacementCache = new TestEventHubQueueCache + { + NextMessage = new TestBatchContainer + { + SequenceToken = recoveredToken, + }, + PositionsToReturn = + [ + new StreamPosition( + StreamId.Create("namespace", Guid.NewGuid()), + recoveredToken), + ], + }; + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver { Messages = [new EventData()] }, + cacheFactory: caches.Dequeue); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Single(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + var resumeToken = MakeToken(9); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + IQueueCacheCursor cursor; + if (useLegacyCursor) + { +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor API. + cursor = ((IQueueCache)receiver).GetCacheCursor(streamId, resumeToken); +#pragma warning restore CS0618 + } + else + { + cursor = AssertCursor(receiver, streamId, resumeToken); + } + + using (cursor) + { + Assert.Equal(QueueCacheCursorMoveResultKind.Success, cursor.MoveNextWithResult().Kind); + + receiver.UpdateDeliveryProgress(MakeToken(9), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + + receiver.UpdateDeliveryProgress(resumeToken, DateTime.UtcNow); + Assert.Equal("9", checkpointer.LastOffset); + Assert.Equal(1, checkpointer.UpdateCount); + } + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorRefresh_WhenReplacementCacheMisses_LegacyMoveNextThrows() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var replacementCache = new TestEventHubQueueCache + { + CursorException = new QueueCacheMissException("requested", "low", "high"), + }; + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor APIs. + var cursor = ((IQueueCache)receiver).GetCacheCursor( + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); +#pragma warning restore CS0618 + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + cursor.Refresh(MakeToken(2)); + cursor.Refresh(MakeToken(3)); + Assert.Equal(0, replacementCache.RefreshCount); + +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor APIs. + var exception = Assert.Throws(() => cursor.MoveNext()); +#pragma warning restore CS0618 + Assert.Equal("requested", exception.Requested); + Assert.Equal("low", exception.Low); + Assert.Equal("high", exception.High); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task CursorRefresh_WhenReplacementAcquisitionIsUnsupported_RetriesAcquisition() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var replacementCache = new SequencedAcquisitionEventHubQueueCache(); + replacementCache.Results.Enqueue(QueueCacheCursorResult.NotSupported); + replacementCache.Results.Enqueue( + QueueCacheCursorResult.FromCursor(replacementCache.Cursor)); + var caches = new Queue([new TestEventHubQueueCache(), replacementCache]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); +#pragma warning disable CS0618 // Verify compatibility of the obsolete cursor API. + using var cursor = ((IQueueCache)receiver).GetCacheCursor( + StreamId.Create("namespace", Guid.NewGuid()), + MakeToken(1)); +#pragma warning restore CS0618 + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Throws(() => cursor.Refresh(MakeToken(2))); + cursor.Refresh(MakeToken(2)); + + Assert.Equal(2, replacementCache.RequestCount); + Assert.Null(replacementCache.RefreshedCursor); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenCheckpointRecoveryFails_RethrowsReadFailure() + { + var checkpointer = new FailingResetCheckpointer + { + LoadedOffset = "123", + }; + var caches = new Queue( + [new TestEventHubQueueCache(), new TestEventHubQueueCache()]); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver(), + cacheFactory: caches.Dequeue); + + var exception = await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + + Assert.Equal("The supplied offset is invalid.", exception.Message); + Assert.Equal(1, checkpointer.ResetCount); + + Assert.Empty(await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)); + receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); + Assert.Equal(0, checkpointer.UpdateCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_RetriesFailedReceiverCloseBeforeReplacement() + { + var checkpointer = new TestCheckpointer + { + LoadedOffset = "123", + }; + var invalidReceiver = new RetryingCloseEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var offsets = new List(); + var receiver = await CreateReceiver( + checkpointer, + onReceiverCreated: offsets.Add, + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(["123"], offsets); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + Assert.Equal(2, invalidReceiver.CloseCount); + Assert.Equal(["123", EventHubConstants.StartOfStream], offsets); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task GetQueueMessagesAsync_WhenShutdownDetachesReceiver_DiscardsInFlightRead() + { + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new BlockingReadEventHubReceiver(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + cache, + eventHubReceiver); + + var read = receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + await eventHubReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + eventHubReceiver.ReleaseReceive.TrySetResult(); + + Assert.Empty(await read); + Assert.Equal(0, cache.AddCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + + [TestSuite("BVT")] + [Theory, TestCategory("BVT")] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ResetReceiver_WhenShutdownWins_DrainsRecoveryBeforeNextLifetime(bool cancelRead, bool timeoutShutdown) + { + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var initialCache = new TestEventHubQueueCache(); + var replacementCache = new TestEventHubQueueCache(); + var caches = new Queue([initialCache, replacementCache]); + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var replacementReceiver = new TestEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" ? invalidReceiver : replacementReceiver, + cacheFactory: caches.Dequeue); + + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var recovery = receiver.GetQueueMessagesAsync(10, cancellation.Token); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + if (cancelRead) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => recovery); + } + + if (timeoutShutdown) + { + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + } + + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.False(shutdown.IsCompleted); + Assert.Equal(CancellationToken.None, checkpointer.ResetCancellationToken); + Assert.Equal(0, checkpointer.FlushCount); + Assert.Equal(0, initialCache.DisposeCount); + Assert.Equal(0, invalidReceiver.CloseCount); + + checkpointer.ReleaseReset.TrySetResult(); + await shutdown; + if (!cancelRead) + { + await Assert.ThrowsAsync(() => recovery); + } + + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, initialCache.DisposeCount); + Assert.Equal(0, replacementCache.DisposeCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(0, replacementReceiver.CloseCount); + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(42)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal("42", checkpointer.FlushedOffset); + Assert.Equal(2, checkpointer.FlushCount); + Assert.Equal(1, initialCache.DisposeCount); + Assert.Equal(1, replacementCache.DisposeCount); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.Equal(1, replacementReceiver.CloseCount); + Assert.Equal(1, checkpointer.ResetCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task ResetReceiver_WhenCanceled_ClosesReceiverWithIndependentToken() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + var checkpointer = new CancelingResetCheckpointer(cancellation) + { + LoadedOffset = "123", + }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver( + checkpointer, + eventHubReceiver: invalidReceiver); + + var exception = await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, invalidReceiver.CloseCount); + Assert.False(invalidReceiver.CloseCancellationToken.IsCancellationRequested); + + receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow); + Assert.Null(checkpointer.LastOffset); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task ResetReceiver_WhenCallerCancels_ResetRemainsRecoveryBarrier() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + var checkpointer = new BlockingResetCheckpointer + { + LoadedOffset = "123", + }; + var receiver = await CreateReceiver( + checkpointer, + receiverFactory: offset => offset == "123" + ? new InvalidOffsetEventHubReceiver() + : new TestEventHubReceiver()); + + var canceledRead = receiver.GetQueueMessagesAsync(10, cancellation.Token); + await checkpointer.ResetStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledRead); - return receiver; + var waitingRead = receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken); + Assert.False(waitingRead.IsCompleted); + + checkpointer.ReleaseReset.TrySetResult(); + Assert.Empty(await waitingRead); } [TestSuite("BVT")] [Fact, TestCategory("BVT")] - public async Task Initialize_WhenCheckpointDoesNotExist_UsesStartOfStream() + public async Task UpdateDeliveryProgress_WhenRecoveryStarts_WritesBeforeReset() { - string? receiverOffset = null; - var checkpointer = new TestCheckpointer + var checkpointer = new BlockingUpdateCheckpointer { - CheckpointExists = false, - LoadedOffset = string.Empty, + LoadedOffset = "123", }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, eventHubReceiver: invalidReceiver); + + var update = Task.Run( + () => receiver.UpdateDeliveryProgress(MakeToken(124), DateTime.UtcNow), + TestContext.Current.CancellationToken); + await checkpointer.UpdateStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var recovery = Task.Run( + async () => await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + await invalidReceiver.ReceiveStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + + Assert.False(checkpointer.ResetStarted.Task.IsCompleted); + checkpointer.ReleaseUpdate.TrySetResult(); + + await update; + await Assert.ThrowsAsync(() => recovery); + Assert.Equal("124", checkpointer.LastOffset); + Assert.True(checkpointer.ResetStarted.Task.IsCompleted); + } - _ = await CreateReceiver( - checkpointer, - onReceiverCreated: offset => receiverOffset = offset); + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task NoOpCheckpointerFactory_CreateHonorsCancellation() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); - Assert.Equal(EventHubConstants.StartOfStream, receiverOffset); + var exception = await Assert.ThrowsAnyAsync( + () => NoOpCheckpointerFactory.Instance.Create("partition", cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); } [TestSuite("BVT")] [Fact, TestCategory("BVT")] - public async Task GetQueueMessagesAsync_TreatsNullReceiverResultAsEmpty() + public async Task GetQueueMessagesAsync_WhenNoCheckpointExists_DoesNotResetArgumentFailure() { - var cache = new TestEventHubQueueCache(); - var eventHubReceiver = new NullReturningEventHubReceiver(); - var receiver = await CreateReceiver(new TestCheckpointer(), cache, eventHubReceiver); + var checkpointer = new TestCheckpointer + { + CheckpointExists = false, + LoadedOffset = string.Empty, + }; + var invalidReceiver = new InvalidOffsetEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, eventHubReceiver: invalidReceiver); + checkpointer.CheckpointExists = true; - var messages = await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); - Assert.Empty(messages); - Assert.Equal(1, eventHubReceiver.ReceiveCount); - Assert.Equal(0, cache.AddCount); + Assert.Equal(0, checkpointer.ResetCount); + Assert.Equal(0, invalidReceiver.CloseCount); } [TestSuite("BVT")] @@ -509,6 +1629,280 @@ public async Task LegacyReceiverCursorPreservesProviderExceptions() Assert.Same(movementInnerException, actualMovementException.InnerException); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterReinitialization_CleansUpEachLifetimeOnce() + { + var checkpointers = new[] { new TestCheckpointer(), new TestCheckpointer() }; + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new TestEventHubReceiver(), new TestEventHubReceiver() }; + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointers[0], + cacheFactory: () => caches[lifetime], + receiverFactory: _ => eventHubReceivers[lifetime], + checkpointerFactory: _ => Task.FromResult>(checkpointers[lifetime]), + initialize: false); + + for (; lifetime < 2; lifetime++) + { + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(100 + lifetime)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal((100 + lifetime).ToString(CultureInfo.InvariantCulture), checkpointers[lifetime].FlushedOffset); + Assert.Equal(1, checkpointers[lifetime].FlushCount); + Assert.Equal(1, caches[lifetime].DisposeCount); + Assert.Equal(1, eventHubReceivers[lifetime].CloseCount); + } + + Assert.All(checkpointers, checkpointer => Assert.Equal(1, checkpointer.FlushCount)); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.All(eventHubReceivers, eventHubReceiver => Assert.Equal(1, eventHubReceiver.CloseCount)); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_ConcurrentWithShutdown_StartsAfterShutdownCompletes() + { + var firstCheckpointer = new BlockingFlushCheckpointer(); + var secondCheckpointer = new TestCheckpointer(); + var checkpointers = new Queue>( + [firstCheckpointer, secondCheckpointer]); + var caches = new Queue( + [new TestEventHubQueueCache(), new TestEventHubQueueCache()]); + var eventHubReceivers = new Queue( + [new TestEventHubReceiver(), new TestEventHubReceiver()]); + var receiver = await CreateReceiver( + firstCheckpointer, + cacheFactory: () => caches.Dequeue(), + receiverFactory: _ => eventHubReceivers.Dequeue(), + checkpointerFactory: _ => Task.FromResult(checkpointers.Dequeue())); + + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await firstCheckpointer.FlushStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + var initialization = receiver.Initialize( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.False(initialization.IsCompleted); + Assert.Single(checkpointers); + Assert.Single(caches); + Assert.Single(eventHubReceivers); + + firstCheckpointer.ReleaseFlush.TrySetResult(); + await shutdown; + await initialization; + + Assert.Empty(await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Empty(checkpointers); + Assert.Empty(caches); + Assert.Empty(eventHubReceivers); + Assert.Equal(1, firstCheckpointer.FlushCount); + Assert.Equal(1, secondCheckpointer.FlushCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_CanceledWhileWaitingForShutdown_PreservesCallerToken() + { + var checkpointer = new BlockingFlushCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await checkpointer.FlushStarted.Task.WaitAsync( + TestContext.Current.CancellationToken); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + + var initialization = receiver.Initialize( + TimeSpan.FromSeconds(30), + cancellation.Token); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + + checkpointer.ReleaseFlush.TrySetResult(); + await shutdown; + } + + [TestSuite("BVT")] + [Theory, TestCategory("BVT")] + [InlineData(false)] + [InlineData(true)] + public async Task Initialize_AfterShutdownAndFailedReinitialization_RetriesAndCleansUp(bool cancelInitialization) + { + var checkpointers = new[] { new TestCheckpointer(), new TestCheckpointer(), new TestCheckpointer() }; + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new CleanupTrackingEventHubReceiver(), new CleanupTrackingEventHubReceiver(), new CleanupTrackingEventHubReceiver() }; + var attempt = 0; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var expected = new InvalidOperationException("Receiver creation failed"); + var receiver = await CreateReceiver( + checkpointers[0], + cacheFactory: () => caches[attempt], + receiverFactory: _ => + { + if (attempt == 1) + { + if (!cancelInitialization) + { + throw expected; + } + + cancellation.Cancel(); + } + + return eventHubReceivers[attempt]; + }, + checkpointerFactory: _ => Task.FromResult>(checkpointers[attempt])); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + attempt++; + var initialization = receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token); + if (cancelInitialization) + { + var exception = await Assert.ThrowsAnyAsync(() => initialization); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False(eventHubReceivers[1].CloseCancellationToken.IsCancellationRequested); + } + else + { + Assert.Same(expected, await Assert.ThrowsAsync(() => initialization)); + } + + attempt++; + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + UpdateDeliveryProgress(receiver, MakeToken(200)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointers[0].FlushCount); + Assert.Equal(0, checkpointers[1].FlushCount); + Assert.Equal(1, checkpointers[2].FlushCount); + Assert.Equal("200", checkpointers[2].FlushedOffset); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.Equal(1, eventHubReceivers[0].CloseCount); + Assert.Equal(cancelInitialization ? 1 : 0, eventHubReceivers[1].CloseCount); + Assert.Equal(1, eventHubReceivers[2].CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Initialize_AfterShutdownWithCanceledToken_PreservesCompletedLifetime() + { + var checkpointer = new TestCheckpointer(); + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new TestEventHubReceiver(); + var creations = 0; + var receiver = await CreateReceiver( + checkpointer, cache, eventHubReceiver, onReceiverCreated: _ => creations++); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync( + () => receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Empty(await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, creations); + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterCanceledReinitialization_RetriesPendingCleanup() + { + var checkpointer = new TestCheckpointer(); + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var originalReceiver = new TestEventHubReceiver(); + var failedReceiver = new RetryingCloseEventHubReceiver(); + using var cancellation = new CancellationTokenSource(); + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => caches[lifetime], + receiverFactory: _ => + { + if (lifetime == 0) + { + return originalReceiver; + } + + cancellation.Cancel(); + return failedReceiver; + }); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + lifetime++; + var failure = await Assert.ThrowsAsync( + () => receiver.Initialize(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.IsAssignableFrom(failure.InnerExceptions[0]); + Assert.Equal("Close failed", failure.InnerExceptions[1].Message); + Assert.Equal(1, failedReceiver.CloseCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, originalReceiver.CloseCount); + Assert.Equal(2, failedReceiver.CloseCount); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_DrainsInitializationBeforeNextLifetime() + { + var checkpointer = new TestCheckpointer(); + var caches = new[] { new TestEventHubQueueCache(), new TestEventHubQueueCache() }; + var eventHubReceivers = new[] { new TestEventHubReceiver(), new TestEventHubReceiver() }; + var factoryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lifetime = 0; + var receiver = await CreateReceiver( + checkpointer, + cacheFactory: () => caches[lifetime], + receiverFactory: _ => eventHubReceivers[lifetime], + checkpointerFactory: async cancellationToken => + { + factoryStarted.TrySetResult(); + await releaseFactory.Task.WaitAsync(cancellationToken); + return checkpointer; + }, + initialize: false); + + var initialization = receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await factoryStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var shutdown = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.False(shutdown.IsCompleted); + releaseFactory.TrySetResult(); + await Task.WhenAll(initialization, shutdown); + Assert.Equal(0, checkpointer.FlushCount); + Assert.Equal(1, caches[0].DisposeCount); + Assert.Equal(1, eventHubReceivers[0].CloseCount); + + lifetime++; + await receiver.Initialize(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + UpdateDeliveryProgress(receiver, MakeToken(300)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal("300", checkpointer.FlushedOffset); + Assert.All(caches, cache => Assert.Equal(1, cache.DisposeCount)); + Assert.All(eventHubReceivers, eventHubReceiver => Assert.Equal(1, eventHubReceiver.CloseCount)); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Shutdown_DisposesCacheAndClosesReceiver_WhenFlushFails() @@ -525,6 +1919,59 @@ public async Task Shutdown_DisposesCacheAndClosesReceiver_WhenFlushFails() Assert.Equal(1, eventHubReceiver.CloseCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_WhenFlushTimesOut_ClosesReceiverWithIndependentToken() + { + var eventHubReceiver = new CleanupTrackingEventHubReceiver(); + var receiver = await CreateReceiver( + new CanceledFlushCheckpointer(), + eventHubReceiver: eventHubReceiver); + + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(eventHubReceiver.CloseCancellationToken.IsCancellationRequested); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_RetriesFailedFlushBeforeCompleting() + { + var checkpointer = new FailOnceFlushCheckpointer(); + var cache = new TestEventHubQueueCache(); + var eventHubReceiver = new TestEventHubReceiver(); + var receiver = await CreateReceiver(checkpointer, cache, eventHubReceiver); + + await Assert.ThrowsAsync( + () => receiver.Shutdown(TimeSpan.FromSeconds(5))); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, checkpointer.FlushAttempts); + Assert.Equal(1, checkpointer.FlushCount); + Assert.Equal(1, cache.DisposeCount); + Assert.Equal(1, eventHubReceiver.CloseCount); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_ConcurrentCallerWaitsForCompletion() + { + var checkpointer = new BlockingFlushCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + + var first = receiver.Shutdown(TimeSpan.FromSeconds(5)); + await checkpointer.FlushStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var second = receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.False(second.IsCompleted); + checkpointer.ReleaseFlush.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, checkpointer.FlushCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task Shutdown_CancelsBlockedReceiverClose() @@ -544,6 +1991,28 @@ await Assert.ThrowsAnyAsync( Assert.Equal(1, eventHubReceiver.CloseCount); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_AfterTimeoutAwaitsExistingReceiverClose() + { + var eventHubReceiver = new NonCancelableCloseEventHubReceiver(); + var receiver = await CreateReceiver( + new TestCheckpointer(), + eventHubReceiver: eventHubReceiver); + + await Assert.ThrowsAnyAsync( + () => receiver.Shutdown(TimeSpan.FromMilliseconds(50))); + + var retry = receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, eventHubReceiver.CloseCount); + Assert.False(retry.IsCompleted); + + eventHubReceiver.ReleaseClose.TrySetResult(); + await retry; + + Assert.Equal(1, eventHubReceiver.CloseCount); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task FlushBeforeLoad_DoesNotPersistUninitializedOffset() @@ -601,6 +2070,32 @@ public void Update_WithNoComparer_TracksOpaqueCheckpoint() Assert.Equal("opaque-checkpoint", GetEntityOffset(checkpointer)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void Update_WhenLatestCheckpointIsNotPersisted_RetriesSave() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetField(checkpointer, "_latestCheckpoint", "20"); + SetField(checkpointer, "_persistedCheckpoint", string.Empty); + var previousSave = GetField(checkpointer, "_inProgressSave"); + + checkpointer.Update("20", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + + Assert.NotSame(previousSave, GetField(checkpointer, "_inProgressSave")); + } + + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void AzureTableCheckpointer_FailedResetRejectsRegressedConcurrentUpdate() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetField(checkpointer, "_latestCheckpoint", "50"); + + InvokeRestoreLatestCheckpoint(checkpointer, "100"); + + Assert.Equal("100", GetLatestOffset(checkpointer)); + } + [TestSuite("BVT")] [Theory, TestCategory("BVT")] [InlineData("", "provider_service")] @@ -612,6 +2107,22 @@ public void PartitionKeyPrefix_IsAppliedPerCheckpointer(string partitionKeyPrefi Assert.Equal(expected, GetEntityPartitionKey(checkpointer)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public void AzureTableCheckpointer_CreatesIndependentWriteEntity() + { + var checkpointer = CreateUninitializedCheckpointer(); + SetPersistedOffset(checkpointer, "10"); + + var writeEntity = InvokeCreateWriteEntity(checkpointer, "20"); + + Assert.NotSame(GetField(checkpointer, "_entity"), writeEntity); + Assert.Equal(GetEntityPartitionKey(checkpointer), GetEntityProperty(writeEntity, "PartitionKey")); + Assert.Equal(GetEntityRowKey(checkpointer), GetEntityProperty(writeEntity, "RowKey")); + Assert.Equal("20", GetEntityProperty(writeEntity, "Offset")); + Assert.Equal("10", GetEntityOffset(checkpointer)); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public void EventHubCheckpointEntity_PreservesLegacyAzureTableSchema() @@ -658,6 +2169,25 @@ public async Task EventHubCompatibilityWrapper_PreservesLoadSemantics( Assert.Equal(expected, await checkpointer.Load(TestContext.Current.CancellationToken)); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task EventHubCompatibilityWrapper_DelegatesReset() + { + var inner = new TestCheckpointer(); + var constructor = typeof(EventHubCheckpointer).GetConstructor( + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [typeof(IStreamQueueCheckpointer)], + modifiers: null); + Assert.NotNull(constructor); + var checkpointer = (EventHubCheckpointer)constructor.Invoke([inner]); + + await checkpointer.Reset(CancellationToken.None); + + Assert.Equal(1, inner.ResetCount); + Assert.False(checkpointer.CheckpointExists); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task SingleSubscription_CheckpointsProcessedOffset() @@ -688,6 +2218,20 @@ public async Task Shutdown_FlushesLatestDeliveryProgress() Assert.Equal("100", checkpointer.FlushedOffset); } + [TestSuite("BVT")] + [Fact, TestCategory("BVT")] + public async Task Shutdown_RejectsSubsequentDeliveryProgress() + { + var checkpointer = new TestCheckpointer(); + var receiver = await CreateReceiver(checkpointer); + + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + receiver.UpdateDeliveryProgress(MakeToken(100), DateTime.UtcNow); + + Assert.Equal(0, checkpointer.UpdateCount); + Assert.Null(checkpointer.LastOffset); + } + [TestSuite("BVT")] [Fact, TestCategory("BVT")] public async Task MultipleSubscriptions_CheckpointsMinimumWatermark() @@ -876,6 +2420,45 @@ private static void SetEntityOffset(AzureTableStreamQueueCheckpointer checkpoint entity.GetType().GetProperty("Offset")!.SetValue(entity, offset); } + private static object InvokeCreateWriteEntity( + AzureTableStreamQueueCheckpointer checkpointer, + string offset) + { + var method = typeof(AzureTableStreamQueueCheckpointer).GetMethod( + "CreateWriteEntity", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(method); + return method.Invoke(checkpointer, [offset])!; + } + + private static Task InvokeInitialize( + EventHubAdapterReceiver receiver, + CancellationToken cancellationToken) + { + var method = typeof(EventHubAdapterReceiver).GetMethod( + "InitializeCore", + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [typeof(CancellationToken)], + modifiers: null); + Assert.NotNull(method); + return (Task)method.Invoke(receiver, [cancellationToken])!; + } + + private static void InvokeRestoreLatestCheckpoint( + AzureTableStreamQueueCheckpointer checkpointer, + string offset) + { + var method = typeof(AzureTableStreamQueueCheckpointer).GetMethod( + "RestoreLatestCheckpoint", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(method); + method.Invoke(checkpointer, [offset]); + } + + private static string GetEntityProperty(object entity, string propertyName) + => (string)entity.GetType().GetProperty(propertyName)!.GetValue(entity)!; + private static object GetField(AzureTableStreamQueueCheckpointer checkpointer, string name) { var field = typeof(AzureTableStreamQueueCheckpointer).GetField( diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs index e552ee4fcdf..509befbb3ea 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs @@ -70,7 +70,7 @@ public EHPurgeLogicTests() [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WontPurge_WhenUnderPressure() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache, make sure will allocate multiple buffers from the pool int itemAddToCache = 100; @@ -93,7 +93,7 @@ public async Task EventHubQueueCache_WontPurge_WhenUnderPressure() [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WontPurge_WhenTimePurgePredicateSaysDontPurge() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -118,7 +118,7 @@ public async Task EventHubQueueCache_WontPurge_WhenTimePurgePredicateSaysDontPur [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_WillPurge_WhenTimePurgePredicateSaysPurge_And_NotUnderPressure() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -144,7 +144,7 @@ public async Task EventHubQueueCache_WillPurge_WhenTimePurgePredicateSaysPurge_A [Fact, TestCategory("BVT")] public async Task EventHubQueueCache_EvictionStrategy_Behavior() { - InitForTesting(); + await InitForTesting(); var tasks = new List(); //add items into cache int itemAddToCache = 100; @@ -200,7 +200,7 @@ public async Task EventHubQueueCache_EvictionStrategy_Behavior() expectedPurgedBuffers.ForEach(buffer => Assert.Contains(buffer, newBufferAllocated)); } - private void InitForTesting() + private async Task InitForTesting() { this.cacheList = new ConcurrentBag(); this.evictionStrategyList = new List(); @@ -211,11 +211,14 @@ private void InitForTesting() }; this.receiver1 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, - new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider); + new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, + static (_, _, _) => new NoOpEventHubReceiver()); this.receiver2 = new EventHubAdapterReceiver(this.ehSettings, this.CacheFactory, this.CheckPointerFactory, NullLoggerFactory.Instance, - new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider); - this.receiver1.Initialize(this.timeOut); - this.receiver2.Initialize(this.timeOut); + new DefaultEventHubReceiverMonitor(monitorDimensions, this.instruments), new LoadSheddingOptions(), environmentStatisticsProvider, + static (_, _, _) => new NoOpEventHubReceiver()); + await Task.WhenAll( + this.receiver1.Initialize(this.timeOut), + this.receiver2.Initialize(this.timeOut)); } private int GetItemCountInAllCache(ConcurrentBag caches) @@ -271,5 +274,13 @@ private IEventHubQueueCache CacheFactory(string partition, IStreamQueueCheckpoin this.cacheList.Add(cache); return cache; } + + private sealed class NoOpEventHubReceiver : IEventHubReceiver + { + public Task> ReceiveAsync(int maxCount, TimeSpan waitTime) + => Task.FromResult>([]); + + public Task CloseAsync() => Task.CompletedTask; + } } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs index ba67802a1d4..e9bed8be8fc 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs @@ -58,6 +58,20 @@ public async ValueTask Update( [TestCategory("AWS"), TestCategory("Kinesis")] public sealed class DynamoDBStreamCheckpointStoreTests { + [Fact] + public async Task LoadTreatsPersistedEmptyCheckpointAsNoCheckpoint() + { + var client = Substitute.For(); + client.GetItemAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateReadResponse(string.Empty, 7))); + var checkpointer = new DynamoDBStreamQueueCheckpointer( + CreateStore(client), + new DynamoDBStreamQueueCheckpointerOptions()); + + Assert.Equal(string.Empty, await checkpointer.Load(TestContext.Current.CancellationToken)); + Assert.False(checkpointer.CheckpointExists); + } + [Fact] public async Task UpdatePersistsArbitrarySizeSequenceNumberAsString() { diff --git a/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs index a1a3d54ecea..9861382e55c 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs @@ -112,6 +112,49 @@ public async Task ConditionalConflict_WithoutComparerAdoptsAuthoritativeCheckpoi Assert.Equal("40", (await store.Load(CancellationToken.None)).Checkpoint); } + [Fact] + public async Task Reset_WhenConflictAdvancesCheckpointThenRetryFails_PreservesAuthoritativeCheckpoint() + { + var store = new ResetConflictThenFailureStore(); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal("10", await checkpointer.Load(CancellationToken.None)); + + await Assert.ThrowsAsync( + () => checkpointer.Reset(CancellationToken.None)); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal([("version-1", string.Empty), ("version-2", string.Empty)], store.Attempts); + Assert.Equal("20", (await store.Load(CancellationToken.None)).Checkpoint); + } + + [Fact] + public async Task Reset_WhenConflictAndConcurrentUpdateRegress_PreservesLargestCheckpoint() + { + var store = new ResetConflictThenFailureStore("100", "80"); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal("100", await checkpointer.Load(CancellationToken.None)); + store.OnConflict = () => checkpointer.Update("50", DateTime.UtcNow, CancellationToken.None); + + await Assert.ThrowsAsync( + () => checkpointer.Reset(CancellationToken.None)); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal( + [("version-1", string.Empty), ("version-2", string.Empty), ("version-2", "100")], + store.Attempts); + Assert.Equal("100", (await store.Load(CancellationToken.None)).Checkpoint); + } + private sealed class TestCheckpointStore(ControllableCheckpointStore store) : IStreamCheckpointStore { public async ValueTask Load(CancellationToken cancellationToken) @@ -167,6 +210,55 @@ public ValueTask Update( } } + private sealed class ResetConflictThenFailureStore : IStreamCheckpointStore + { + private StreamCheckpointStoreState state; + private readonly string conflictCheckpoint; + private int updateCount; + + public List<(string ExpectedVersion, string Checkpoint)> Attempts { get; } = []; + + public Action? OnConflict { get; set; } + + public ResetConflictThenFailureStore( + string initialCheckpoint = "10", + string conflictCheckpoint = "20") + { + state = new(initialCheckpoint, "version-1"); + this.conflictCheckpoint = conflictCheckpoint; + } + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(state); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Attempts.Add((expectedVersion, checkpoint)); + updateCount++; + if (updateCount == 1) + { + state = new(conflictCheckpoint, "version-2"); + OnConflict?.Invoke(); + return ValueTask.FromResult(state); + } + + if (updateCount == 2) + { + throw new InvalidOperationException("checkpoint reset failed"); + } + + state = new(checkpoint, "version-3"); + return ValueTask.FromResult(state); + } + } + private sealed class EmptyCheckpointConflictStore : IStreamCheckpointStore { private StreamCheckpointStoreState state = new(string.Empty, "version-1"); diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index 032005ed111..03470527867 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -60,6 +60,220 @@ public async Task Load_WhenCheckpointExists_ReturnsPersistedCheckpoint() Assert.Empty(store.WriteAttempts); } + [Fact] + public async Task Reset_ClearsPersistedCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + + await checkpointer.Reset(CancellationToken.None); + + Assert.False(checkpointer.CheckpointExists); + Assert.Equal(NoCheckpoint, await checkpointer.Load(CancellationToken.None)); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.Equal([NoCheckpoint], store.WriteAttempts); + Assert.Equal([NoCheckpoint], store.CompletedWrites); + } + + [Fact] + public async Task Reset_WhenCanceledBehindPendingWrite_RetryRemainsSerialized() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedWrite = store.BlockNextWrite(); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForWriteAttempts(1); + using var cancellation = new CancellationTokenSource(); + + var canceledReset = checkpointer.Reset(cancellation.Token); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledReset); + + var retry = checkpointer.Reset(CancellationToken.None); + Assert.False(retry.IsCompleted); + Assert.Equal(["20"], store.WriteAttempts); + + blockedWrite.SetResult(); + await retry; + + Assert.Equal(["20", NoCheckpoint], store.WriteAttempts); + Assert.Equal(["20", NoCheckpoint], store.CompletedWrites); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.False(checkpointer.CheckpointExists); + } + + [Fact] + public async Task Reset_AfterPendingWriteFails_ClearsCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + store.FailNextWrite(new InvalidOperationException("checkpoint write failed")); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + + await checkpointer.Reset(CancellationToken.None); + + Assert.Equal(["20", NoCheckpoint], store.WriteAttempts); + Assert.Equal([NoCheckpoint], store.CompletedWrites); + Assert.Equal(NoCheckpoint, store.PersistedCheckpoint); + Assert.False(checkpointer.CheckpointExists); + } + + [Fact] + public async Task Reset_WhenWriteFails_PreservesConcurrentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("checkpoint reset failed")); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => reset); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal([NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal(["20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenWriteFails_ReconcilesConcurrentRegressedUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("100"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("checkpoint reset failed")); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("50", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => reset); + await checkpointer.FlushAsync(CancellationToken.None); + + var expected = RegressionPolicy is OffsetRegressionPolicy.Ignore ? "100" : "50"; + Assert.Equal(expected, store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenNestedResetFails_ClearsThrottleMarker() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + store.FailNextWrite(new InvalidOperationException("nested checkpoint reset failed")); + var secondReset = checkpointer.Reset(CancellationToken.None); + + blockedReset.SetResult(); + await firstReset; + await Assert.ThrowsAsync(() => secondReset); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForCompletedWrites(2).WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal([NoCheckpoint, NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenUpdatePrecedesFailedNestedReset_RestoresUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + store.FailNextWrite(new InvalidOperationException("nested checkpoint reset failed")); + var secondReset = checkpointer.Reset(CancellationToken.None); + + blockedReset.SetResult(); + await firstReset; + await Assert.ThrowsAsync(() => secondReset); + + Assert.True(checkpointer.CheckpointExists); + Assert.Equal([NoCheckpoint, NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenAllNestedResetsFail_RestoresCheckpoint() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + store.FailNextWrite(new InvalidOperationException("first checkpoint reset failed")); + var firstReset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + var secondReset = checkpointer.Reset(CancellationToken.None); + store.FailNextWrite(new InvalidOperationException("second checkpoint reset failed")); + + blockedReset.SetResult(); + await Assert.ThrowsAsync(() => firstReset); + await Assert.ThrowsAsync(() => secondReset); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.True(checkpointer.CheckpointExists); + Assert.Equal([NoCheckpoint, NoCheckpoint], store.WriteAttempts); + Assert.Empty(store.CompletedWrites); + Assert.Equal("10", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenUpdateAndFlushOverlap_PersistsSubsequentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var resetWrite = store.BlockNextWrite(); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + var flush = checkpointer.FlushAsync(CancellationToken.None); + resetWrite.SetResult(); + + await Task.WhenAll(reset, flush); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + + checkpointer.Update("30", TestTimeUtc + PersistInterval, CancellationToken.None); + await store.WaitForCompletedWrites(3).WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal([NoCheckpoint, "20", "30"], store.CompletedWrites); + Assert.Equal("30", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenQueuedBehindSave_PreservesSubsequentUpdate() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedWrite = store.BlockNextWrite(); + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + await store.WaitForWriteAttempts(1); + var reset = checkpointer.Reset(CancellationToken.None); + + checkpointer.Update("30", TestTimeUtc + PersistInterval, CancellationToken.None); + blockedWrite.SetResult(); + await reset; + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(["20", NoCheckpoint, "30"], store.WriteAttempts); + Assert.Equal(["20", NoCheckpoint, "30"], store.CompletedWrites); + Assert.Equal("30", store.PersistedCheckpoint); + } + + [Fact] + public async Task Reset_WhenUpdateArrivesDuringReset_PersistsBeforeReturning() + { + var (checkpointer, store) = await CreateLoadedSubject("10"); + var blockedReset = store.BlockNextWrite(); + var reset = checkpointer.Reset(CancellationToken.None); + await store.WaitForWriteAttempts(1); + + checkpointer.Update("20", TestTimeUtc, CancellationToken.None); + blockedReset.SetResult(); + await reset; + + Assert.Equal([NoCheckpoint, "20"], store.WriteAttempts); + Assert.Equal([NoCheckpoint, "20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + [Fact] public async Task Update_PersistsCheckpoint() { diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs index dea5034d7c2..0b1e76ddb9b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamCancellationCompatibilityTests.cs @@ -32,12 +32,13 @@ public async Task CancellationOverloads_FallBackToLegacyImplementations() Assert.True(((LegacyQueueAdapterReceiver)receiver).MessagesDelivered); Assert.True(((LegacyQueueAdapterReceiver)receiver).ShutdownCalled); + var cancellationToken = CancellationToken.None; IStreamQueueCheckpointer checkpointer = new LegacyStreamQueueCheckpointer(); IStreamQueueCheckpointerFactory checkpointerFactory = new LegacyStreamQueueCheckpointerFactory(checkpointer); - Assert.Same(checkpointer, await checkpointerFactory.Create("partition", cancellation.Token)); - Assert.Equal("10", await checkpointer.Load(cancellation.Token)); + Assert.Same(checkpointer, await checkpointerFactory.Create("partition", cancellationToken)); + Assert.Equal("10", await checkpointer.Load(cancellationToken)); - checkpointer.Update("20", DateTime.UtcNow, cancellation.Token); + checkpointer.Update("20", DateTime.UtcNow, cancellationToken); Assert.Equal("20", ((LegacyStreamQueueCheckpointer)checkpointer).Checkpoint); } @@ -96,6 +97,28 @@ await Assert.ThrowsAnyAsync( () => receiver.Shutdown(TimeSpan.FromSeconds(1), cancellation.Token)); } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task CheckpointerCompatibilityOverloads_HonorPreCanceledOperations() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + IStreamQueueCheckpointer checkpointer = new LegacyStreamQueueCheckpointer(); + IStreamQueueCheckpointerFactory checkpointerFactory = new LegacyStreamQueueCheckpointerFactory(checkpointer); + + await Assert.ThrowsAnyAsync( + () => checkpointerFactory.Create("partition", cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.Load(cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.Reset(cancellation.Token)); + Assert.ThrowsAny( + () => checkpointer.Update("20", DateTime.UtcNow, cancellation.Token)); + await Assert.ThrowsAnyAsync(() => checkpointer.FlushAsync(cancellation.Token)); + Assert.Equal("10", ((LegacyStreamQueueCheckpointer)checkpointer).Checkpoint); + } + private sealed class LegacyQueueAdapterFactory(IQueueAdapter adapter) : IQueueAdapterFactory { public Task CreateAdapter() => Task.FromResult(adapter); @@ -153,6 +176,5 @@ private sealed class LegacyStreamQueueCheckpointer : IStreamQueueCheckpointer Checkpoint = offset; - public Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; } }