diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs
index 0f023d86359..d878aad8e27 100644
--- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs
+++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs
@@ -256,7 +256,7 @@ private async Task LoadCore(CancellationToken cancellationToken)
}
if (!item.TryGetValue(CheckpointAttribute, out var checkpoint)
- || string.IsNullOrEmpty(checkpoint.S)
+ || checkpoint.S is null
|| !item.TryGetValue(VersionAttribute, out var version)
|| !long.TryParse(version.N, NumberStyles.None, CultureInfo.InvariantCulture, out _version)
|| _version <= 0)
diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs
index 56ba751fb24..4297dd0a647 100644
--- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs
+++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs
@@ -4,74 +4,80 @@
using Orleans.Configuration;
using Orleans.Streams;
-namespace Orleans.Streaming.Kinesis
+namespace Orleans.Streaming.Kinesis;
+
+///
+/// Persists stream queue checkpoints in DynamoDB.
+///
+internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer
{
- ///
- /// Persists stream queue checkpoints in DynamoDB.
- ///
- internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer
+ private readonly GrainStreamQueueCheckpointer _inner;
+
+ internal DynamoDBStreamQueueCheckpointer(
+ IDynamoDBStreamCheckpointStore store,
+ DynamoDBStreamQueueCheckpointerOptions options)
{
- private readonly GrainStreamQueueCheckpointer _inner;
+ _inner = new GrainStreamQueueCheckpointer(
+ new StreamCheckpointStoreAdapter(store),
+ new GrainStreamQueueCheckpointerOptions
+ {
+ CheckpointComparer = StreamCheckpointComparers.Numeric,
+ PersistInterval = options.PersistInterval,
+ });
+ }
- internal DynamoDBStreamQueueCheckpointer(
- IDynamoDBStreamCheckpointStore store,
- DynamoDBStreamQueueCheckpointerOptions options)
- {
- _inner = new GrainStreamQueueCheckpointer(
- new StreamCheckpointStoreAdapter(store),
- new GrainStreamQueueCheckpointerOptions
- {
- CheckpointComparer = StreamCheckpointComparers.Numeric,
- PersistInterval = options.PersistInterval,
- });
- }
+ ///
+ public bool CheckpointExists => _inner.CheckpointExists;
- ///
- public bool CheckpointExists => _inner.CheckpointExists;
+ internal static async Task> Create(
+ IDynamoDBStreamCheckpointStore store,
+ DynamoDBStreamQueueCheckpointerOptions options)
+ => await Create(store, options, CancellationToken.None);
- internal static async Task> Create(
- IDynamoDBStreamCheckpointStore store,
- DynamoDBStreamQueueCheckpointerOptions options)
- => await Create(store, options, CancellationToken.None);
+ internal static async Task> Create(
+ IDynamoDBStreamCheckpointStore store,
+ DynamoDBStreamQueueCheckpointerOptions options,
+ CancellationToken cancellationToken)
+ {
+ var result = new DynamoDBStreamQueueCheckpointer(store, options);
+ _ = await result.Load(cancellationToken);
+ return result;
+ }
- internal static async Task> Create(
- IDynamoDBStreamCheckpointStore store,
- DynamoDBStreamQueueCheckpointerOptions options,
- CancellationToken cancellationToken)
- {
- var result = new DynamoDBStreamQueueCheckpointer(store, options);
- _ = await result.Load(cancellationToken);
- return result;
- }
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task Load() => Load(CancellationToken.None);
+
+ ///
+ public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken);
- ///
- [Obsolete("Use the overload which accepts a CancellationToken.")]
- public Task Load() => Load(CancellationToken.None);
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task Reset() => Reset(CancellationToken.None);
- ///
- public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken);
+ ///
+ public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken);
- ///
- [Obsolete("Use the overload which accepts a CancellationToken.")]
- public void Update(string offset, DateTime utcNow)
- => Update(offset, utcNow, CancellationToken.None);
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public void Update(string offset, DateTime utcNow)
+ => Update(offset, utcNow, CancellationToken.None);
- ///
- public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken)
- => _inner.Update(offset, utcNow, cancellationToken);
+ ///
+ public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken)
+ => _inner.Update(offset, utcNow, cancellationToken);
- ///
- public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken);
+ ///
+ public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken);
- private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain
- {
- public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken);
+ private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain
+ {
+ public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken);
- public ValueTask Update(
- string checkpoint,
- string expectedCheckpoint,
- CancellationToken cancellationToken)
- => store.Update(checkpoint, expectedCheckpoint, cancellationToken);
- }
+ public ValueTask Update(
+ string checkpoint,
+ string expectedCheckpoint,
+ CancellationToken cancellationToken)
+ => store.Update(checkpoint, expectedCheckpoint, cancellationToken);
}
}
diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs
index ff76c59de58..04dc841f450 100644
--- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs
+++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointerFactory.cs
@@ -77,6 +77,7 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser
}
///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
public async Task> Create(string partition)
=> await Create(partition, CancellationToken.None);
diff --git a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs
index 63a904a0cbc..e62c2461618 100644
--- a/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs
+++ b/src/Azure/Orleans.Streaming.EventHubs/OrleansServiceBusErrorCode.cs
@@ -12,5 +12,6 @@ internal enum OrleansEventHubErrorCode
FailedPartitionRead = ServiceBus + 1,
RetryReceiverInit = ServiceBus + 2,
+ FailedPartitionRecovery = ServiceBus + 3,
}
}
diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs
index be76b2e5df1..183645aa2a2 100644
--- a/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs
+++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/EventDataGeneratorStreamProvider/NoOpCheckpointer.cs
@@ -1,52 +1,49 @@
-using Orleans.Streams;
using System;
+using System.Threading;
using System.Threading.Tasks;
+using Orleans.Streams;
+
+namespace Orleans.Streaming.EventHubs.Testing;
-namespace Orleans.Streaming.EventHubs.Testing
+public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory
{
- ///
- /// Creates checkpointers which keep no persistent checkpoint state.
- ///
- public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory
+ public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory();
+
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task> Create(string partition)
+ => Create(partition, CancellationToken.None);
+
+ public Task> Create(
+ string partition,
+ CancellationToken cancellationToken)
+ => cancellationToken.IsCancellationRequested
+ ? Task.FromCanceled>(cancellationToken)
+ : Task.FromResult>(NoOpCheckpointer.Instance);
+}
+///
+/// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider,
+/// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there.
+/// So a checkpointer with no ops should be enough.
+///
+public class NoOpCheckpointer : IStreamQueueCheckpointer
+{
+ public static NoOpCheckpointer Instance = new NoOpCheckpointer();
+
+ public bool CheckpointExists => false;
+ public Task Load()
{
- ///
- /// Gets the shared factory instance.
- ///
- public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory();
-
- ///
- public Task> Create(string partition)
- {
- return Task.FromResult>(NoOpCheckpointer.Instance);
- }
+ return Task.FromResult(EventHubConstants.StartOfStream);
}
- ///
- /// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider,
- /// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there.
- /// So a checkpointer with no ops should be enough.
- ///
- public class NoOpCheckpointer : IStreamQueueCheckpointer
+
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task Reset() => Task.CompletedTask;
+
+ public Task Reset(CancellationToken cancellationToken)
+ => cancellationToken.IsCancellationRequested
+ ? Task.FromCanceled(cancellationToken)
+ : Task.CompletedTask;
+
+ public void Update(string offset, DateTime utcNow)
{
- ///
- /// Gets the shared checkpointer instance.
- ///
- public static NoOpCheckpointer Instance = new NoOpCheckpointer();
-
- ///
- public bool CheckpointExists => true;
-
- ///
- public Task Load()
- {
- return Task.FromResult(EventHubConstants.StartOfStream);
- }
- ///
- /// Ignores a checkpoint update because generated-event streams do not persist checkpoints.
- ///
- /// The ignored offset.
- /// The ignored update time.
- public void Update(string offset, DateTime utcNow)
- {
- }
}
}
diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs
index 7c69c12b365..12407b99099 100644
--- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs
+++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs
@@ -6,220 +6,497 @@
using Orleans.Configuration;
using Orleans.Streaming.EventHubs;
-namespace Orleans.Streams
+namespace Orleans.Streams;
+
+///
+/// Persists stream queue checkpoints using Azure Table Storage.
+///
+public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer
{
+ private readonly AzureTableDataManager _dataManager;
+ private readonly TimeSpan _persistInterval;
+ private readonly IComparer? _checkpointComparer;
+ private readonly object _lock = new();
+
+ private StreamQueueCheckpointEntity _entity;
+ private Task _inProgressSave = Task.CompletedTask;
+ private DateTime? _throttleSavesUntilUtc;
+ private string _latestCheckpoint = string.Empty;
+ private string _persistedCheckpoint = string.Empty;
+ private int _pendingResetCount;
+ private long _updateGeneration;
+
+ private AzureTableStreamQueueCheckpointer(
+ AzureTableStreamCheckpointerOptions options,
+ string streamProviderName,
+ string partition,
+ string serviceId,
+ ILoggerFactory loggerFactory,
+ IComparer? defaultComparer = null,
+ string? partitionKeyPrefix = null)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName);
+ ArgumentException.ThrowIfNullOrWhiteSpace(partition);
+ ArgumentNullException.ThrowIfNull(loggerFactory);
+ if (options.PersistInterval <= TimeSpan.Zero)
+ {
+ throw new ArgumentOutOfRangeException(
+ nameof(options),
+ options.PersistInterval,
+ $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero.");
+ }
+
+ _persistInterval = options.PersistInterval;
+ _checkpointComparer = options.CheckpointComparer ?? defaultComparer;
+ _dataManager = new AzureTableDataManager(
+ options,
+ loggerFactory.CreateLogger());
+ _entity = StreamQueueCheckpointEntity.Create(
+ partitionKeyPrefix ?? options.PartitionKeyPrefix,
+ streamProviderName,
+ serviceId,
+ partition);
+ LogCreatingCheckpointer(
+ loggerFactory.CreateLogger(),
+ partition,
+ streamProviderName,
+ serviceId);
+ }
+
+ ///
+ public bool CheckpointExists
+ {
+ get
+ {
+ lock (_lock)
+ {
+ return !string.IsNullOrEmpty(_latestCheckpoint);
+ }
+ }
+ }
+
///
- /// Persists stream queue checkpoints using Azure Table Storage.
+ /// Creates and initializes an Azure Table stream queue checkpointer.
///
- public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public static Task> Create(
+ AzureTableStreamCheckpointerOptions options,
+ string streamProviderName,
+ string partition,
+ string serviceId,
+ ILoggerFactory loggerFactory)
+ => Create(
+ options,
+ streamProviderName,
+ partition,
+ serviceId,
+ loggerFactory,
+ CancellationToken.None);
+
+ ///
+ /// Creates and initializes an Azure Table stream queue checkpointer.
+ ///
+ public static Task> Create(
+ AzureTableStreamCheckpointerOptions options,
+ string streamProviderName,
+ string partition,
+ string serviceId,
+ ILoggerFactory loggerFactory,
+ CancellationToken cancellationToken)
+ {
+ return Create(
+ options,
+ streamProviderName,
+ partition,
+ serviceId,
+ loggerFactory,
+ defaultComparer: null,
+ cancellationToken: cancellationToken);
+ }
+
+ internal static async Task> Create(
+ AzureTableStreamCheckpointerOptions options,
+ string streamProviderName,
+ string partition,
+ string serviceId,
+ ILoggerFactory loggerFactory,
+ IComparer? defaultComparer,
+ string? partitionKeyPrefix = null,
+ CancellationToken cancellationToken = default)
+ {
+ var checkpointer = new AzureTableStreamQueueCheckpointer(
+ options,
+ streamProviderName,
+ partition,
+ serviceId,
+ loggerFactory,
+ defaultComparer,
+ partitionKeyPrefix);
+ await checkpointer._dataManager.InitTableAsync(cancellationToken);
+ return checkpointer;
+ }
+
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task Load() => Load(CancellationToken.None);
+
+ ///
+ public async Task Load(CancellationToken cancellationToken)
{
- private readonly AzureTableDataManager _dataManager;
- private readonly TimeSpan _persistInterval;
- private readonly IComparer? _checkpointComparer;
- private readonly object _lock = new();
-
- private StreamQueueCheckpointEntity _entity;
- private Task _inProgressSave = Task.CompletedTask;
- private DateTime? _throttleSavesUntilUtc;
- private string _latestCheckpoint = string.Empty;
- private string _persistedCheckpoint = string.Empty;
-
- private AzureTableStreamQueueCheckpointer(
- AzureTableStreamCheckpointerOptions options,
- string streamProviderName,
- string partition,
- string serviceId,
- ILoggerFactory loggerFactory,
- IComparer? defaultComparer = null,
- string? partitionKeyPrefix = null)
- {
- ArgumentNullException.ThrowIfNull(options);
- ArgumentException.ThrowIfNullOrWhiteSpace(streamProviderName);
- ArgumentException.ThrowIfNullOrWhiteSpace(partition);
- ArgumentNullException.ThrowIfNull(loggerFactory);
- if (options.PersistInterval <= TimeSpan.Zero)
- {
- throw new ArgumentOutOfRangeException(
- nameof(options),
- options.PersistInterval,
- $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero.");
- }
-
- _persistInterval = options.PersistInterval;
- _checkpointComparer = options.CheckpointComparer ?? defaultComparer;
- _dataManager = new AzureTableDataManager(
- options,
- loggerFactory.CreateLogger());
- _entity = StreamQueueCheckpointEntity.Create(
- partitionKeyPrefix ?? options.PartitionKeyPrefix,
- streamProviderName,
- serviceId,
- partition);
- LogCreatingCheckpointer(
- loggerFactory.CreateLogger(),
- partition,
- streamProviderName,
- serviceId);
- }
-
- ///
- public bool CheckpointExists
- {
- get
+ var result = await _dataManager.ReadSingleTableEntryAsync(
+ _entity.PartitionKey,
+ _entity.RowKey,
+ cancellationToken);
+ var checkpoint = result.Entity?.Offset ?? string.Empty;
+ lock (_lock)
+ {
+ if (result.Entity is not null)
{
+ _entity = result.Entity;
+ }
+
+ _latestCheckpoint = checkpoint;
+ _persistedCheckpoint = checkpoint;
+ }
+
+ return checkpoint;
+ }
+
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task Reset() => Reset(CancellationToken.None);
+
+ ///
+ public Task Reset(CancellationToken cancellationToken)
+ {
+ var completion = new TaskCompletionSource(
+ TaskCreationOptions.RunContinuationsAsynchronously);
+ lock (_lock)
+ {
+ _pendingResetCount++;
+ _throttleSavesUntilUtc = DateTime.MaxValue;
+ var inProgressSave = _inProgressSave;
+ _inProgressSave = completion.Task;
+ RunReset(
+ completion,
+ inProgressSave,
+ _inProgressSave,
+ _latestCheckpoint,
+ _updateGeneration,
+ cancellationToken).Ignore();
+ completion.Task.Ignore();
+ }
+
+ return completion.Task.WaitAsync(cancellationToken);
+ }
+
+ private async Task RunReset(
+ TaskCompletionSource completion,
+ Task inProgressSave,
+ Task resetTask,
+ string enqueuedCheckpoint,
+ long updateGeneration,
+ CancellationToken cancellationToken)
+ {
+ try
+ {
+ await ResetCore(
+ inProgressSave,
+ enqueuedCheckpoint,
+ updateGeneration,
+ cancellationToken);
+
+ while (true)
+ {
+ string checkpoint;
lock (_lock)
{
- return !string.IsNullOrEmpty(_latestCheckpoint);
+ if (!ReferenceEquals(resetTask, _inProgressSave)
+ || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal))
+ {
+ CompleteReset();
+ completion.TrySetResult();
+ return;
+ }
+
+ checkpoint = _latestCheckpoint;
}
+
+ await Save(checkpoint, cancellationToken);
}
}
-
- ///
- /// Creates and initializes an Azure Table stream queue checkpointer.
- ///
- public static Task> Create(
- AzureTableStreamCheckpointerOptions options,
- string streamProviderName,
- string partition,
- string serviceId,
- ILoggerFactory loggerFactory)
+ catch (OperationCanceledException exception)
{
- return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null);
+ await CompleteFailedReset(
+ completion,
+ resetTask,
+ exception,
+ canceled: true);
}
-
- internal static async Task> Create(
- AzureTableStreamCheckpointerOptions options,
- string streamProviderName,
- string partition,
- string serviceId,
- ILoggerFactory loggerFactory,
- IComparer? defaultComparer,
- string? partitionKeyPrefix = null)
+ catch (Exception exception)
{
- var checkpointer = new AzureTableStreamQueueCheckpointer(
- options,
- streamProviderName,
- partition,
- serviceId,
- loggerFactory,
- defaultComparer,
- partitionKeyPrefix);
- await checkpointer._dataManager.InitTableAsync();
- return checkpointer;
+ await CompleteFailedReset(
+ completion,
+ resetTask,
+ exception,
+ canceled: false);
}
+ }
- ///
- public async Task Load()
+ private async Task CompleteFailedReset(
+ TaskCompletionSource completion,
+ Task resetTask,
+ Exception resetFailure,
+ bool canceled)
+ {
+ try
{
- var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey);
- var checkpoint = result.Entity?.Offset ?? string.Empty;
- lock (_lock)
+ while (true)
{
- if (result.Entity is not null)
+ string checkpoint;
+ lock (_lock)
{
- _entity = result.Entity;
+ if (!ReferenceEquals(resetTask, _inProgressSave)
+ || string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal))
+ {
+ CompleteReset();
+ if (canceled)
+ {
+ completion.TrySetCanceled(
+ ((OperationCanceledException)resetFailure).CancellationToken);
+ }
+ else
+ {
+ completion.TrySetException(resetFailure);
+ }
+
+ return;
+ }
+
+ checkpoint = _latestCheckpoint;
}
- _latestCheckpoint = checkpoint;
- _persistedCheckpoint = checkpoint;
+ await Save(checkpoint, CancellationToken.None);
}
-
- return checkpoint;
}
-
- ///
- public void Update(string offset, DateTime utcNow)
+ catch (Exception persistenceFailure)
{
- ArgumentNullException.ThrowIfNull(offset);
-
lock (_lock)
{
- if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal)
- || (_checkpointComparer is { } comparer
- && !string.IsNullOrEmpty(_latestCheckpoint)
- && comparer.Compare(offset, _latestCheckpoint) <= 0))
- {
- return;
- }
+ CompleteReset();
+ completion.TrySetException(
+ new AggregateException(resetFailure, persistenceFailure));
+ }
+ }
+ }
+
+ private void CompleteReset()
+ {
+ _pendingResetCount--;
+ if (_pendingResetCount == 0)
+ {
+ _throttleSavesUntilUtc = null;
+ }
+ }
+
+ ///
+ public void Update(string offset, DateTime utcNow)
+ {
+ ArgumentNullException.ThrowIfNull(offset);
- _latestCheckpoint = offset;
- if (_throttleSavesUntilUtc.HasValue
- && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted))
+ lock (_lock)
+ {
+ if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal))
+ {
+ if (string.Equals(_persistedCheckpoint, offset, StringComparison.Ordinal)
+ || !_inProgressSave.IsCompleted
+ || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow))
{
return;
}
_throttleSavesUntilUtc = utcNow + _persistInterval;
- _inProgressSave = Save(offset);
+ _inProgressSave = Save(offset, CancellationToken.None);
_inProgressSave.Ignore();
+ return;
}
+
+ if (_checkpointComparer is { } comparer
+ && !string.IsNullOrEmpty(_latestCheckpoint)
+ && comparer.Compare(offset, _latestCheckpoint) <= 0)
+ {
+ return;
+ }
+
+ _latestCheckpoint = offset;
+ _updateGeneration++;
+ _entity.Offset = offset;
+ if (!_inProgressSave.IsCompleted
+ || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow))
+ {
+ return;
+ }
+
+ _throttleSavesUntilUtc = utcNow + _persistInterval;
+ _inProgressSave = Save(offset, CancellationToken.None);
+ _inProgressSave.Ignore();
}
+ }
- ///
- public async Task FlushAsync(CancellationToken cancellationToken)
+ ///
+ public async Task FlushAsync(CancellationToken cancellationToken)
+ {
+ var retryingSave = false;
+ while (true)
{
- var retryingSave = false;
- while (true)
+ Task inProgressSave;
+ lock (_lock)
{
- Task inProgressSave;
- lock (_lock)
- {
- inProgressSave = _inProgressSave;
- }
+ inProgressSave = _inProgressSave;
+ }
- if (retryingSave)
+ if (retryingSave)
+ {
+ await inProgressSave.WaitAsync(cancellationToken);
+ }
+ else
+ {
+ try
{
await inProgressSave.WaitAsync(cancellationToken);
}
- else
+ catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
- try
- {
- await inProgressSave.WaitAsync(cancellationToken);
- }
- catch (Exception) when (!cancellationToken.IsCancellationRequested)
- {
- }
-
- cancellationToken.ThrowIfCancellationRequested();
}
- lock (_lock)
- {
- if (!ReferenceEquals(inProgressSave, _inProgressSave))
- {
- retryingSave = false;
- continue;
- }
+ cancellationToken.ThrowIfCancellationRequested();
+ }
- if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal))
- {
- return;
- }
+ lock (_lock)
+ {
+ if (!ReferenceEquals(inProgressSave, _inProgressSave))
+ {
+ retryingSave = false;
+ continue;
+ }
- _inProgressSave = Save(_latestCheckpoint);
- retryingSave = true;
+ if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal))
+ {
+ return;
}
+
+ _inProgressSave = Save(_latestCheckpoint, cancellationToken);
+ _inProgressSave.Ignore();
+ retryingSave = true;
}
}
+ }
- private async Task Save(string checkpoint)
+ private async Task Save(string checkpoint, CancellationToken cancellationToken)
+ {
+ var entity = CreateWriteEntity(checkpoint);
+ await _dataManager.UpsertTableEntryAsync(entity, cancellationToken);
+ lock (_lock)
{
_entity.Offset = checkpoint;
- await _dataManager.UpsertTableEntryAsync(_entity);
+ _persistedCheckpoint = checkpoint;
+ }
+ }
+
+ private async Task ResetCore(
+ Task inProgressSave,
+ string enqueuedCheckpoint,
+ long updateGeneration,
+ CancellationToken cancellationToken)
+ {
+ try
+ {
+ await inProgressSave;
+ }
+ catch
+ {
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ string rollbackCheckpoint;
+ lock (_lock)
+ {
+ if (_updateGeneration <= updateGeneration)
+ {
+ rollbackCheckpoint = _latestCheckpoint;
+ _latestCheckpoint = string.Empty;
+ }
+ else
+ {
+ rollbackCheckpoint = enqueuedCheckpoint;
+ }
+ }
+
+ try
+ {
+ var entity = CreateWriteEntity(string.Empty);
+ await _dataManager.UpsertTableEntryAsync(entity, cancellationToken);
lock (_lock)
{
- _persistedCheckpoint = checkpoint;
+ _persistedCheckpoint = string.Empty;
+ if (string.IsNullOrEmpty(_latestCheckpoint))
+ {
+ _entity.Offset = string.Empty;
+ }
}
}
+ catch
+ {
+ lock (_lock)
+ {
+ RestoreLatestCheckpoint(rollbackCheckpoint);
+ }
+
+ throw;
+ }
+ }
+
+ private void RestoreLatestCheckpoint(string latestCheckpoint)
+ {
+ if (_checkpointComparer is { } comparer
+ && !string.IsNullOrEmpty(_latestCheckpoint)
+ && !string.IsNullOrEmpty(latestCheckpoint))
+ {
+ if (comparer.Compare(_latestCheckpoint, latestCheckpoint) < 0)
+ {
+ _latestCheckpoint = latestCheckpoint;
+ }
+
+ return;
+ }
+
+ if (string.IsNullOrEmpty(_latestCheckpoint))
+ {
+ _latestCheckpoint = latestCheckpoint;
+ }
+ }
- [LoggerMessage(
- Level = LogLevel.Information,
- Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")]
- private static partial void LogCreatingCheckpointer(
- ILogger logger,
- string partition,
- string streamProviderName,
- string serviceId);
+ private StreamQueueCheckpointEntity CreateWriteEntity(string checkpoint)
+ {
+ lock (_lock)
+ {
+ return new StreamQueueCheckpointEntity
+ {
+ PartitionKey = _entity.PartitionKey,
+ RowKey = _entity.RowKey,
+ Offset = checkpoint,
+ };
+ }
}
+
+ [LoggerMessage(
+ Level = LogLevel.Information,
+ Message = "Creating Azure Table checkpointer for partition {Partition} of stream provider {StreamProviderName} with service ID {ServiceId}.")]
+ private static partial void LogCreatingCheckpointer(
+ ILogger logger,
+ string partition,
+ string streamProviderName,
+ string serviceId);
}
diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs
index 6aae03ee714..69f24caf8fe 100644
--- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs
+++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs
@@ -1,4 +1,5 @@
using System;
+using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@@ -6,56 +7,63 @@
using Orleans.Configuration;
using Orleans.Configuration.Overrides;
-namespace Orleans.Streams
+namespace Orleans.Streams;
+
+///
+/// Creates Azure Table stream queue checkpointers.
+///
+public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory
{
+ private readonly string _providerName;
+ private readonly AzureTableStreamCheckpointerOptions _options;
+ private readonly ClusterOptions _clusterOptions;
+ private readonly ILoggerFactory _loggerFactory;
+
///
- /// Creates Azure Table stream queue checkpointers.
+ /// Initializes a new instance.
///
- public class AzureTableStreamQueueCheckpointerFactory : IStreamQueueCheckpointerFactory
+ public AzureTableStreamQueueCheckpointerFactory(
+ string providerName,
+ AzureTableStreamCheckpointerOptions options,
+ IOptions clusterOptions,
+ ILoggerFactory loggerFactory)
{
- private readonly string _providerName;
- private readonly AzureTableStreamCheckpointerOptions _options;
- private readonly ClusterOptions _clusterOptions;
- private readonly ILoggerFactory _loggerFactory;
+ _providerName = providerName;
+ _options = options;
+ _clusterOptions = clusterOptions.Value;
+ _loggerFactory = loggerFactory;
+ }
- ///
- /// Initializes a new instance.
- ///
- public AzureTableStreamQueueCheckpointerFactory(
- string providerName,
- AzureTableStreamCheckpointerOptions options,
- IOptions clusterOptions,
- ILoggerFactory loggerFactory)
- {
- _providerName = providerName;
- _options = options;
- _clusterOptions = clusterOptions.Value;
- _loggerFactory = loggerFactory;
- }
+ ///
+ /// Creates a factory from a service provider.
+ ///
+ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName)
+ {
+ var options = services.GetOptionsByName(providerName);
+ var clusterOptions = services.GetProviderClusterOptions(providerName);
+ return ActivatorUtilities.CreateInstance(
+ services,
+ providerName,
+ options,
+ clusterOptions);
+ }
- ///
- /// Creates a factory from a service provider.
- ///
- public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName)
- {
- var options = services.GetOptionsByName(providerName);
- var clusterOptions = services.GetProviderClusterOptions(providerName);
- return ActivatorUtilities.CreateInstance(
- services,
- providerName,
- options,
- clusterOptions);
- }
+ ///
+ [Obsolete("Use the overload which accepts a CancellationToken.")]
+ public Task> Create(string partition)
+ => Create(partition, CancellationToken.None);
- ///
- public Task> Create(string partition)
- {
- return AzureTableStreamQueueCheckpointer.Create(
- _options,
- _providerName,
- partition,
- _clusterOptions.ServiceId.ToString(),
- _loggerFactory);
- }
+ ///
+ public Task> Create(
+ string partition,
+ CancellationToken cancellationToken)
+ {
+ return AzureTableStreamQueueCheckpointer.Create(
+ _options,
+ _providerName,
+ partition,
+ _clusterOptions.ServiceId.ToString(),
+ _loggerFactory,
+ cancellationToken);
}
}
diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs
index 3489a9a8d12..8642c99857e 100644
--- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs
+++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterReceiver.cs
@@ -52,6 +52,7 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa
private readonly IQueueAdapterReceiverMonitor monitor;
private readonly LoadSheddingOptions loadSheddingOptions;
private readonly IEnvironmentStatisticsProvider environmentStatisticsProvider;
+ private readonly object cacheLock = new();
private IEventHubQueueCache? cache;
private IEventHubReceiver? receiver;
@@ -59,17 +60,32 @@ internal partial class EventHubAdapterReceiver : IQueueAdapterReceiver, IQueueCa
private readonly Func eventHubReceiverFactory;
private IStreamQueueCheckpointer? checkpointer;
- private AggregatedQueueFlowController flowController = null!;
+ private AggregatedQueueFlowController flowController = new(MaxMessagesPerRead);
+ private bool receiverUsesCheckpoint;
+ private IEventHubQueueCache? recoveryCache;
+ private Dictionary? recoveredCursorProgress;
+ private readonly HashSet cursors = new(ReferenceEqualityComparer.Instance);
+ private HashSet? recoveryPendingCursors;
+ private Task? recoveryTask;
+ private readonly SemaphoreSlim initializationLock = new(1, 1);
+ private readonly SemaphoreSlim receiverCloseLock = new(1, 1);
+ private readonly SemaphoreSlim shutdownLock = new(1, 1);
+ private IEventHubReceiver? receiverPendingClose;
+ private Task? receiverCloseTask;
// Receiver life cycle
private int receiverState = ReceiverShutdown;
private const int ReceiverShutdown = 0;
private const int ReceiverRunning = 1;
+ private const int ReceiverShutdownCompleted = 2;
public int GetMaxAddCount()
{
- return this.flowController.GetMaxAddCount();
+ lock (this.cacheLock)
+ {
+ return this.flowController.GetMaxAddCount();
+ }
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
@@ -112,18 +128,39 @@ public EventHubAdapterReceiver(EventHubPartitionSettings settings,
this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
}
- public async Task Initialize(TimeSpan timeout)
+ public Task Initialize(TimeSpan timeout)
+ => Initialize(timeout, CancellationToken.None);
+
+ public async Task Initialize(
+ TimeSpan timeout,
+ CancellationToken cancellationToken)
{
+ cancellationToken.ThrowIfCancellationRequested();
+ using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeoutCancellation.CancelAfter(timeout);
LogInfoInitializingEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition);
+ try
+ {
+ await this.shutdownLock.WaitAsync(timeoutCancellation.Token);
+ try
+ {
+ lock (this.cacheLock)
+ {
+ this.receiverState = ReceiverRunning;
+ }
+ }
+ finally
+ {
+ this.shutdownLock.Release();
+ }
- // if receiver was already running, do nothing
- if (ReceiverRunning == Interlocked.Exchange(ref this.receiverState, ReceiverRunning))
+ await EnsureInitialized(timeoutCancellation.Token);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
- return;
+ cancellationToken.ThrowIfCancellationRequested();
+ throw;
}
-
- using var cancellation = new CancellationTokenSource(timeout);
- await Initialize(cancellation.Token);
}
///
@@ -131,37 +168,170 @@ public async Task Initialize(TimeSpan timeout)
/// it will be retried when messages are requested
///
///
- private async Task Initialize(CancellationToken cancellationToken)
+ private async Task EnsureInitialized(CancellationToken cancellationToken)
+ {
+ while (true)
+ {
+ if (Volatile.Read(ref this.receiverPendingClose) is not null)
+ {
+ await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken);
+ }
+
+ Task? recoveryTask;
+ lock (this.cacheLock)
+ {
+ if (this.receiverState != ReceiverRunning || this.receiver is not null)
+ {
+ return;
+ }
+
+ recoveryTask = this.recoveryTask;
+ }
+
+ if (recoveryTask is not null)
+ {
+ await recoveryTask.WaitAsync(cancellationToken);
+ continue;
+ }
+
+ await this.initializationLock.WaitAsync(cancellationToken);
+ try
+ {
+ bool hasPendingClose;
+ lock (this.cacheLock)
+ {
+ if (this.receiverState != ReceiverRunning || this.receiver is not null)
+ {
+ return;
+ }
+
+ recoveryTask = this.recoveryTask;
+ hasPendingClose = this.receiverPendingClose is not null;
+ }
+
+ if (hasPendingClose)
+ {
+ await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken);
+ continue;
+ }
+
+ if (recoveryTask is null)
+ {
+ await InitializeCore(cancellationToken);
+ return;
+ }
+ }
+ finally
+ {
+ this.initializationLock.Release();
+ }
+
+ await recoveryTask.WaitAsync(cancellationToken);
+ }
+ }
+
+ private async Task InitializeCore(CancellationToken cancellationToken)
{
var watch = Stopwatch.StartNew();
+ IEventHubQueueCache? cache = null;
+ IEventHubReceiver? receiver = null;
+ var installed = false;
+ var failures = new List();
try
{
- this.checkpointer = await this.checkpointerFactory(
+ var checkpointer = await this.checkpointerFactory(
this.settings.Partition,
cancellationToken);
- if (this.cache != null)
- {
- this.cache.Dispose();
- this.cache = null;
- }
- this.cache = this.cacheFactory(this.settings.Partition, this.checkpointer, this.loggerFactory);
- this.flowController = new AggregatedQueueFlowController(MaxMessagesPerRead) { this.cache, LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider) };
- string offset = await this.checkpointer.Load(cancellationToken);
- if (!this.checkpointer.CheckpointExists)
+ string offset = await checkpointer.Load(cancellationToken);
+ var receiverUsesCheckpoint = checkpointer.CheckpointExists;
+ if (!receiverUsesCheckpoint)
{
offset = EventHubConstants.StartOfStream;
}
- this.receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger);
- watch.Stop();
- this.monitor?.TrackInitialization(true, watch.Elapsed, null);
+ cache = this.cacheFactory(this.settings.Partition, checkpointer, this.loggerFactory);
+ var flowController = new AggregatedQueueFlowController(MaxMessagesPerRead)
+ {
+ cache,
+ LoadShedQueueFlowController.CreateAsPercentOfLoadSheddingLimit(this.loadSheddingOptions, environmentStatisticsProvider)
+ };
+ receiver = this.eventHubReceiverFactory(this.settings, offset, this.logger);
+ cancellationToken.ThrowIfCancellationRequested();
+
+ lock (this.cacheLock)
+ {
+ if (this.receiverState == ReceiverRunning)
+ {
+ this.cache?.Dispose();
+ this.checkpointer = checkpointer;
+ this.cache = cache;
+ this.flowController = flowController;
+ this.receiverUsesCheckpoint = receiverUsesCheckpoint;
+ this.receiver = receiver;
+ if (this.recoveredCursorProgress is not null)
+ {
+ this.recoveryCache = cache;
+ }
+
+ installed = true;
+ }
+ }
}
catch (Exception ex)
{
- watch.Stop();
- this.monitor?.TrackInitialization(false, watch.Elapsed, ex);
- throw;
+ failures.Add(ex);
+ }
+
+ if (!installed)
+ {
+ try
+ {
+ cache?.Dispose();
+ }
+ catch (Exception exception)
+ {
+ failures.Add(exception);
+ }
+
+ try
+ {
+ if (receiver is not null)
+ {
+ lock (this.cacheLock)
+ {
+ if (this.receiverPendingClose is not null
+ && !ReferenceEquals(this.receiverPendingClose, receiver))
+ {
+ throw new InvalidOperationException(
+ "A previous Event Hub receiver is still pending cleanup.");
+ }
+
+ this.receiverPendingClose = receiver;
+ this.receiverCloseTask = null;
+ }
+
+ await ClosePendingReceiverWithTimeout();
+ }
+ }
+ catch (Exception exception)
+ {
+ failures.Add(exception);
+ }
}
+
+ watch.Stop();
+ if (failures.Count == 0)
+ {
+ this.monitor?.TrackInitialization(true, watch.Elapsed, null);
+ return;
+ }
+
+ var failure = failures.Count == 1
+ ? failures[0]
+ : new AggregateException(failures);
+ this.monitor?.TrackInitialization(false, watch.Elapsed, failure);
+ ExceptionDispatchInfo.Capture(failure).Throw();
+ throw new UnreachableException();
}
[Obsolete("Use the overload which accepts a CancellationToken.")]
@@ -173,29 +343,48 @@ public async Task> GetQueueMessagesAsync(
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
- if (this.receiverState == ReceiverShutdown || maxCount <= 0)
+ if (this.receiverState != ReceiverRunning || maxCount <= 0)
{
return new List();
}
+ var recoveryTask = Volatile.Read(ref this.recoveryTask);
+ if (recoveryTask is not null)
+ {
+ await recoveryTask.WaitAsync(cancellationToken);
+ }
+
+ if (Volatile.Read(ref this.receiverPendingClose) is not null)
+ {
+ await ClosePendingReceiverWithTimeout().WaitAsync(cancellationToken);
+ }
+
// if receiver initialization failed, retry
if (this.receiver == null)
{
LogWarningRetryingInitializationOfEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition);
- await Initialize(cancellationToken);
+ await EnsureInitialized(cancellationToken);
if (this.receiver == null)
{
// should not get here, should throw instead, but just incase.
return new List();
}
}
+
+ var receiver = Volatile.Read(ref this.receiver);
+ if (Volatile.Read(ref this.receiverState) != ReceiverRunning || receiver is null)
+ {
+ return new List();
+ }
+
+ var cache = Volatile.Read(ref this.cache);
var watch = Stopwatch.StartNew();
List? messages;
try
{
// Receivers built against older Orleans versions can still return null.
- messages = (await this.receiver.ReceiveAsync(
+ messages = (await receiver.ReceiveAsync(
maxCount,
ReceiveTimeout,
cancellationToken))?.ToList();
@@ -208,6 +397,33 @@ public async Task> GetQueueMessagesAsync(
watch.Stop();
this.monitor?.TrackRead(false, watch.Elapsed, ex);
LogWarningFailedToReadFromEventHubPartition(this.settings.Hub.EventHubName, this.settings.Partition, ex);
+
+ bool shouldRecover;
+ lock (this.cacheLock)
+ {
+ shouldRecover = this.receiverState == ReceiverRunning
+ && ReferenceEquals(receiver, this.receiver)
+ && this.receiverUsesCheckpoint;
+ }
+
+ if (shouldRecover && IsInvalidOffsetException(ex))
+ {
+ try
+ {
+ await ResetReceiver(receiver, cancellationToken);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception recoveryException)
+ {
+ LogWarningFailedToRecoverFromInvalidCheckpoint(
+ this.settings.Hub.EventHubName,
+ this.settings.Partition,
+ recoveryException);
+ }
+ }
throw;
}
@@ -226,7 +442,19 @@ public async Task> GetQueueMessagesAsync(
this.monitor?.TrackMessagesReceived(messages.Count, oldestMessageEnqueueTime, newestMessageEnqueueTime);
- List messageStreamPositions = this.cache!.Add(messages, dequeueTimeUtc);
+ List messageStreamPositions;
+ lock (this.cacheLock)
+ {
+ if (!ReferenceEquals(receiver, this.receiver)
+ || !ReferenceEquals(cache, this.cache)
+ || cache is null)
+ {
+ return batches;
+ }
+
+ messageStreamPositions = cache.Add(messages, dequeueTimeUtc);
+ }
+
foreach (var streamPosition in messageStreamPositions)
{
batches.Add(new StreamActivityNotificationBatch(streamPosition));
@@ -234,6 +462,216 @@ public async Task> GetQueueMessagesAsync(
return batches;
}
+ // The SDK translates this service rejection to ArgumentException, preserving the service description.
+ private static bool IsInvalidOffsetException(Exception exception)
+ => exception is ArgumentException
+ && exception.Message.StartsWith("The supplied offset", StringComparison.OrdinalIgnoreCase)
+ && exception.Message.Contains(" is invalid.", StringComparison.OrdinalIgnoreCase);
+
+ private async Task ResetReceiver(
+ IEventHubReceiver failedReceiver,
+ CancellationToken cancellationToken)
+ {
+ Task? existingRecovery;
+ lock (this.cacheLock)
+ {
+ existingRecovery = this.recoveryTask;
+ }
+
+ if (existingRecovery is not null)
+ {
+ await existingRecovery.WaitAsync(cancellationToken);
+ return;
+ }
+
+ await this.initializationLock.WaitAsync(cancellationToken);
+ var releaseInitializationLock = true;
+ try
+ {
+ IStreamQueueCheckpointer checkpointer;
+ IEventHubQueueCache? cache;
+ TaskCompletionSource recoveryCompletion;
+ lock (this.cacheLock)
+ {
+ if (this.receiverState != ReceiverRunning
+ || !ReferenceEquals(failedReceiver, this.receiver))
+ {
+ return;
+ }
+
+ recoveryCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ recoveryCompletion.Task.Ignore();
+ this.recoveryTask = recoveryCompletion.Task;
+ checkpointer = this.checkpointer!;
+ this.recoveryCache = null;
+ this.recoveredCursorProgress = new Dictionary(ReferenceEqualityComparer.Instance);
+ this.recoveryPendingCursors = new HashSet(this.cursors, ReferenceEqualityComparer.Instance);
+ cache = this.cache;
+ this.cache = null;
+ this.flowController = new(MaxMessagesPerRead);
+ this.receiverPendingClose = Interlocked.Exchange(ref this.receiver, null);
+ this.receiverCloseTask = null;
+ }
+
+ RunRecovery(checkpointer, cache, recoveryCompletion).Ignore();
+ releaseInitializationLock = false;
+ cancellationToken.ThrowIfCancellationRequested();
+ await recoveryCompletion.Task.WaitAsync(cancellationToken);
+ }
+ finally
+ {
+ if (releaseInitializationLock)
+ {
+ this.initializationLock.Release();
+ }
+ }
+ }
+
+ private async Task RunRecovery(
+ IStreamQueueCheckpointer checkpointer,
+ IEventHubQueueCache? cache,
+ TaskCompletionSource recoveryCompletion)
+ {
+ var exceptions = new List();
+ try
+ {
+ try
+ {
+ await checkpointer.Reset(CancellationToken.None);
+ lock (this.cacheLock)
+ {
+ this.receiverUsesCheckpoint = false;
+ }
+
+ }
+ catch (Exception exception)
+ {
+ exceptions.Add(exception);
+ }
+
+ try
+ {
+ cache?.Dispose();
+ }
+ catch (Exception exception)
+ {
+ exceptions.Add(exception);
+ }
+
+ try
+ {
+ await ClosePendingReceiverWithTimeout();
+ }
+ catch (Exception exception)
+ {
+ exceptions.Add(exception);
+ }
+
+ bool isRunning;
+ lock (this.cacheLock)
+ {
+ isRunning = this.receiverState == ReceiverRunning;
+ }
+
+ if (exceptions.Count == 0 && isRunning)
+ {
+ try
+ {
+ using var initializationCancellation = new CancellationTokenSource(ReceiveTimeout);
+ await InitializeCore(initializationCancellation.Token);
+ }
+ catch (Exception exception)
+ {
+ exceptions.Add(exception);
+ }
+ }
+ }
+ catch (Exception exception)
+ {
+ exceptions.Add(exception);
+ }
+ finally
+ {
+ if (exceptions.Count == 0)
+ {
+ recoveryCompletion.TrySetResult();
+ }
+ else if (exceptions.Count == 1)
+ {
+ recoveryCompletion.TrySetException(exceptions[0]);
+ }
+ else
+ {
+ recoveryCompletion.TrySetException(new AggregateException(exceptions));
+ }
+
+ lock (this.cacheLock)
+ {
+ if (ReferenceEquals(this.recoveryTask, recoveryCompletion.Task))
+ {
+ this.recoveryTask = null;
+ }
+ }
+
+ this.initializationLock.Release();
+ }
+ }
+
+ private async Task ClosePendingReceiverWithTimeout()
+ {
+ using var cleanupCancellation = new CancellationTokenSource(ReceiveTimeout);
+ await ClosePendingReceiver(cleanupCancellation.Token);
+ }
+
+ private async Task ClosePendingReceiver(CancellationToken cancellationToken)
+ {
+ await this.receiverCloseLock.WaitAsync(cancellationToken);
+ try
+ {
+ IEventHubReceiver? receiver;
+ Task closeTask;
+ lock (this.cacheLock)
+ {
+ receiver = this.receiverPendingClose;
+ if (receiver is null)
+ {
+ return;
+ }
+
+ if (this.receiverCloseTask is { IsCompletedSuccessfully: true })
+ {
+ this.receiverPendingClose = null;
+ this.receiverCloseTask = null;
+ return;
+ }
+
+ var existingCloseTask = this.receiverCloseTask;
+ if (existingCloseTask is null || existingCloseTask.IsCompleted)
+ {
+ existingCloseTask = receiver.CloseAsync(cancellationToken);
+ existingCloseTask.Ignore();
+ this.receiverCloseTask = existingCloseTask;
+ }
+
+ closeTask = existingCloseTask;
+ }
+
+ await closeTask.WaitAsync(cancellationToken);
+ lock (this.cacheLock)
+ {
+ if (ReferenceEquals(this.receiverPendingClose, receiver))
+ {
+ this.receiverPendingClose = null;
+ this.receiverCloseTask = null;
+ }
+ }
+ }
+ finally
+ {
+ this.receiverCloseLock.Release();
+ }
+ }
+
public void AddToCache(IList messages)
{
// do nothing, we add data directly into cache. No need for agent involvement
@@ -243,10 +681,21 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList
{
purgedItems = null;
- //if not under pressure, signal the cache to do a time based purge
- //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache
- if (!this.IsUnderPressure())
- this.cache!.SignalPurge();
+ lock (this.cacheLock)
+ {
+ var cache = this.cache;
+ if (cache is null)
+ {
+ return false;
+ }
+
+ //if not under pressure, signal the cache to do a time based purge
+ //if under pressure, which means consuming speed is less than producing speed, then shouldn't purge, and don't read more message into the cache
+ if (!this.IsUnderPressure())
+ {
+ cache.SignalPurge();
+ }
+ }
return false;
}
@@ -254,27 +703,71 @@ public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList
[Obsolete("Use IQueueCache.TryGetCacheCursor instead.")]
public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token)
{
- return new Cursor(this.cache!, streamId, token);
+ lock (this.cacheLock)
+ {
+ var cache = this.cache
+ ?? throw new InvalidOperationException("The Event Hub receiver is recovering.");
+#pragma warning disable CS0618 // Preserve the exact legacy exception and cursor behavior.
+ var cursor = cache.GetCursor(streamId, token);
+#pragma warning restore CS0618
+ return new Cursor(
+ this,
+ streamId,
+ cache,
+ cursor,
+ ReferenceEquals(cache, this.recoveryCache) ? token : null);
+ }
}
QueueCacheCursorResult IQueueCache.TryGetCacheCursor(
StreamId streamId,
StreamSequenceToken? token)
{
- return WrapCursorResult(this.cache!.TryGetCursor(streamId, token));
+ lock (this.cacheLock)
+ {
+ var cache = this.cache;
+ if (cache is null)
+ {
+ return QueueCacheCursorResult.NotSupported;
+ }
+
+ return WrapCursorResult(
+ streamId,
+ cache,
+ cache.TryGetCursor(streamId, token),
+ ReferenceEquals(cache, this.recoveryCache) ? token : null);
+ }
}
QueueCacheCursorResult IQueueCache.TryGetCacheCursorAtPosition(
StreamId streamId,
StreamSubscriptionStartPosition startPosition)
{
- return WrapCursorResult(this.cache!.TryGetCursorAtPosition(streamId, startPosition));
+ lock (this.cacheLock)
+ {
+ var cache = this.cache;
+ if (cache is null)
+ {
+ return QueueCacheCursorResult.NotSupported;
+ }
+
+ return WrapCursorResult(
+ streamId,
+ cache,
+ cache.TryGetCursorAtPosition(streamId, startPosition),
+ resumeToken: null);
+ }
}
- private QueueCacheCursorResult WrapCursorResult(QueueCacheCursorResult