Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
5b36e2b
add Reset() method to IStreamQueueCheckpointer
oising Nov 9, 2021
039c4e8
fix(streaming): harden stale checkpoint recovery
ReubenBond Aug 23, 2026
c115797
fix(streaming): propagate checkpoint cancellation
ReubenBond Aug 23, 2026
03ef0eb
docs(streaming): explain Event Hubs checkpoint recovery
ReubenBond Aug 23, 2026
a08d634
fix(streaming): honor checkpointer semantics
ReubenBond Aug 23, 2026
c498705
fix(streaming): complete cache recovery contracts
ReubenBond Aug 25, 2026
fa6a4c7
fix(streaming): honor canceled checkpointer operations
ReubenBond Aug 26, 2026
02d8803
fix(streaming): honor all checkpointer cancellations
ReubenBond Aug 26, 2026
1990c7a
fix(streaming): preserve current stream contracts
ReubenBond Sep 3, 2026
249c8b9
fix(azure): propagate storage cancellation
ReubenBond Sep 3, 2026
881531c
fix(streaming): align Event Hubs diagnostics
ReubenBond Sep 4, 2026
938a3a6
fix(streaming): obsolete legacy checkpointer factories
ReubenBond Sep 5, 2026
ff70b14
fix(streaming): isolate Azure checkpoint writes
ReubenBond Sep 5, 2026
933cc94
fix(streaming): adapt checkpoint recovery to typed cursors
ReubenBond Sep 5, 2026
5520a12
fix(streaming): distinguish checkpoint recovery failures
ReubenBond Sep 7, 2026
b98ce8b
fix(streaming): preserve pending cache misses
ReubenBond Sep 7, 2026
065ce00
fix(streaming): reconcile checkpoint resets with main
ReubenBond Sep 10, 2026
3d81a0d
fix(streaming): close checkpoint recovery races
ReubenBond Sep 11, 2026
d7aa4e5
test(streaming): await Event Hubs cache initialization
ReubenBond Sep 11, 2026
7daeeb7
test(streaming): await Event Hubs purge setup
ReubenBond Sep 11, 2026
cf20abb
fix(streaming): serialize Event Hubs recovery
ReubenBond Sep 11, 2026
ca93fc8
fix(streaming): harden checkpoint recovery cleanup
ReubenBond Sep 11, 2026
ddc731d
fix(streaming): reject progress while recovering
ReubenBond Sep 11, 2026
c692bf1
fix(streaming): serialize receiver initialization
ReubenBond Sep 11, 2026
8e17aaf
fix(streaming): retain failed initialization cleanup
ReubenBond Sep 11, 2026
3fdac89
fix(streaming): keep reset recovery in flight
ReubenBond Sep 11, 2026
0163ee9
fix(streaming): preserve recovery after reset failure
ReubenBond Sep 12, 2026
ac25afa
fix(streaming): propagate receiver initialization cancellation
ReubenBond Sep 12, 2026
eeb9c05
fix(streaming): drain Event Hubs receiver lifetimes
ReubenBond Sep 16, 2026
b163790
fix(streaming): serialize receiver lifetime transitions
ReubenBond Sep 18, 2026
455f039
Merge branch 'main' into gh-7256-pulling-agent-checkpoint-outofbounds
oising Oct 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ private async Task LoadCore(CancellationToken cancellationToken)
}

if (!item.TryGetValue(CheckpointAttribute, out var checkpoint)
|| string.IsNullOrEmpty(checkpoint.S)
|| checkpoint.S is null
|| !item.TryGetValue(VersionAttribute, out var version)
|| !long.TryParse(version.N, NumberStyles.None, CultureInfo.InvariantCulture, out _version)
|| _version <= 0)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,74 +4,80 @@
using Orleans.Configuration;
using Orleans.Streams;

namespace Orleans.Streaming.Kinesis
namespace Orleans.Streaming.Kinesis;

/// <summary>
/// Persists stream queue checkpoints in DynamoDB.
/// </summary>
internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer<string>
{
/// <summary>
/// Persists stream queue checkpoints in DynamoDB.
/// </summary>
internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer<string>
private readonly GrainStreamQueueCheckpointer _inner;

internal DynamoDBStreamQueueCheckpointer(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options)
{
private readonly GrainStreamQueueCheckpointer _inner;
_inner = new GrainStreamQueueCheckpointer(
new StreamCheckpointStoreAdapter(store),
new GrainStreamQueueCheckpointerOptions
{
CheckpointComparer = StreamCheckpointComparers.Numeric,
PersistInterval = options.PersistInterval,
});
}

internal DynamoDBStreamQueueCheckpointer(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options)
{
_inner = new GrainStreamQueueCheckpointer(
new StreamCheckpointStoreAdapter(store),
new GrainStreamQueueCheckpointerOptions
{
CheckpointComparer = StreamCheckpointComparers.Numeric,
PersistInterval = options.PersistInterval,
});
}
/// <inheritdoc />
public bool CheckpointExists => _inner.CheckpointExists;

/// <inheritdoc />
public bool CheckpointExists => _inner.CheckpointExists;
internal static async Task<IStreamQueueCheckpointer<string>> Create(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options)
=> await Create(store, options, CancellationToken.None);

internal static async Task<IStreamQueueCheckpointer<string>> Create(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options)
=> await Create(store, options, CancellationToken.None);
internal static async Task<IStreamQueueCheckpointer<string>> Create(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options,
CancellationToken cancellationToken)
{
var result = new DynamoDBStreamQueueCheckpointer(store, options);
_ = await result.Load(cancellationToken);
return result;
}

internal static async Task<IStreamQueueCheckpointer<string>> Create(
IDynamoDBStreamCheckpointStore store,
DynamoDBStreamQueueCheckpointerOptions options,
CancellationToken cancellationToken)
{
var result = new DynamoDBStreamQueueCheckpointer(store, options);
_ = await result.Load(cancellationToken);
return result;
}
/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public Task<string> Load() => Load(CancellationToken.None);

/// <inheritdoc />
public Task<string> Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken);

/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public Task<string> Load() => Load(CancellationToken.None);
/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public Task Reset() => Reset(CancellationToken.None);

/// <inheritdoc />
public Task<string> Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken);
/// <inheritdoc />
public Task Reset(CancellationToken cancellationToken) => _inner.Reset(cancellationToken);
Comment thread
ReubenBond marked this conversation as resolved.
Resolved

/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public void Update(string offset, DateTime utcNow)
=> Update(offset, utcNow, CancellationToken.None);
/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public void Update(string offset, DateTime utcNow)
=> Update(offset, utcNow, CancellationToken.None);

/// <inheritdoc />
public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken)
=> _inner.Update(offset, utcNow, cancellationToken);
/// <inheritdoc />
public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken)
=> _inner.Update(offset, utcNow, cancellationToken);

/// <inheritdoc />
public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken);
/// <inheritdoc />
public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken);

private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain
{
public ValueTask<string> Load(CancellationToken cancellationToken) => store.Load(cancellationToken);
private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain
{
public ValueTask<string> Load(CancellationToken cancellationToken) => store.Load(cancellationToken);

public ValueTask<string> Update(
string checkpoint,
string expectedCheckpoint,
CancellationToken cancellationToken)
=> store.Update(checkpoint, expectedCheckpoint, cancellationToken);
}
public ValueTask<string> Update(
string checkpoint,
string expectedCheckpoint,
CancellationToken cancellationToken)
=> store.Update(checkpoint, expectedCheckpoint, cancellationToken);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser
}

/// <inheritdoc />
[Obsolete("Use the overload which accepts a CancellationToken.")]
public async Task<IStreamQueueCheckpointer<string>> Create(string partition)
=> await Create(partition, CancellationToken.None);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,6 @@ internal enum OrleansEventHubErrorCode

FailedPartitionRead = ServiceBus + 1,
RetryReceiverInit = ServiceBus + 2,
FailedPartitionRecovery = ServiceBus + 3,
}
}
Original file line number Diff line number Diff line change
@@ -1,52 +1,49 @@
using Orleans.Streams;
using System;
using System.Threading;
using System.Threading.Tasks;
using Orleans.Streams;

namespace Orleans.Streaming.EventHubs.Testing;

namespace Orleans.Streaming.EventHubs.Testing
public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory
{
/// <summary>
/// Creates checkpointers which keep no persistent checkpoint state.
/// </summary>
public class NoOpCheckpointerFactory : IStreamQueueCheckpointerFactory
public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory();

[Obsolete("Use the overload which accepts a CancellationToken.")]
public Task<IStreamQueueCheckpointer<string>> Create(string partition)
=> Create(partition, CancellationToken.None);

public Task<IStreamQueueCheckpointer<string>> Create(
string partition,
CancellationToken cancellationToken)
=> cancellationToken.IsCancellationRequested
? Task.FromCanceled<IStreamQueueCheckpointer<string>>(cancellationToken)
: Task.FromResult<IStreamQueueCheckpointer<string>>(NoOpCheckpointer.Instance);
}
/// <summary>
Comment on lines +22 to +23
/// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider,
/// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there.
/// So a checkpointer with no ops should be enough.
/// </summary>
public class NoOpCheckpointer : IStreamQueueCheckpointer<string>
{
public static NoOpCheckpointer Instance = new NoOpCheckpointer();

public bool CheckpointExists => false;
public Task<string> Load()
{
/// <summary>
/// Gets the shared factory instance.
/// </summary>
public static NoOpCheckpointerFactory Instance = new NoOpCheckpointerFactory();

/// <inheritdoc />
public Task<IStreamQueueCheckpointer<string>> Create(string partition)
{
return Task.FromResult<IStreamQueueCheckpointer<string>>(NoOpCheckpointer.Instance);
}
return Task.FromResult(EventHubConstants.StartOfStream);
}
/// <summary>
/// NoOpCheckpointer is used in EventDataGeneratorStreamProvider ecosystem to replace the default Checkpointer which requires a back end storage. In EventHubDataGeneratorStreamProvider,
/// it is generating EventData on the fly when receiver pull messages from the queue, which means it doesn't support recoverable stream, hence check pointing won't bring much value there.
/// So a checkpointer with no ops should be enough.
/// </summary>
public class NoOpCheckpointer : IStreamQueueCheckpointer<string>

[Obsolete("Use the overload which accepts a CancellationToken.")]
public Task Reset() => Task.CompletedTask;

public Task Reset(CancellationToken cancellationToken)
=> cancellationToken.IsCancellationRequested
? Task.FromCanceled(cancellationToken)
: Task.CompletedTask;

public void Update(string offset, DateTime utcNow)
{
/// <summary>
/// Gets the shared checkpointer instance.
/// </summary>
public static NoOpCheckpointer Instance = new NoOpCheckpointer();

/// <inheritdoc />
public bool CheckpointExists => true;

/// <inheritdoc />
public Task<string> Load()
{
return Task.FromResult(EventHubConstants.StartOfStream);
}
/// <summary>
/// Ignores a checkpoint update because generated-event streams do not persist checkpoints.
/// </summary>
/// <param name="offset">The ignored offset.</param>
/// <param name="utcNow">The ignored update time.</param>
public void Update(string offset, DateTime utcNow)
{
}
}
}
Loading
Loading