Skip to content

Commit cfe4cfd

Browse files
ReubenBondCopilot
andcommitted
fix(reminders): preserve fired occurrence tombstones
Track the scheduled occurrence consumed before outside-window eviction so an exact-timestamp refresh cannot redeliver it. Registration tombstones remain eligible for exact-due recovery, and deterministic tests cover both boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent e4efeb9 commit cfe4cfd

3 files changed

Lines changed: 109 additions & 7 deletions

File tree

‎src/Orleans.Reminders/ReminderService/LocalReminderService.cs‎

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,13 @@ private Task ReadAndUpdateReminders()
353353
return task;
354354
}
355355

356+
internal Task TestOnlyRefresh()
357+
{
358+
var refreshTask = new Task<Task>(ReadAndUpdateReminders);
359+
Scheduler.QueueTask(refreshTask);
360+
return refreshTask.Unwrap();
361+
}
362+
356363
private void RemoveOutOfRangeReminders(List<Task> removedReminderTasks)
357364
{
358365
CheckRuntimeContext();
@@ -593,9 +600,15 @@ private void ReconcileTableEntry(ReminderEntry entry, long tableSequence, DateTi
593600
return;
594601
}
595602

596-
// A due-now tombstone represents an occurrence which this owner already fired. Do not resurrect it
597-
// from a refresh captured at the same timestamp, but allow due-now entries with no local history.
598-
if (!isWithinLoadingWindow || (state is LocalReminderState.Tombstone && nextTick <= now))
603+
// Retain the marker for an unchanged occurrence which this owner fired at the same timestamp.
604+
if (state is LocalReminderState.Tombstone
605+
&& localReminder.ShouldRetainFiredOccurrenceTombstone(entry, nextTick))
606+
{
607+
localReminder.LocalSequenceNumber = tableSequence;
608+
return;
609+
}
610+
611+
if (!isWithinLoadingWindow)
599612
{
600613
LogTraceRemovingReminder(localReminder);
601614
stopTasks.Add(
@@ -875,6 +888,13 @@ internal static bool IsReminderWithinLoadingWindow(ReminderEntry entry, DateTime
875888
return CalculateNextTickTime(entry, now) <= now.AddClamped(loadingWindow);
876889
}
877890

891+
internal static bool ShouldRetainFiredOccurrenceTombstone(
892+
ReminderEntry localEntry,
893+
ReminderEntry tableEntry,
894+
DateTime? lastFiredTickTime,
895+
DateTime nextTick)
896+
=> lastFiredTickTime == nextTick && StringComparer.Ordinal.Equals(localEntry.ETag, tableEntry.ETag);
897+
878898
internal static DateTime CalculateFollowingTickTime(ReminderEntry entry, DateTime previousTickTime, DateTime now)
879899
{
880900
Debug.Assert(previousTickTime.Kind == DateTimeKind.Utc);
@@ -883,7 +903,7 @@ internal static DateTime CalculateFollowingTickTime(ReminderEntry entry, DateTim
883903
return nextTick > previousTickTime ? nextTick : previousTickTime.AddClamped(entry.Period);
884904
}
885905

886-
private bool TryEvictLocalReminder(LocalReminderData reminder, long scheduleVersion)
906+
private bool TryEvictLocalReminder(LocalReminderData reminder, long scheduleVersion, DateTime? lastFiredTickTime)
887907
{
888908
CheckRuntimeContext();
889909

@@ -897,7 +917,7 @@ private bool TryEvictLocalReminder(LocalReminderData reminder, long scheduleVers
897917
}
898918

899919
var sequence = GetLocalMutationSequence();
900-
if (!reminder.TryStopOutsideLoadingWindow(scheduleVersion, sequence))
920+
if (!reminder.TryStopOutsideLoadingWindow(scheduleVersion, sequence, lastFiredTickTime))
901921
{
902922
return false;
903923
}
@@ -924,6 +944,7 @@ private sealed class LocalReminderData
924944
#endif
925945
private readonly ReminderIdentity _identity;
926946
private ReminderEntry? _entry;
947+
private DateTime? _lastFiredTickTime;
927948
private CancellationTokenSource _scheduleChangedCancellation = new();
928949
private long _scheduleVersion;
929950

@@ -1025,6 +1046,19 @@ public LocalReminderState State
10251046
}
10261047
}
10271048

1049+
public bool ShouldRetainFiredOccurrenceTombstone(ReminderEntry entry, DateTime nextTick)
1050+
{
1051+
lock (_lock)
1052+
{
1053+
return _entry is { } localEntry
1054+
&& LocalReminderService.ShouldRetainFiredOccurrenceTombstone(
1055+
localEntry,
1056+
entry,
1057+
_lastFiredTickTime,
1058+
nextTick);
1059+
}
1060+
}
1061+
10281062
public bool TryStart()
10291063
{
10301064
GrainId grainId;
@@ -1102,7 +1136,10 @@ public Task StopAsync(ReminderEvents.LocalReminderStopReason reason, long? local
11021136
return runTask ?? Task.CompletedTask;
11031137
}
11041138

1105-
public bool TryStopOutsideLoadingWindow(long scheduleVersion, long localSequenceNumber)
1139+
public bool TryStopOutsideLoadingWindow(
1140+
long scheduleVersion,
1141+
long localSequenceNumber,
1142+
DateTime? lastFiredTickTime)
11061143
{
11071144
ReminderEntry entry;
11081145
CancellationTokenSource scheduleChangedCancellation;
@@ -1114,6 +1151,7 @@ public bool TryStopOutsideLoadingWindow(long scheduleVersion, long localSequence
11141151
}
11151152

11161153
_localSequenceNumber = localSequenceNumber;
1154+
_lastFiredTickTime = lastFiredTickTime;
11171155
_stopReason = (int)ReminderEvents.LocalReminderStopReason.OutsideLoadingWindow;
11181156
entry = _entry ?? throw new InvalidOperationException($"Reminder {_identity} is a tombstone.");
11191157
scheduleChangedCancellation = _scheduleChangedCancellation;
@@ -1238,7 +1276,7 @@ private async Task RunAsync(GrainId grainId, string reminderName)
12381276
// a future table refresh will load it again as it approaches the window.
12391277
if (tickTime > now.AddClamped(_shared.reminderOptions.ReminderLoadingWindow))
12401278
{
1241-
if (_shared.TryEvictLocalReminder(this, scheduleVersion))
1279+
if (_shared.TryEvictLocalReminder(this, scheduleVersion, previousTickTime))
12421280
{
12431281
return null;
12441282
}

‎test/Orleans.Reminders.Tests/TimerTests/LocalReminderServiceTests.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,34 @@ public void IsReminderWithinLoadingWindow_UsesInclusiveBoundary(int milliseconds
8686
Assert.Equal(expected, result);
8787
}
8888

89+
[Theory, TestCategory("BVT")]
90+
[InlineData(true, 0, true, true)]
91+
[InlineData(false, 0, true, false)]
92+
[InlineData(true, 0, false, false)]
93+
[InlineData(true, 1, true, false)]
94+
public void ShouldRetainFiredOccurrenceTombstone_RequiresMatchingFiredTickAndSchedule(
95+
bool hasFiredTick,
96+
int nextTickOffsetTicks,
97+
bool sameETag,
98+
bool expected)
99+
{
100+
var now = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
101+
var localEntry = CreateReminderEntry(now, TimeSpan.FromMinutes(10));
102+
var tableEntry = CreateReminderEntry(now, TimeSpan.FromMinutes(10));
103+
if (!sameETag)
104+
{
105+
tableEntry.ETag = "updated-etag";
106+
}
107+
108+
var result = LocalReminderService.ShouldRetainFiredOccurrenceTombstone(
109+
localEntry,
110+
tableEntry,
111+
hasFiredTick ? now : null,
112+
now.AddTicks(nextTickOffsetTicks));
113+
114+
Assert.Equal(expected, result);
115+
}
116+
89117
[Fact, TestCategory("BVT")]
90118
public void ReminderOptions_DefaultLoadingWindowIsTwiceDefaultRefreshPeriod()
91119
{

‎test/Orleans.Reminders.Tests/TimerTests/ReminderTests_TableGrain.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using Microsoft.Extensions.DependencyInjection;
88
using Orleans.Internal;
99
using Orleans.Reminders;
10+
using Orleans.Runtime.ReminderService;
1011
using Orleans.Testing.Reminders;
1112
using Orleans.TestingHost;
1213
using TestExtensions;
@@ -192,6 +193,41 @@ public async Task Rem_Grain_DistantReminder_IsLoadedWithinWindowAndEvictedAfterF
192193
Assert.Null(await grain.GetReminderObject(reminderName));
193194
}
194195

196+
[Fact]
197+
public async Task Rem_Grain_RefreshAtFiredTickDoesNotRedeliverOccurrence()
198+
{
199+
const string reminderName = "stale_refresh_at_fired_tick";
200+
var grain = this.GrainFactory.GetGrain<IReminderTestGrain2>(Guid.NewGuid());
201+
var grainId = grain.GetGrainId();
202+
var dueTime = ReminderLoadingWindow + TimeSpan.FromSeconds(5);
203+
var period = ReminderLoadingWindow + TimeSpan.FromSeconds(10);
204+
var firstTickTime = ReminderUtcNow.UtcDateTime + dueTime;
205+
using var cts = new CancellationTokenSource(TestConstants.InitTimeout);
206+
207+
await grain.StartReminder(reminderName, dueTime, period);
208+
209+
var activatedTask = observer.WaitForActiveReminderCountAsync(grainId, 1, cts.Token, reminderName);
210+
await AdvanceUntilAsync(activatedTask, cts.Token);
211+
await observer.WaitForLocalReminderScheduleAsync(grainId, reminderName, cts.Token);
212+
213+
var firstTickTask = observer.WaitForReminderTickAsync(grainId, cts.Token, reminderName);
214+
var firstEvictionTask = observer.WaitForReminderQuiescenceAsync(grainId, reminderName, cts.Token);
215+
await AdvanceReminderTimeAsync(firstTickTime - ReminderUtcNow.UtcDateTime, cts.Token);
216+
var firstTick = await firstTickTask;
217+
await firstEvictionTask;
218+
219+
Assert.Equal(firstTickTime, firstTick.Status.CurrentTickTime);
220+
Assert.Equal(1, observer.GetTickCount(grainId, reminderName));
221+
222+
var reminderService = HostedCluster.Silos.Single().ServiceProvider.GetRequiredService<LocalReminderService>();
223+
await reminderService.TestOnlyRefresh();
224+
225+
Assert.Equal(0, observer.GetActiveReminderCount(grainId, reminderName));
226+
Assert.Equal(1, observer.GetTickCount(grainId, reminderName));
227+
228+
await grain.StopReminder(reminderName);
229+
}
230+
195231
[Fact]
196232
public async Task Rem_Grain_DirectUpdatesMoveReminderInAndOutOfLoadingWindow()
197233
{

0 commit comments

Comments
 (0)