From 4680a302fb1564323a9983a67f4d98e97051bac2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 14:46:55 -0700 Subject: [PATCH 1/2] feat(runtime): add activation keep-alive holds --- .../Runtime/IActivationKeepAlive.cs | 20 +++++++ src/Orleans.Runtime/Catalog/ActivationData.cs | 56 ++++++++++++++++--- .../ActivationGCTestGrainInterfaces.cs | 2 + .../ActivationGCTestGrains.cs | 12 ++++ .../ActivationCollectorTests.cs | 27 +++++++++ 5 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 src/Orleans.Core/Runtime/IActivationKeepAlive.cs diff --git a/src/Orleans.Core/Runtime/IActivationKeepAlive.cs b/src/Orleans.Core/Runtime/IActivationKeepAlive.cs new file mode 100644 index 00000000000..5bc21e99097 --- /dev/null +++ b/src/Orleans.Core/Runtime/IActivationKeepAlive.cs @@ -0,0 +1,20 @@ +namespace Orleans.Runtime; + +/// +/// An activation which components can keep alive while they have pending work. +/// +internal interface IActivationKeepAlive +{ + /// + /// Prevents idle collection of the activation until a matching call to . + /// + /// + /// Holds are counted. Memory-pressure collection and explicit deactivation do not observe holds. + /// + void AcquireKeepAlive(); + + /// + /// Releases a hold acquired by . Releasing the last hold restarts the activation's idle period. + /// + void ReleaseKeepAlive(); +} diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index ba6b807001b..3ada9b23e37 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -29,6 +29,7 @@ namespace Orleans.Runtime; internal sealed partial class ActivationData : IGrainContext, ICollectibleGrainContext, + IActivationKeepAlive, IGrainExtensionBinder, IActivationWorkingSetMember, IGrainTimerRegistry, @@ -584,6 +585,27 @@ public void DelayDeactivation(TimeSpan timespan) } } + void IActivationKeepAlive.AcquireKeepAlive() + { + lock (_lock) + { + _extras ??= new(); + ++_extras.KeepAliveHolds; + } + } + + void IActivationKeepAlive.ReleaseKeepAlive() + { + lock (_lock) + { + Debug.Assert(_extras is { KeepAliveHolds: > 0 }); + if (--_extras!.KeepAliveHolds == 0) + { + MarkActive(); + } + } + } + public void ResetKeepAliveRequest() { lock (_lock) @@ -613,7 +635,10 @@ ActivationCollectionResult ICollectibleGrainContext.TryDeactivateForCollection( TimeSpan.FromTicks(Math.Max(keepAliveDuration.Ticks, CollectionAgeLimit.Ticks))); } - if (_waitingRequests.Count > 0 || _runningRequests.Count > 0 || _idleDuration.Elapsed < ageLimit) + if (_waitingRequests.Count > 0 + || _runningRequests.Count > 0 + || _idleDuration.Elapsed < ageLimit + || respectKeepAlive && _extras is { KeepAliveHolds: > 0 }) { return ActivationCollectionResult.Reschedule(CollectionAgeLimit); } @@ -1100,7 +1125,7 @@ bool IActivationWorkingSetMember.IsCandidateForRemoval(bool wouldRemove) const int IdlenessLowerBound = 10_000; lock (_lock) { - var inactive = IsInactive && _idleDuration.ElapsedMilliseconds > IdlenessLowerBound; + var inactive = IsInactive && _idleDuration.ElapsedMilliseconds > IdlenessLowerBound && _extras is not { KeepAliveHolds: > 0 }; // This instance will remain in the working set if it is either not pending removal or if it is currently active. _isInWorkingSet = !wouldRemove || !inactive; @@ -1615,13 +1640,7 @@ private void OnCompletedRequest(Message message) // is in the activation working set. if (message.IsKeepAlive) { - _idleDuration = CoarseStopwatch.StartNew(); - - if (!_isInWorkingSet) - { - _isInWorkingSet = true; - _shared.InternalRuntime.ActivationWorkingSet.OnActive(this); - } + MarkActive(); } // The below logic only works for non-reentrant activations @@ -1636,6 +1655,20 @@ private void OnCompletedRequest(Message message) _workSignal.Signal(); } + /// + /// Restarts the idle period and ensures the activation is in the working set. Must be called under . + /// + private void MarkActive() + { + _idleDuration = CoarseStopwatch.StartNew(); + + if (!_isInWorkingSet) + { + _isInWorkingSet = true; + _shared.InternalRuntime.ActivationWorkingSet.OnActive(this); + } + } + public void ReceiveMessage(object message) => ReceiveMessage((Message)message); public void ReceiveMessage(Message message) { @@ -2474,6 +2507,11 @@ private class ActivationDataExtra : Dictionary private const int IsDisposingFlag = 1 << 2; private byte _flags; + /// + /// The number of holds which prevent idle collection. + /// + public int KeepAliveHolds; + public HashSet? Timers { get => GetValueOrDefault>(nameof(Timers)); set => SetOrRemoveValue(nameof(Timers), value); } /// diff --git a/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs b/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs index 2bcd00e3938..5ec8e7e19b8 100644 --- a/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs +++ b/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs @@ -44,5 +44,7 @@ public interface IKeepAliveActivationGcTestGrain : IGrainWithGuidKey { Task SetKeepAlive(TimeSpan keepAlive); Task CancelKeepAlive(); + Task AcquireKeepAliveHold(); + Task ReleaseKeepAliveHold(); } } diff --git a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs index c33f0dc40d6..eef58398611 100644 --- a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs +++ b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs @@ -188,5 +188,17 @@ public Task CancelKeepAlive() DelayDeactivation(TimeSpan.Zero); return Task.CompletedTask; } + + public Task AcquireKeepAliveHold() + { + ((IActivationKeepAlive)GrainContext).AcquireKeepAlive(); + return Task.CompletedTask; + } + + public Task ReleaseKeepAliveHold() + { + ((IActivationKeepAlive)GrainContext).ReleaseKeepAlive(); + return Task.CompletedTask; + } } } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs index 0a658d7fc54..c73a1c88279 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs @@ -760,6 +760,33 @@ public async Task ActivationCollectorShouldCollectAfterCancellingKeepAlive() Assert.Equal(0, activationsNotCollected); } + [TestSuite("Functional")] + [TestProvider("None")] + [Fact, TestCategory("ActivationCollector"), TestCategory("Functional")] + public async Task ActivationCollectorShouldCollectAfterReleasingKeepAliveHolds() + { + var cancellationToken = TestContext.Current.CancellationToken; + await Initialize(DEFAULT_IDLE_TIMEOUT, cancellationToken); + + var fullGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(KeepAliveActivationGcTestGrain)); + var grain = this.testCluster.GrainFactory!.GetGrain(Guid.NewGuid()); + await grain.AcquireKeepAliveHold(); + await grain.AcquireKeepAliveHold(); + + // Holds are counted, so the activation stays alive until every hold is released. + var mgmtGrain = this.testCluster.GrainFactory!.GetGrain(0); + await mgmtGrain.ForceActivationCollection(TimeSpan.Zero, cancellationToken); + Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + + await grain.ReleaseKeepAliveHold(); + await mgmtGrain.ForceActivationCollection(TimeSpan.Zero, cancellationToken); + Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + + await grain.ReleaseKeepAliveHold(); + await WaitForActivationCountToConverge(fullGrainTypeName, expectedCount: 0, cancellationToken); + Assert.Equal(0, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + } + [TestSuite("SlowBVT")] [TestProvider("None")] [Fact, TestCategory("SlowBVT"), TestCategory("Timers")] From ddd664cb9f800cc5a094a4e9f62ea41f1a1285a6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 15:04:27 -0700 Subject: [PATCH 2/2] fix(runtime): only restart idle period for valid activations on hold release --- .../Runtime/IActivationKeepAlive.cs | 2 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 4 ++- .../ActivationCollectorTests.cs | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Core/Runtime/IActivationKeepAlive.cs b/src/Orleans.Core/Runtime/IActivationKeepAlive.cs index 5bc21e99097..e9302882cd7 100644 --- a/src/Orleans.Core/Runtime/IActivationKeepAlive.cs +++ b/src/Orleans.Core/Runtime/IActivationKeepAlive.cs @@ -14,7 +14,7 @@ internal interface IActivationKeepAlive void AcquireKeepAlive(); /// - /// Releases a hold acquired by . Releasing the last hold restarts the activation's idle period. + /// Releases a hold acquired by . Releasing the last hold restarts the idle period of a valid activation. /// void ReleaseKeepAlive(); } diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 3ada9b23e37..eb877f5220f 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -599,7 +599,9 @@ void IActivationKeepAlive.ReleaseKeepAlive() lock (_lock) { Debug.Assert(_extras is { KeepAliveHolds: > 0 }); - if (--_extras!.KeepAliveHolds == 0) + + // A deactivating or disposed activation must not re-enter the working set. + if (--_extras!.KeepAliveHolds == 0 && State is ActivationState.Valid) { MarkActive(); } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs index c73a1c88279..4ac0c474ec8 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs @@ -787,6 +787,35 @@ public async Task ActivationCollectorShouldCollectAfterReleasingKeepAliveHolds() Assert.Equal(0, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); } + [TestSuite("Functional")] + [TestProvider("None")] + [Fact, TestCategory("ActivationCollector"), TestCategory("Functional")] + public async Task ActivationCollectorShouldRestartIdlePeriodWhenLastKeepAliveHoldIsReleased() + { + var cancellationToken = TestContext.Current.CancellationToken; + await Initialize(DEFAULT_IDLE_TIMEOUT, cancellationToken); + + var fullGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(KeepAliveActivationGcTestGrain)); + var grain = this.testCluster.GrainFactory!.GetGrain(Guid.NewGuid()); + await grain.AcquireKeepAliveHold(); + + var directory = ((InProcessSiloHandle)this.testCluster.Primary!).SiloHost.Services.GetRequiredService(); + var keepAlive = Assert.IsAssignableFrom(directory.FindTarget(((GrainReference)grain).GrainId)); + + // The idle period which began after the last request expires while the hold is outstanding. + await Task.Delay(DEFAULT_IDLE_TIMEOUT + DEFAULT_COLLECTION_QUANTUM, cancellationToken); + var mgmtGrain = this.testCluster.GrainFactory!.GetGrain(0); + await mgmtGrain.ForceActivationCollection(DEFAULT_IDLE_TIMEOUT, cancellationToken); + Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + + // Releasing the last hold outside request processing starts a fresh idle period. + keepAlive.ReleaseKeepAlive(); + await mgmtGrain.ForceActivationCollection(DEFAULT_IDLE_TIMEOUT, cancellationToken); + Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + + await WaitForActivationCountToConverge(fullGrainTypeName, expectedCount: 0, cancellationToken); + } + [TestSuite("SlowBVT")] [TestProvider("None")] [Fact, TestCategory("SlowBVT"), TestCategory("Timers")]