Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/Orleans.Core/Runtime/IActivationKeepAlive.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
namespace Orleans.Runtime;

/// <summary>
/// An activation which components can keep alive while they have pending work.
/// </summary>
internal interface IActivationKeepAlive
{
/// <summary>
/// Prevents idle collection of the activation until a matching call to <see cref="ReleaseKeepAlive"/>.
/// </summary>
/// <remarks>
/// Holds are counted. Memory-pressure collection and explicit deactivation do not observe holds.
/// </remarks>
void AcquireKeepAlive();

/// <summary>
/// Releases a hold acquired by <see cref="AcquireKeepAlive"/>. Releasing the last hold restarts the idle period of a valid activation.
/// </summary>
void ReleaseKeepAlive();
}
58 changes: 49 additions & 9 deletions src/Orleans.Runtime/Catalog/ActivationData.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ namespace Orleans.Runtime;
internal sealed partial class ActivationData :
IGrainContext,
ICollectibleGrainContext,
IActivationKeepAlive,
IGrainExtensionBinder,
IActivationWorkingSetMember,
IGrainTimerRegistry,
Expand Down Expand Up @@ -584,6 +585,29 @@ public void DelayDeactivation(TimeSpan timespan)
}
}

void IActivationKeepAlive.AcquireKeepAlive()
{
lock (_lock)
{
_extras ??= new();
++_extras.KeepAliveHolds;
}
}

void IActivationKeepAlive.ReleaseKeepAlive()
{
lock (_lock)
{
Debug.Assert(_extras is { KeepAliveHolds: > 0 });

// A deactivating or disposed activation must not re-enter the working set.
if (--_extras!.KeepAliveHolds == 0 && State is ActivationState.Valid)
{
MarkActive();
}
}
}

public void ResetKeepAliveRequest()
{
lock (_lock)
Expand Down Expand Up @@ -613,7 +637,10 @@ ActivationCollectionResult ICollectibleGrainContext.TryDeactivateForCollection(
TimeSpan.FromTicks(Math.Max(keepAliveDuration.Ticks, CollectionAgeLimit.Ticks)));
}

if (_waitingRequests.Count > 0 || _runningRequests.Count > 0 || _idleDuration.Elapsed < ageLimit)
if (_waitingRequests.Count > 0
|| _runningRequests.Count > 0
|| _idleDuration.Elapsed < ageLimit
|| respectKeepAlive && _extras is { KeepAliveHolds: > 0 })
{
return ActivationCollectionResult.Reschedule(CollectionAgeLimit);
}
Expand Down Expand Up @@ -1100,7 +1127,7 @@ bool IActivationWorkingSetMember.IsCandidateForRemoval(bool wouldRemove)
const int IdlenessLowerBound = 10_000;
lock (_lock)
{
var inactive = IsInactive && _idleDuration.ElapsedMilliseconds > IdlenessLowerBound;
var inactive = IsInactive && _idleDuration.ElapsedMilliseconds > IdlenessLowerBound && _extras is not { KeepAliveHolds: > 0 };

// This instance will remain in the working set if it is either not pending removal or if it is currently active.
_isInWorkingSet = !wouldRemove || !inactive;
Expand Down Expand Up @@ -1615,13 +1642,7 @@ private void OnCompletedRequest(Message message)
// is in the activation working set.
if (message.IsKeepAlive)
{
_idleDuration = CoarseStopwatch.StartNew();

if (!_isInWorkingSet)
{
_isInWorkingSet = true;
_shared.InternalRuntime.ActivationWorkingSet.OnActive(this);
}
MarkActive();
}

// The below logic only works for non-reentrant activations
Expand All @@ -1636,6 +1657,20 @@ private void OnCompletedRequest(Message message)
_workSignal.Signal();
}

/// <summary>
/// Restarts the idle period and ensures the activation is in the working set. Must be called under <see cref="_lock"/>.
/// </summary>
private void MarkActive()
{
_idleDuration = CoarseStopwatch.StartNew();

if (!_isInWorkingSet)
{
_isInWorkingSet = true;
_shared.InternalRuntime.ActivationWorkingSet.OnActive(this);
}
}

public void ReceiveMessage(object message) => ReceiveMessage((Message)message);
public void ReceiveMessage(Message message)
{
Expand Down Expand Up @@ -2474,6 +2509,11 @@ private class ActivationDataExtra : Dictionary<object, object>
private const int IsDisposingFlag = 1 << 2;
private byte _flags;

/// <summary>
/// The number of <see cref="IActivationKeepAlive"/> holds which prevent idle collection.
/// </summary>
public int KeepAliveHolds;

public HashSet<IGrainTimer>? Timers { get => GetValueOrDefault<HashSet<IGrainTimer>>(nameof(Timers)); set => SetOrRemoveValue(nameof(Timers), value); }

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,7 @@ public interface IKeepAliveActivationGcTestGrain : IGrainWithGuidKey
{
Task SetKeepAlive(TimeSpan keepAlive);
Task CancelKeepAlive();
Task AcquireKeepAliveHold();
Task ReleaseKeepAliveHold();
}
}
12 changes: 12 additions & 0 deletions test/Grains/TestInternalGrains/ActivationGCTestGrains.cs
Original file line number Diff line number Diff line change
Expand Up @@ -188,5 +188,17 @@ public Task CancelKeepAlive()
DelayDeactivation(TimeSpan.Zero);
return Task.CompletedTask;
}

public Task AcquireKeepAliveHold()
{
((IActivationKeepAlive)GrainContext).AcquireKeepAlive();
return Task.CompletedTask;
}

public Task ReleaseKeepAliveHold()
{
((IActivationKeepAlive)GrainContext).ReleaseKeepAlive();
return Task.CompletedTask;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -760,6 +760,62 @@ public async Task ActivationCollectorShouldCollectAfterCancellingKeepAlive()
Assert.Equal(0, activationsNotCollected);
}

[TestSuite("Functional")]
[TestProvider("None")]
[Fact, TestCategory("ActivationCollector"), TestCategory("Functional")]
public async Task ActivationCollectorShouldCollectAfterReleasingKeepAliveHolds()
{
var cancellationToken = TestContext.Current.CancellationToken;
await Initialize(DEFAULT_IDLE_TIMEOUT, cancellationToken);

var fullGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(KeepAliveActivationGcTestGrain));
var grain = this.testCluster.GrainFactory!.GetGrain<IKeepAliveActivationGcTestGrain>(Guid.NewGuid());
await grain.AcquireKeepAliveHold();
await grain.AcquireKeepAliveHold();

// Holds are counted, so the activation stays alive until every hold is released.
var mgmtGrain = this.testCluster.GrainFactory!.GetGrain<IManagementGrain>(0);
await mgmtGrain.ForceActivationCollection(TimeSpan.Zero, cancellationToken);
Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName));

await grain.ReleaseKeepAliveHold();
await mgmtGrain.ForceActivationCollection(TimeSpan.Zero, cancellationToken);
Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName));

await grain.ReleaseKeepAliveHold();
await WaitForActivationCountToConverge(fullGrainTypeName, expectedCount: 0, cancellationToken);
Comment thread
ReubenBond marked this conversation as resolved.
Resolved
Assert.Equal(0, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName));
}

[TestSuite("Functional")]
[TestProvider("None")]
[Fact, TestCategory("ActivationCollector"), TestCategory("Functional")]
public async Task ActivationCollectorShouldRestartIdlePeriodWhenLastKeepAliveHoldIsReleased()
{
var cancellationToken = TestContext.Current.CancellationToken;
await Initialize(DEFAULT_IDLE_TIMEOUT, cancellationToken);

var fullGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(KeepAliveActivationGcTestGrain));
var grain = this.testCluster.GrainFactory!.GetGrain<IKeepAliveActivationGcTestGrain>(Guid.NewGuid());
await grain.AcquireKeepAliveHold();

var directory = ((InProcessSiloHandle)this.testCluster.Primary!).SiloHost.Services.GetRequiredService<ActivationDirectory>();
var keepAlive = Assert.IsAssignableFrom<IActivationKeepAlive>(directory.FindTarget(((GrainReference)grain).GrainId));

// The idle period which began after the last request expires while the hold is outstanding.
await Task.Delay(DEFAULT_IDLE_TIMEOUT + DEFAULT_COLLECTION_QUANTUM, cancellationToken);
var mgmtGrain = this.testCluster.GrainFactory!.GetGrain<IManagementGrain>(0);
await mgmtGrain.ForceActivationCollection(DEFAULT_IDLE_TIMEOUT, cancellationToken);
Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName));

// Releasing the last hold outside request processing starts a fresh idle period.
keepAlive.ReleaseKeepAlive();
await mgmtGrain.ForceActivationCollection(DEFAULT_IDLE_TIMEOUT, cancellationToken);
Assert.Equal(1, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName));

await WaitForActivationCountToConverge(fullGrainTypeName, expectedCount: 0, cancellationToken);
}

[TestSuite("SlowBVT")]
[TestProvider("None")]
[Fact, TestCategory("SlowBVT"), TestCategory("Timers")]
Expand Down
Loading