From aec5e6476f30a47d2215c004efcd96c52e0e2dd6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 14:35:33 -0700 Subject: [PATCH 01/31] feat(runtime): add RunRequestAsync --- .../content/docs/grains/request-scheduling.md | 2 + docs/site/src/content/docs/grains/timers.md | 2 + .../Core/IGrainBase.cs | 95 ++++++++++ .../Core/IGrainRequestRunner.cs | 23 +++ src/Orleans.Runtime/Catalog/ActivationData.cs | 32 ++++ .../Core/InsideRuntimeClient.cs | 4 +- .../Core/InternalGrainRuntime.cs | 4 +- src/Orleans.Runtime/OrleansContracts.txt | 3 + src/Orleans.Runtime/Timers/GrainRequest.cs | 144 ++++++++++++++ .../Orleans.Core.Abstractions.cs | 8 + .../TestGrainInterfaces/IRunRequestGrain.cs | 44 +++++ test/Grains/TestGrains/RunRequestGrain.cs | 144 ++++++++++++++ .../RunRequestTests.cs | 175 ++++++++++++++++++ 13 files changed, 678 insertions(+), 2 deletions(-) create mode 100644 src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs create mode 100644 src/Orleans.Runtime/Timers/GrainRequest.cs create mode 100644 test/Grains/TestGrainInterfaces/IRunRequestGrain.cs create mode 100644 test/Grains/TestGrains/RunRequestGrain.cs create mode 100644 test/Orleans.DefaultCluster.Tests/RunRequestTests.cs diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index 87ef6da544d..b665604be34 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -62,4 +62,6 @@ The scope permits callbacks associated with that call chain until it is disposed Grain timer callbacks participate in scheduling. Callbacks don't interleave by default; configure `GrainTimerCreationOptions.Interleave` when intentional. + runs a callback as a request on the activation, from any thread. The request follows the grain's reentrancy rules, passes through grain call filters, starts a new call chain, and keeps the activation alive. Use it to apply results from background work, such as stream or messaging handlers, under the same scheduling as grain calls. On a non-reentrant grain, awaiting the request from within another request on the same activation deadlocks. + Use only to isolate unavoidable synchronous blocking or CPU work from the Orleans scheduler. Don't access grain state from the thread-pool delegate. Await it and update grain state after execution resumes on the activation scheduler. See [External tasks and grains](external-tasks-and-grains.md). diff --git a/docs/site/src/content/docs/grains/timers.md b/docs/site/src/content/docs/grains/timers.md index b5e40e3814b..2efa9eb20a5 100644 --- a/docs/site/src/content/docs/grains/timers.md +++ b/docs/site/src/content/docs/grains/timers.md @@ -11,6 +11,8 @@ A grain timer schedules periodic work for one grain activation. Each callback ex Use a grain timer for frequent work whose lifetime and state belong to the current activation. Use a [reminder](reminders.md) when the schedule belongs to the logical grain and must survive activation or cluster lifecycle changes. +To run a callback once as a grain request, use . See [Request scheduling](request-scheduling.md#timers-and-external-work). + ## Register a grain timer Register timers with . is obsolete. diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 91ea87df564..8974cda35cb 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -53,6 +53,101 @@ public static void DeactivateOnIdle(this IGrainBase grain) => /// public static void MigrateOnIdle(this IGrainBase grain) => grain.GrainContext.Migrate(RequestContext.CallContextData?.Value.Values); + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(callback); + return GetRequestRunner(grain).RunRequestAsync, object?>( + static (callback, cancellationToken) => callback(cancellationToken), + callback, + cancellationToken); + } + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the parameter. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(callback); + return GetRequestRunner(grain).RunRequestAsync(callback, state, cancellationToken); + } + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the callback's result. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(callback); + return GetRequestRunner(grain).RunRequestAsync>, TResult>( + static (callback, cancellationToken) => callback(cancellationToken), + callback, + cancellationToken); + } + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the parameter. + /// The type of the callback's result. + /// A which completes with the outcome of the callback. + /// + /// + /// The request is scheduled with the activation's other requests and follows the grain's reentrancy rules: + /// on a non-reentrant grain, the callback runs exclusively, the same as a grain method call. + /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync, + /// starts a new call chain, and keeps the activation alive. + /// + /// + /// Use this method to run code from a background task, such as a stream or messaging handler, as a grain request. + /// It can be called from any thread. On a non-reentrant grain, awaiting the returned task from within a request on the same activation deadlocks, + /// as does awaiting a call to the same activation from within the callback. + /// + /// + /// If the activation deactivates before the request starts, the returned task faults with + /// . + /// + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(callback); + return GetRequestRunner(grain).RunRequestAsync(callback, state, cancellationToken); + } + + private static IGrainRequestRunner GetRequestRunner(IGrainBase grain) => + grain.GrainContext as IGrainRequestRunner ?? throw new NotSupportedException($"{grain.GrainContext.GetType()} does not support running grain requests."); + /// /// Creates a grain timer. /// diff --git a/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs b/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs new file mode 100644 index 00000000000..c3bddecb079 --- /dev/null +++ b/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs @@ -0,0 +1,23 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Runtime; + +/// +/// A grain context which runs callbacks as requests on its activation. +/// +internal interface IGrainRequestRunner +{ + /// + /// Queues as a request on this activation. + /// + /// + /// The returned task completes with the result of the returned by the callback, or with + /// when the callback returns a non-generic . + /// + Task RunRequestAsync( + Func callback, + TState state, + CancellationToken cancellationToken); +} diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 81a44aa82f5..5ebd4ce932d 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -30,6 +30,7 @@ internal sealed partial class ActivationData : IGrainContext, ICollectibleGrainContext, IActivationKeepAlive, + IGrainRequestRunner, IGrainExtensionBinder, IActivationWorkingSetMember, IGrainTimerRegistry, @@ -534,6 +535,7 @@ internal List DequeueAllWaitingRequests() // Local-only messages are not allowed to escape the activation. if (message.IsLocalOnly) { + (message.BodyObject as ILocalRequest)?.Reject(new OrleansMessageRejectionException($"Activation {this} is no longer valid.")); continue; } @@ -608,6 +610,32 @@ void IActivationKeepAlive.ReleaseKeepAlive() } } + Task IGrainRequestRunner.RunRequestAsync(Func callback, TState state, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + var request = new GrainRequest(callback, state, cancellationToken); + + // Requests start a new call chain, matching timer ticks. + var message = _shared.InternalRuntime.MessageFactory.CreateMessage(body: request, options: CodeGeneration.InvokeMethodOptions.OneWay, requestContextData: null); + message.SetInfiniteTimeToLive(); + message.SendingGrain = GrainId; + message.TargetGrain = GrainId; + message.SendingSilo = Address.SiloAddress; + message.TargetSilo = Address.SiloAddress; + message.InterfaceType = IGrainRequestInvoker.InterfaceType; + message.IsKeepAlive = true; + + // The request completes on this activation or is rejected with it. + message.IsLocalOnly = true; + + ReceiveMessage(message); + return request.Task; + } + public void ResetKeepAliveRequest() { lock (_lock) @@ -1253,6 +1281,10 @@ void ProcessPendingRequests() { _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); } + else + { + (message.BodyObject as ILocalRequest)?.Reject(exception); + } _waitingRequests.RemoveAt(i); continue; diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 8ec03015ce6..ad903f43ff5 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -340,7 +340,9 @@ public async Task Invoke(IGrainContext target, Message message) if (response.Exception is { } invocationException) { - LogGrainInvokeException(this.invokeExceptionLogger, message.Direction != Message.Directions.OneWay ? LogLevel.Debug : LogLevel.Warning, invocationException, message); + // Callers observe two-way failures, and local-only request owners report their own failures. + var logLevel = message.Direction != Message.Directions.OneWay || message.IsLocalOnly ? LogLevel.Debug : LogLevel.Warning; + LogGrainInvokeException(this.invokeExceptionLogger, logLevel, invocationException, message); // If a grain allowed an inconsistent state exception to escape and the exception originated from // this activation, then deactivate it. diff --git a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs index 3fb20d38c2f..8bada07015f 100644 --- a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs +++ b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs @@ -20,7 +20,8 @@ internal class InternalGrainRuntime( IOptions collectionOptions, ILocalGrainDirectory localGrainDirectory, IActivationWorkingSet activationWorkingSet, - ActivationCollector activationCollector) + ActivationCollector activationCollector, + MessageFactory messageFactory) { public InsideRuntimeClient RuntimeClient { get; } = catalog.RuntimeClient; public MessageCenter MessageCenter { get; } = messageCenter; @@ -33,5 +34,6 @@ internal class InternalGrainRuntime( public ILocalGrainDirectory LocalGrainDirectory { get; } = localGrainDirectory; public IActivationWorkingSet ActivationWorkingSet { get; } = activationWorkingSet; public ActivationCollector ActivationCollector { get; } = activationCollector; + public MessageFactory MessageFactory { get; } = messageFactory; } } diff --git a/src/Orleans.Runtime/OrleansContracts.txt b/src/Orleans.Runtime/OrleansContracts.txt index 3dd8082bb7d..ed58edddb4b 100644 --- a/src/Orleans.Runtime/OrleansContracts.txt +++ b/src/Orleans.Runtime/OrleansContracts.txt @@ -70,6 +70,9 @@ interface [GrainInterfaceType("Orleans.Runtime.ICatalog")] Orleans.Runtime.ICata interface [GrainInterfaceType("Orleans.Runtime.IGrainCallCancellationManagerSystemTarget")] Orleans.Runtime.IGrainCallCancellationManagerSystemTarget [Version(0)] AF79F3FA: CancelCallsAsync(System.Collections.Generic.List, System.Threading.CancellationToken) -> ValueTask +interface [GrainInterfaceType("Orleans.Runtime.IGrainRequestInvoker")] Orleans.Runtime.IGrainRequestInvoker [Version(0)] + 6D1E58B4: RunRequestAsync(System.Threading.CancellationToken) -> Task + interface [GrainInterfaceType("Orleans.Runtime.IGrainTimerInvoker")] Orleans.Runtime.IGrainTimerInvoker [Version(0)] 3F6C2672: InvokeCallbackAsync(System.Threading.CancellationToken) -> Task diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Runtime/Timers/GrainRequest.cs new file mode 100644 index 00000000000..11cc22aa364 --- /dev/null +++ b/src/Orleans.Runtime/Timers/GrainRequest.cs @@ -0,0 +1,144 @@ +using System.Reflection; +using Orleans.Serialization.Invocation; + +namespace Orleans.Runtime; + +/// +/// A request queued on an activation by . +/// +/// +/// The request is its own completion source, so queuing a request allocates only this object and its message. +/// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. +/// +internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest +{ + private readonly Func _callback; + private readonly TState _state; + private readonly CancellationTokenRegistration _registration; + + public GrainRequest(Func callback, TState state, CancellationToken cancellationToken) + : base(TaskCreationOptions.RunContinuationsAsynchronously) + { + _callback = callback; + _state = state; + _registration = cancellationToken.UnsafeRegister( + static (state, cancellationToken) => ((GrainRequest)state!).TrySetCanceled(cancellationToken), + this); + } + + public ValueTask Invoke() + { + var cancellationToken = _registration.Token; + if (Task.IsCompleted || cancellationToken.CanBeCanceled && !_registration.Unregister()) + { + // The request was canceled while it was queued. + return new(Response.Completed); + } + + Task task; + try + { + task = _callback(_state, cancellationToken); + } + catch (Exception exception) + { + TrySetException(exception); + return new(Response.FromException(exception)); + } + + if (task.IsCompletedSuccessfully) + { + TrySetResult(GetResult(task)); + return new(Response.Completed); + } + + return AwaitCallbackAsync(task); + } + + private async ValueTask AwaitCallbackAsync(Task task) + { + try + { + await task; + TrySetResult(GetResult(task)); + return Response.Completed; + } + catch (OperationCanceledException exception) when (task.IsCanceled) + { + TrySetCanceled(exception.CancellationToken); + return Response.FromException(exception); + } + catch (Exception exception) + { + TrySetException(exception); + return Response.FromException(exception); + } + } + + private static TResult GetResult(Task task) => task is Task typed ? typed.Result : default!; + + void ILocalRequest.Reject(Exception exception) + { + _registration.Dispose(); + TrySetException(exception); + } + + public object? GetTarget() => this; + + public void SetTarget(ITargetHolder holder) + { + } + + // Declared for call filters, which inspect the interface and implementation methods. + Task IGrainRequestInvoker.RunRequestAsync(CancellationToken cancellationToken) => throw new InvalidOperationException(); + + public int GetArgumentCount() => 0; + + public object? GetArgument(int index) => throw new InvalidOperationException(); + + public void SetArgument(int index, object? value) => throw new InvalidOperationException(); + + public string GetMethodName() => nameof(IGrainRequestInvoker.RunRequestAsync); + + public string GetInterfaceName() => nameof(IGrainRequestInvoker); + + public string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{nameof(IGrainRequestInvoker.RunRequestAsync)}"; + + public MethodInfo GetMethod() => IGrainRequestInvoker.MethodInfo; + + public Type GetInterfaceType() => typeof(IGrainRequestInvoker); + + public TimeSpan? GetDefaultResponseTimeout() => null; + + public void Dispose() + { + } + + public override string ToString() => $"Request '{_callback.Target}.{_callback.Method}'"; +} + +/// +/// The interface which call filters observe for requests queued by . +/// +internal interface IGrainRequestInvoker : IAddressable +{ + internal static readonly GrainInterfaceType InterfaceType = GrainInterfaceType.Create("Orleans.Runtime.IGrainRequestInvoker"); + internal static readonly MethodInfo MethodInfo = typeof(IGrainRequestInvoker).GetMethod(nameof(RunRequestAsync), BindingFlags.Instance | BindingFlags.Public)!; + + /// + /// Runs the request callback. + /// + [Alias("6D1E58B4")] + Task RunRequestAsync(CancellationToken cancellationToken = default); +} + +/// +/// A local-only request which must be notified when its activation drops it without running it. +/// +internal interface ILocalRequest +{ + /// + /// Completes the request with . + /// + void Reject(Exception exception); +} diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 1232b483615..5b7e6cec207 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -1118,6 +1118,14 @@ public static void MigrateOnIdle(this IGrainBase grain) { } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, Runtime.GrainTimerCreationOptions options) { throw null; } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, System.TimeSpan dueTime, System.TimeSpan period) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } } public delegate System.Threading.Tasks.Task GrainCallFilterDelegate(IGrainCallContext context); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs new file mode 100644 index 00000000000..d7cb0c44e98 --- /dev/null +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -0,0 +1,44 @@ +using Orleans.Concurrency; + +namespace UnitTests.GrainInterfaces; + +public interface IRunRequestGrain : IGrainWithGuidKey +{ + Task StartRequestDuringCall(); + + Task StartBlockingRequest(); + + [AlwaysInterleave] + Task ReleaseBlockingRequest(); + + Task Ping(); + + Task StartValueRequest(int value); + + Task StartRequestContextRequest(string value); + + Task StartFaultingRequest(); + + Task StartCanceledRequest(bool cancelBeforeStart); + + [AlwaysInterleave] + Task AwaitPendingRequest(); + + [AlwaysInterleave] + Task> GetLog(); + + [AlwaysInterleave] + Task> GetFilteredCalls(); + + Task RetainInstance(); + + Task DeactivateAndRetainInstance(); +} + +public interface INonReentrantRunRequestGrain : IRunRequestGrain +{ +} + +public interface IReentrantRunRequestGrain : IRunRequestGrain +{ +} diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs new file mode 100644 index 00000000000..c2555822fac --- /dev/null +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -0,0 +1,144 @@ +using System.Collections.Concurrent; +using Orleans.Concurrency; +using UnitTests.GrainInterfaces; + +namespace UnitTests.Grains; + +public abstract class RunRequestGrainBase : Grain, IRunRequestGrain, IIncomingGrainCallFilter +{ + private static readonly ConcurrentDictionary RetainedInstances = new(); + private readonly List _log = []; + private readonly List _filteredCalls = []; + private TaskCompletionSource? _gate; + private Task _pending = Task.CompletedTask; + + public async Task Invoke(IIncomingGrainCallContext context) + { + _filteredCalls.Add($"{context.InterfaceName}.{context.MethodName}"); + await context.Invoke(); + } + + public async Task StartRequestDuringCall() + { + _log.Add("call-start"); + _pending = this.RunRequestAsync(_ => + { + _log.Add("request"); + return Task.CompletedTask; + }); + + await Task.Delay(200); + _log.Add("call-end"); + } + + public Task StartBlockingRequest() + { + var gate = _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); + _pending = this.RunRequestAsync(async _ => + { + _log.Add("request-start"); + await gate.Task; + _log.Add("request-end"); + }); + + return Task.CompletedTask; + } + + public Task ReleaseBlockingRequest() + { + _gate!.SetResult(); + return Task.CompletedTask; + } + + public Task Ping() + { + _log.Add("ping"); + return Task.CompletedTask; + } + + public Task StartValueRequest(int value) + { + _pending = this.RunRequestAsync(static (value, _) => Task.FromResult((value * 2).ToString()), value); + return Task.CompletedTask; + } + + public Task StartRequestContextRequest(string value) + { + RequestContext.Set("run-request", value); + _pending = this.RunRequestAsync(static _ => Task.FromResult(RequestContext.Get("run-request") as string)); + return Task.CompletedTask; + } + + public Task StartFaultingRequest() + { + _pending = this.RunRequestAsync(static async _ => + { + await Task.Yield(); + throw new InvalidOperationException("Request failed."); + }); + + return Task.CompletedTask; + } + + public Task StartCanceledRequest(bool cancelBeforeStart) + { + using var cancellation = new CancellationTokenSource(); + if (cancelBeforeStart) + { + cancellation.Cancel(); + } + + _pending = this.RunRequestAsync(_ => + { + _log.Add("canceled-request"); + return Task.CompletedTask; + }, cancellation.Token); + + cancellation.Cancel(); + return Task.CompletedTask; + } + + public async Task AwaitPendingRequest() + { + await _pending; + return (_pending as Task)?.Result; + } + + public Task> GetLog() => Task.FromResult(_log.ToList()); + + public Task> GetFilteredCalls() => Task.FromResult(_filteredCalls.ToList()); + + public Task RetainInstance() + { + RetainedInstances[this.GetPrimaryKey()] = this; + return Task.CompletedTask; + } + + public Task DeactivateAndRetainInstance() + { + RetainedInstances[this.GetPrimaryKey()] = this; + DeactivateOnIdle(); + return Task.CompletedTask; + } + + /// + /// Queues a request on the retained instance from outside of the activation. + /// + public static Task RunRequestOnRetainedInstance(Guid key) => + RetainedInstances[key].RunRequestAsync( + static (grain, _) => + { + grain._log.Add("outside"); + return Task.FromResult("ran"); + }, + RetainedInstances[key]); +} + +public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReentrantRunRequestGrain +{ +} + +[Reentrant] +public sealed class ReentrantRunRequestGrain : RunRequestGrainBase, IReentrantRunRequestGrain +{ +} diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs new file mode 100644 index 00000000000..b6ce9fe9515 --- /dev/null +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -0,0 +1,175 @@ +using Orleans.Runtime; +using TestExtensions; +using UnitTests.GrainInterfaces; +using UnitTests.Grains; +using Xunit; + +namespace DefaultCluster.Tests.General; + +/// +/// Tests for , +/// which runs a delegate as a request on the grain's activation. +/// +[TestCategory("BVT"), TestCategory("Scheduler")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +public class RunRequestTests : HostedTestClusterEnsureDefaultStarted +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); + + public RunRequestTests(DefaultClusterFixture fixture) : base(fixture) { } + + [Fact] + public async Task NonReentrantGrain_RunsRequestAfterCurrentCall() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartRequestDuringCall(); + await grain.AwaitPendingRequest(); + + Assert.Equal(["call-start", "call-end", "request"], await grain.GetLog()); + } + + [Fact] + public async Task ReentrantGrain_InterleavesRequestWithCurrentCall() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartRequestDuringCall(); + await grain.AwaitPendingRequest(); + + Assert.Equal(["call-start", "request", "call-end"], await grain.GetLog()); + } + + [Fact] + public async Task NonReentrantGrain_RequestExcludesGrainCalls() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.StartBlockingRequest(); + await WaitForLogEntryAsync(grain, "request-start"); + + var ping = grain.Ping(); + await Task.Delay(200, TestContext.Current.CancellationToken); + Assert.False(ping.IsCompleted); + + await grain.ReleaseBlockingRequest(); + await ping; + await grain.AwaitPendingRequest(); + + Assert.Equal(["request-start", "request-end", "ping"], await grain.GetLog()); + } + + [Fact] + public async Task ReentrantGrain_GrainCallsInterleaveWithRequest() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.StartBlockingRequest(); + await WaitForLogEntryAsync(grain, "request-start"); + + await grain.Ping(); + await grain.ReleaseBlockingRequest(); + await grain.AwaitPendingRequest(); + + Assert.Equal(["request-start", "ping", "request-end"], await grain.GetLog()); + } + + [Fact] + public async Task RunRequest_ReturnsCallbackResult() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartValueRequest(21); + + Assert.Equal("42", await grain.AwaitPendingRequest()); + } + + [Fact] + public async Task RunRequest_StartsNewCallChain() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartRequestContextRequest("caller"); + + Assert.Null(await grain.AwaitPendingRequest()); + } + + [Fact] + public async Task RunRequest_PropagatesCallbackException() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartFaultingRequest(); + + var exception = await Assert.ThrowsAsync(grain.AwaitPendingRequest); + Assert.Equal("Request failed.", exception.Message); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task RunRequest_CanceledBeforeStart_DoesNotRunCallback(bool cancelBeforeQueue) + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartCanceledRequest(cancelBeforeQueue); + + await Assert.ThrowsAnyAsync(grain.AwaitPendingRequest); + Assert.Empty(await grain.GetLog()); + } + + [Fact] + public async Task RunRequest_PassesThroughGrainCallFilter() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartValueRequest(1); + await grain.AwaitPendingRequest(); + + Assert.Contains("IGrainRequestInvoker.RunRequestAsync", await grain.GetFilteredCalls()); + } + + [Fact] + public async Task RunRequest_FromOutsideActivation_RunsOnActivation() + { + var key = Guid.NewGuid(); + var grain = GrainFactory.GetGrain(key); + await grain.RetainInstance(); + + Assert.Equal("ran", await RunRequestGrainBase.RunRequestOnRetainedInstance(key).WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Equal(["outside"], await grain.GetLog()); + } + + [Fact] + public async Task RunRequest_OnDeactivatedActivation_IsRejected() + { + var key = Guid.NewGuid(); + var grain = GrainFactory.GetGrain(key); + await grain.DeactivateAndRetainInstance(); + + // Requests queued while the activation is still deactivating run; once it is invalid, they are rejected. + using var timeout = new CancellationTokenSource(Timeout); + while (true) + { + try + { + await RunRequestGrainBase.RunRequestOnRetainedInstance(key).WaitAsync(timeout.Token); + } + catch (OrleansMessageRejectionException) + { + return; + } + + await Task.Delay(50, timeout.Token); + } + } + + private static async Task WaitForLogEntryAsync(IRunRequestGrain grain, string entry) + { + using var timeout = new CancellationTokenSource(Timeout); + while (!(await grain.GetLog()).Contains(entry)) + { + await Task.Delay(20, timeout.Token); + } + } +} From 4139c42142d6fc22aac748b2f1b3d4363adeb2ce Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 15:02:04 -0700 Subject: [PATCH 02/31] fix(runtime): complete RunRequestAsync when call filters end the invocation --- .../Core/IGrainBase.cs | 1 + .../Core/InsideRuntimeClient.cs | 8 ++++-- src/Orleans.Runtime/Timers/GrainRequest.cs | 27 ++++++++++++++++++- .../TestGrainInterfaces/IRunRequestGrain.cs | 13 +++++++++ test/Grains/TestGrains/RunRequestGrain.cs | 19 +++++++++++++ .../RunRequestTests.cs | 23 ++++++++++++++++ 6 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 8974cda35cb..05a69276de8 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -128,6 +128,7 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// on a non-reentrant grain, the callback runs exclusively, the same as a grain method call. /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync, /// starts a new call chain, and keeps the activation alive. + /// A call filter which fails or sets the result without invoking the request completes the returned task with that outcome. /// /// /// Use this method to run code from a background task, such as a stream or messaging handler, as a grain request. diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index ad903f43ff5..a569307c077 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -338,10 +338,13 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } + var localRequest = message.IsLocalOnly ? message.BodyObject as ILocalRequest : null; + localRequest?.OnInvoked(response); + if (response.Exception is { } invocationException) { - // Callers observe two-way failures, and local-only request owners report their own failures. - var logLevel = message.Direction != Message.Directions.OneWay || message.IsLocalOnly ? LogLevel.Debug : LogLevel.Warning; + // Callers observe two-way failures, and local requests report failures through their tasks. + var logLevel = message.Direction != Message.Directions.OneWay || localRequest is not null ? LogLevel.Debug : LogLevel.Warning; LogGrainInvokeException(this.invokeExceptionLogger, logLevel, invocationException, message); // If a grain allowed an inconsistent state exception to escape and the exception originated from @@ -374,6 +377,7 @@ public async Task Invoke(IGrainContext target, Message message) catch (Exception exc2) { LogWarningInvokeException(this.logger, exc2, message); + (message.BodyObject as ILocalRequest)?.Reject(exc2); if (message.Direction != Message.Directions.OneWay) { diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Runtime/Timers/GrainRequest.cs index 11cc22aa364..fdb0f75530f 100644 --- a/src/Orleans.Runtime/Timers/GrainRequest.cs +++ b/src/Orleans.Runtime/Timers/GrainRequest.cs @@ -9,6 +9,7 @@ namespace Orleans.Runtime; /// /// The request is its own completion source, so queuing a request allocates only this object and its message. /// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. +/// When a call filter completes the invocation without running the callback, the task reflects the filter's outcome. /// internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest { @@ -83,6 +84,25 @@ void ILocalRequest.Reject(Exception exception) TrySetException(exception); } + void ILocalRequest.OnInvoked(Response response) + { + if (Task.IsCompleted) + { + return; + } + + // A call filter completed the invocation without running the callback. + _registration.Dispose(); + if (response.Exception is { } exception) + { + TrySetException(exception); + } + else + { + TrySetResult(response.Result is TResult result ? result : default!); + } + } + public object? GetTarget() => this; public void SetTarget(ITargetHolder holder) @@ -133,7 +153,7 @@ internal interface IGrainRequestInvoker : IAddressable } /// -/// A local-only request which must be notified when its activation drops it without running it. +/// A local-only request whose task reports its outcome to the code which queued it. /// internal interface ILocalRequest { @@ -141,4 +161,9 @@ internal interface ILocalRequest /// Completes the request with . /// void Reject(Exception exception); + + /// + /// Completes the request with the invocation's if the invocation did not run the request. + /// + void OnInvoked(Response response); } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index d7cb0c44e98..2e71ecf242c 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -15,6 +15,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartValueRequest(int value); + Task SetRequestFilterMode(RequestFilterMode mode); + Task StartRequestContextRequest(string value); Task StartFaultingRequest(); @@ -42,3 +44,14 @@ public interface INonReentrantRunRequestGrain : IRunRequestGrain public interface IReentrantRunRequestGrain : IRunRequestGrain { } + +/// +/// How the grain's call filter handles requests queued by RunRequestAsync. +/// +[GenerateSerializer] +public enum RequestFilterMode +{ + Invoke, + Throw, + ShortCircuit, +} diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index c2555822fac..6cb5b5465e5 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -11,13 +11,32 @@ public abstract class RunRequestGrainBase : Grain, IRunRequestGrain, IIncomingGr private readonly List _filteredCalls = []; private TaskCompletionSource? _gate; private Task _pending = Task.CompletedTask; + private RequestFilterMode _requestFilterMode; public async Task Invoke(IIncomingGrainCallContext context) { _filteredCalls.Add($"{context.InterfaceName}.{context.MethodName}"); + if (context.InterfaceName == "IGrainRequestInvoker") + { + switch (_requestFilterMode) + { + case RequestFilterMode.Throw: + throw new InvalidOperationException("Filter failed."); + case RequestFilterMode.ShortCircuit: + context.Result = "filtered"; + return; + } + } + await context.Invoke(); } + public Task SetRequestFilterMode(RequestFilterMode mode) + { + _requestFilterMode = mode; + return Task.CompletedTask; + } + public async Task StartRequestDuringCall() { _log.Add("call-start"); diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index b6ce9fe9515..7638cb67449 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -129,6 +129,29 @@ public async Task RunRequest_PassesThroughGrainCallFilter() Assert.Contains("IGrainRequestInvoker.RunRequestAsync", await grain.GetFilteredCalls()); } + [Fact] + public async Task RunRequest_FilterThrows_FaultsRequest() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.SetRequestFilterMode(RequestFilterMode.Throw); + + await grain.StartValueRequest(1); + + var exception = await Assert.ThrowsAsync(() => grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Equal("Filter failed.", exception.Message); + } + + [Fact] + public async Task RunRequest_FilterShortCircuits_CompletesWithFilterResult() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.SetRequestFilterMode(RequestFilterMode.ShortCircuit); + + await grain.StartValueRequest(1); + + Assert.Equal("filtered", await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + [Fact] public async Task RunRequest_FromOutsideActivation_RunsOnActivation() { From 98d945fdb09fc525a0bff80bfe141c76294a67a5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 15:41:54 -0700 Subject: [PATCH 03/31] fix(runtime): honor queue limits and late cancellation in RunRequestAsync Reject RunRequestAsync requests with LimitExceededException when the activation exceeds MaxEnqueuedRequestsHardLimit; grain timer ticks remain exempt. Cancel a queued request whose token was canceled while earlier cancellation callbacks were still running. Keep Warning logging for call filter failures which the request's task does not report. --- .../Core/IGrainBase.cs | 2 + src/Orleans.Runtime/Catalog/ActivationData.cs | 18 +++- .../Core/InsideRuntimeClient.cs | 8 +- src/Orleans.Runtime/Timers/GrainRequest.cs | 28 ++++-- .../TestGrainInterfaces/IRunRequestGrain.cs | 3 + test/Grains/TestGrains/RunRequestGrain.cs | 31 +++++++ .../Runtime/GrainRequestTests.cs | 89 +++++++++++++++++++ .../RunRequestTests.cs | 51 +++++++++++ 8 files changed, 215 insertions(+), 15 deletions(-) create mode 100644 test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 05a69276de8..7ed01ec637a 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -138,6 +138,8 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// /// If the activation deactivates before the request starts, the returned task faults with /// . + /// If the activation's queue exceeds the configured maximum number of enqueued requests, the returned task faults with + /// . /// /// public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 5ebd4ce932d..afa091098f8 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1746,11 +1746,21 @@ private void ReceiveResponse(Message message) private void ReceiveRequest(Message message) { var overloadException = CheckOverloaded(); - if (overloadException != null && !message.IsLocalOnly) + if (overloadException != null) { - _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); - _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); - return; + if (!message.IsLocalOnly) + { + _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); + _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); + return; + } + + // Requests queued by RunRequestAsync honor the limit. Grain timer ticks are exempt. + if (message.BodyObject is ILocalRequest localRequest) + { + localRequest.Reject(overloadException); + return; + } } lock (_lock) diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index a569307c077..4a62131c166 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -338,13 +338,13 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } - var localRequest = message.IsLocalOnly ? message.BodyObject as ILocalRequest : null; - localRequest?.OnInvoked(response); + // Local requests report the response's exception through their tasks unless a filter failed after the callback completed. + var reportedByRequest = message.IsLocalOnly && message.BodyObject is ILocalRequest localRequest && localRequest.OnInvoked(response); if (response.Exception is { } invocationException) { - // Callers observe two-way failures, and local requests report failures through their tasks. - var logLevel = message.Direction != Message.Directions.OneWay || localRequest is not null ? LogLevel.Debug : LogLevel.Warning; + // Callers observe two-way failures. + var logLevel = message.Direction != Message.Directions.OneWay || reportedByRequest ? LogLevel.Debug : LogLevel.Warning; LogGrainInvokeException(this.invokeExceptionLogger, logLevel, invocationException, message); // If a grain allowed an inconsistent state exception to escape and the exception originated from diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Runtime/Timers/GrainRequest.cs index fdb0f75530f..58a90f43dbd 100644 --- a/src/Orleans.Runtime/Timers/GrainRequest.cs +++ b/src/Orleans.Runtime/Timers/GrainRequest.cs @@ -30,12 +30,19 @@ public GrainRequest(Func callback, TState state public ValueTask Invoke() { var cancellationToken = _registration.Token; - if (Task.IsCompleted || cancellationToken.CanBeCanceled && !_registration.Unregister()) + if (Task.IsCompleted) { // The request was canceled while it was queued. return new(Response.Completed); } + // Unregistering succeeds while cancellation is still running earlier callbacks, so check the token too. + if (cancellationToken.CanBeCanceled && (!_registration.Unregister() || cancellationToken.IsCancellationRequested)) + { + TrySetCanceled(cancellationToken); + return new(Response.Completed); + } + Task task; try { @@ -84,23 +91,29 @@ void ILocalRequest.Reject(Exception exception) TrySetException(exception); } - void ILocalRequest.OnInvoked(Response response) + bool ILocalRequest.OnInvoked(Response response) { - if (Task.IsCompleted) + var task = Task; + if (task.IsCompleted) { - return; + // The callback completed the task, so the task reports the callback's failure but not a later filter's failure. + return response.Exception is not { } exception + || task.IsCanceled && exception is OperationCanceledException + || task.IsFaulted && ReferenceEquals(task.Exception!.InnerException, exception); } // A call filter completed the invocation without running the callback. _registration.Dispose(); - if (response.Exception is { } exception) + if (response.Exception is { } filterException) { - TrySetException(exception); + TrySetException(filterException); } else { TrySetResult(response.Result is TResult result ? result : default!); } + + return true; } public object? GetTarget() => this; @@ -165,5 +178,6 @@ internal interface ILocalRequest /// /// Completes the request with the invocation's if the invocation did not run the request. /// - void OnInvoked(Response response); + /// if the request's task reports the response's exception, if any. + bool OnInvoked(Response response); } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 2e71ecf242c..1c521dafe5c 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -15,6 +15,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartValueRequest(int value); + Task> QueueRequests(int count); + Task SetRequestFilterMode(RequestFilterMode mode); Task StartRequestContextRequest(string value); @@ -53,5 +55,6 @@ public enum RequestFilterMode { Invoke, Throw, + ThrowAfterInvoke, ShortCircuit, } diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 6cb5b5465e5..21b46acc94d 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -22,6 +22,9 @@ public async Task Invoke(IIncomingGrainCallContext context) { case RequestFilterMode.Throw: throw new InvalidOperationException("Filter failed."); + case RequestFilterMode.ThrowAfterInvoke: + await context.Invoke(); + throw new InvalidOperationException("Filter failed after invoke."); case RequestFilterMode.ShortCircuit: context.Result = "filtered"; return; @@ -81,6 +84,34 @@ public Task StartValueRequest(int value) return Task.CompletedTask; } + public Task> QueueRequests(int count) + { + var outcomes = new List(count); + var admitted = new List(count); + for (var i = 0; i < count; i++) + { + var entry = $"request-{i}"; + var request = this.RunRequestAsync(static (state, _) => + { + state.Grain._log.Add(state.Entry); + return Task.CompletedTask; + }, (Grain: this, Entry: entry)); + + if (request.IsFaulted) + { + outcomes.Add(request.Exception!.InnerException!.GetType().Name); + } + else + { + outcomes.Add("queued"); + admitted.Add(request); + } + } + + _pending = Task.WhenAll(admitted); + return Task.FromResult(outcomes); + } + public Task StartRequestContextRequest(string value) { RequestContext.Set("run-request", value); diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs new file mode 100644 index 00000000000..13978fb6329 --- /dev/null +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -0,0 +1,89 @@ +using Orleans.Runtime; +using Orleans.Serialization.Invocation; +using Xunit; + +namespace NonSilo.Tests.Runtime; + +/// +/// Tests for the request which queues. +/// +[TestCategory("BVT"), TestCategory("Scheduler")] +public class GrainRequestTests +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); + + [Fact] + public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRunCallback() + { + using var cancellation = new CancellationTokenSource(); + var callbackRan = false; + var request = new GrainRequest( + (_, _) => + { + callbackRan = true; + return Task.CompletedTask; + }, + null, + cancellation.Token); + + // Cancellation runs callbacks in reverse registration order, so this one runs before the request's. + using var entered = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + using var blocker = cancellation.Token.Register(() => + { + entered.Set(); + release.Wait(Timeout); + }); + + var cancel = Task.Run(cancellation.Cancel, TestContext.Current.CancellationToken); + Assert.True(entered.Wait(Timeout, TestContext.Current.CancellationToken)); + + await request.Invoke(); + release.Set(); + await cancel; + + Assert.False(callbackRan); + await Assert.ThrowsAnyAsync(() => request.Task); + } + + [Fact] + public async Task OnInvoked_CallbackFailure_IsReportedByRequest() + { + var exception = new InvalidOperationException("Request failed."); + var request = new GrainRequest((_, _) => Task.FromException(exception), null, default); + + var response = await request.Invoke(); + + Assert.True(((ILocalRequest)request).OnInvoked(response)); + Assert.Same(exception, await Assert.ThrowsAsync(() => request.Task)); + } + + [Fact] + public async Task OnInvoked_FilterFailureAfterCallback_IsNotReportedByRequest() + { + var request = new GrainRequest((_, _) => Task.FromResult("result"), null, default); + + await request.Invoke(); + + Assert.False(((ILocalRequest)request).OnInvoked(Response.FromException(new InvalidOperationException("Filter failed.")))); + Assert.Equal("result", await request.Task); + } + + [Fact] + public async Task OnInvoked_FilterFailureWithoutCallback_IsReportedByRequest() + { + var callbackRan = false; + var request = new GrainRequest( + (_, _) => + { + callbackRan = true; + return Task.CompletedTask; + }, + null, + default); + + Assert.True(((ILocalRequest)request).OnInvoked(Response.FromException(new InvalidOperationException("Filter failed.")))); + Assert.False(callbackRan); + await Assert.ThrowsAsync(() => request.Task); + } +} diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 7638cb67449..f455c375b93 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -1,4 +1,6 @@ +using Orleans.Configuration; using Orleans.Runtime; +using Orleans.TestingHost; using TestExtensions; using UnitTests.GrainInterfaces; using UnitTests.Grains; @@ -152,6 +154,17 @@ public async Task RunRequest_FilterShortCircuits_CompletesWithFilterResult() Assert.Equal("filtered", await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); } + [Fact] + public async Task RunRequest_FilterThrowsAfterInvoke_CompletesWithCallbackResult() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.SetRequestFilterMode(RequestFilterMode.ThrowAfterInvoke); + + await grain.StartValueRequest(21); + + Assert.Equal("42", await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + [Fact] public async Task RunRequest_FromOutsideActivation_RunsOnActivation() { @@ -196,3 +209,41 @@ private static async Task WaitForLogEntryAsync(IRunRequestGrain grain, string en } } } + +/// +/// Tests that requests queued by +/// honor . +/// +[TestCategory("BVT"), TestCategory("Scheduler")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +public class RunRequestQueueLimitTests : TestClusterPerTest +{ + private const int HardLimit = 3; + + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 1; + builder.AddSiloBuilderConfigurator(); + } + + [Fact] + public async Task RunRequest_BeyondHardLimit_FaultsWithLimitExceeded() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + // The queuing call counts toward the limit, so the activation admits requests until its count exceeds the limit. + var outcomes = await grain.QueueRequests(HardLimit + 2); + await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(["queued", "queued", "queued", nameof(LimitExceededException), nameof(LimitExceededException)], outcomes); + Assert.Equal(["request-0", "request-1", "request-2"], await grain.GetLog()); + } + + private sealed class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) => + hostBuilder.Configure(options => options.MaxEnqueuedRequestsHardLimit = HardLimit); + } +} From 15872b968fc51ee632ada22028ebb01638b5bb71 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 18:51:44 -0700 Subject: [PATCH 04/31] fix(runtime): admit requests atomically against the enqueued request limit Count and enqueue under a single lock acquisition so concurrent producers cannot exceed MaxEnqueuedRequestsHardLimit. Logging and rejection run outside the lock, and grain timer ticks remain exempt. --- src/Orleans.Runtime/Catalog/ActivationData.cs | 80 ++++++++----------- .../TestGrainInterfaces/IRunRequestGrain.cs | 2 + test/Grains/TestGrains/RunRequestGrain.cs | 39 +++++++-- .../RunRequestTests.cs | 12 +++ 4 files changed, 78 insertions(+), 55 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index afa091098f8..eb06251aae4 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -481,42 +481,6 @@ private void SetState(ActivationState state) } } - /// - /// Check whether this activation is overloaded. - /// Returns LimitExceededException if overloaded, otherwise nullc> - /// - /// Returns LimitExceededException if overloaded, otherwise nullc> - public LimitExceededException? CheckOverloaded() - { - string limitName = nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit); - int maxRequestsHardLimit = _shared.MessagingOptions.MaxEnqueuedRequestsHardLimit; - int maxRequestsSoftLimit = _shared.MessagingOptions.MaxEnqueuedRequestsSoftLimit; - if (IsStatelessWorker) - { - limitName = nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker); - maxRequestsHardLimit = _shared.MessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker; - maxRequestsSoftLimit = _shared.MessagingOptions.MaxEnqueuedRequestsSoftLimit_StatelessWorker; - } - - if (maxRequestsHardLimit <= 0 && maxRequestsSoftLimit <= 0) return null; // No limits are set - - int count = GetRequestCount(); - - if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit - { - LogRejectActivationTooManyRequests(_shared.Logger, count, this, maxRequestsHardLimit); - return new LimitExceededException(limitName, count, maxRequestsHardLimit, ToString()); - } - - if (maxRequestsSoftLimit > 0 && count > maxRequestsSoftLimit) // Soft limit - { - LogWarnActivationTooManyRequests(_shared.Logger, count, this, maxRequestsSoftLimit); - return null; - } - - return null; - } - internal int GetRequestCount() { lock (_lock) @@ -1745,27 +1709,47 @@ private void ReceiveResponse(Message message) private void ReceiveRequest(Message message) { - var overloadException = CheckOverloaded(); - if (overloadException != null) + var options = _shared.MessagingOptions; + var (hardLimit, softLimit) = IsStatelessWorker + ? (options.MaxEnqueuedRequestsHardLimit_StatelessWorker, options.MaxEnqueuedRequestsSoftLimit_StatelessWorker) + : (options.MaxEnqueuedRequestsHardLimit, options.MaxEnqueuedRequestsSoftLimit); + + // Requests queued by RunRequestAsync honor the hard limit. Grain timer ticks are exempt. + var enforceHardLimit = hardLimit > 0 && (!message.IsLocalOnly || message.BodyObject is ILocalRequest); + int count; + bool admitted; + lock (_lock) { - if (!message.IsLocalOnly) + // Counting and enqueuing under one lock keeps concurrent producers within the hard limit. + count = _runningRequests.Count + _waitingRequests.Count; + admitted = !enforceHardLimit || count <= hardLimit; + if (admitted) { - _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); - _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); - return; + _waitingRequests.Add((message, CoarseStopwatch.StartNew())); } + } - // Requests queued by RunRequestAsync honor the limit. Grain timer ticks are exempt. - if (message.BodyObject is ILocalRequest localRequest) + if (!admitted) + { + LogRejectActivationTooManyRequests(_shared.Logger, count, this, hardLimit); + var limitName = IsStatelessWorker ? nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker) : nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit); + var overloadException = new LimitExceededException(limitName, count, hardLimit, ToString()); + if (message.IsLocalOnly) { - localRequest.Reject(overloadException); - return; + ((ILocalRequest)message.BodyObject!).Reject(overloadException); } + else + { + _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); + _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); + } + + return; } - lock (_lock) + if (softLimit > 0 && count > softLimit) { - _waitingRequests.Add((message, CoarseStopwatch.StartNew())); + LogWarnActivationTooManyRequests(_shared.Logger, count, this, softLimit); } _workSignal.Signal(); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 1c521dafe5c..b7bb8bc9cfa 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -17,6 +17,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task> QueueRequests(int count); + Task QueueRequestsConcurrently(int count); + Task SetRequestFilterMode(RequestFilterMode mode); Task StartRequestContextRequest(string value); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 21b46acc94d..f7e5087c9f6 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -90,13 +90,7 @@ public Task> QueueRequests(int count) var admitted = new List(count); for (var i = 0; i < count; i++) { - var entry = $"request-{i}"; - var request = this.RunRequestAsync(static (state, _) => - { - state.Grain._log.Add(state.Entry); - return Task.CompletedTask; - }, (Grain: this, Entry: entry)); - + var request = QueueLoggedRequest($"request-{i}"); if (request.IsFaulted) { outcomes.Add(request.Exception!.InnerException!.GetType().Name); @@ -112,6 +106,37 @@ public Task> QueueRequests(int count) return Task.FromResult(outcomes); } + public async Task QueueRequestsConcurrently(int count) + { + // Dedicated threads, released together, queue the requests while this call blocks the activation. + using var barrier = new Barrier(count); + var requests = await Task.WhenAll(Enumerable.Range(0, count).Select(i => Task.Factory.StartNew( + () => + { + barrier.SignalAndWait(); + return QueueLoggedRequest($"request-{i}"); + }, + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default))); + + var admitted = requests.Where(request => !request.IsFaulted).ToList(); + if (requests.FirstOrDefault(request => request.IsFaulted && request.Exception!.InnerException is not LimitExceededException) is { } unexpected) + { + await unexpected; + } + + _pending = Task.WhenAll(admitted); + return admitted.Count; + } + + private Task QueueLoggedRequest(string entry) => + this.RunRequestAsync(static (state, _) => + { + state.Grain._log.Add(state.Entry); + return Task.CompletedTask; + }, (Grain: this, Entry: entry)); + public Task StartRequestContextRequest(string value) { RequestContext.Set("run-request", value); diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index f455c375b93..07a089e0876 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -241,6 +241,18 @@ public async Task RunRequest_BeyondHardLimit_FaultsWithLimitExceeded() Assert.Equal(["request-0", "request-1", "request-2"], await grain.GetLog()); } + [Fact] + public async Task RunRequest_ConcurrentProducers_StayWithinHardLimit() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + var admitted = await grain.QueueRequestsConcurrently(64); + await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(HardLimit, admitted); + Assert.Equal(HardLimit, (await grain.GetLog()).Count); + } + private sealed class SiloConfigurator : ISiloConfigurator { public void Configure(ISiloBuilder hostBuilder) => From 260f3e5067b670fe33fb6362553c0361482be181 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 06:59:50 -0700 Subject: [PATCH 05/31] fix(runtime): keep warning when cancellation masks a call filter failure --- src/Orleans.Runtime/Timers/GrainRequest.cs | 8 +++--- .../Runtime/GrainRequestTests.cs | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Runtime/Timers/GrainRequest.cs index 58a90f43dbd..361ba10e598 100644 --- a/src/Orleans.Runtime/Timers/GrainRequest.cs +++ b/src/Orleans.Runtime/Timers/GrainRequest.cs @@ -103,16 +103,14 @@ bool ILocalRequest.OnInvoked(Response response) } // A call filter completed the invocation without running the callback. + // Cancellation can complete the task first, in which case the task does not report the filter's failure. _registration.Dispose(); if (response.Exception is { } filterException) { - TrySetException(filterException); - } - else - { - TrySetResult(response.Result is TResult result ? result : default!); + return TrySetException(filterException); } + TrySetResult(response.Result is TResult result ? result : default!); return true; } diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index 13978fb6329..d3e7b7548a2 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -86,4 +86,29 @@ public async Task OnInvoked_FilterFailureWithoutCallback_IsReportedByRequest() Assert.False(callbackRan); await Assert.ThrowsAsync(() => request.Task); } + + [Fact] + public async Task OnInvoked_FilterFailureRacingCancellation_ReportsOnlyFailuresTheTaskCarries() + { + var filterException = new InvalidOperationException("Filter failed."); + var response = Response.FromException(filterException); + for (var i = 0; i < 2_000; i++) + { + using var cancellation = new CancellationTokenSource(); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, cancellation.Token); + using var start = new Barrier(2); + var cancel = Task.Run(() => + { + start.SignalAndWait(); + cancellation.Cancel(); + }, TestContext.Current.CancellationToken); + + start.SignalAndWait(TestContext.Current.CancellationToken); + var reported = ((ILocalRequest)request).OnInvoked(response); + await cancel; + + // The request reports the filter's failure exactly when its task carries that failure. + Assert.Equal(request.Task.IsFaulted && ReferenceEquals(request.Task.Exception!.InnerException, filterException), reported); + } + } } From 94d62a5b8eaa0330619b2b42064880f679b80cbe Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 10:28:54 -0700 Subject: [PATCH 06/31] refactor(runtime): add RunRequestAsync options and share local request construction Add GrainRequestOptions to configure keep-alive and interleaving, and to name requests and describe their arguments for call filters. Unnamed requests keep their previous size; names and arguments are stored only on NamedGrainRequest. Replace ILocalRequest.OnInvoked/Reject with Complete(Response) and a default Reject helper. Share local request message construction between RunRequestAsync and grain timers through MessageFactory.CreateLocalRequest, which removes TimerRegistry's ILocalSiloDetails dependency. --- .../content/docs/grains/request-scheduling.md | 2 +- .../Core/GrainRequestOptions.cs | 77 +++++++++++++++ .../Core/IGrainBase.cs | 88 +++++++++++++++--- .../Core/IGrainRequestRunner.cs | 1 + src/Orleans.Core/Messaging/MessageFactory.cs | 22 +++++ src/Orleans.Runtime/Catalog/ActivationData.cs | 19 +--- .../Core/InsideRuntimeClient.cs | 2 +- src/Orleans.Runtime/Timers/GrainRequest.cs | 87 +++++++++++++---- src/Orleans.Runtime/Timers/GrainTimer.cs | 16 +--- src/Orleans.Runtime/Timers/TimerRegistry.cs | 3 +- .../Orleans.Core.Abstractions.cs | 23 +++++ .../TestGrainInterfaces/IRunRequestGrain.cs | 4 + test/Grains/TestGrains/RunRequestGrain.cs | 29 ++++++ .../ActivationGCTestGrainInterfaces.cs | 1 + .../ActivationGCTestGrains.cs | 6 ++ .../Runtime/GrainRequestTests.cs | 93 +++++++++++++++++-- .../Runtime/GrainTimerTests.cs | 8 +- .../RunRequestTests.cs | 23 +++++ .../ActivationCollectorTests.cs | 28 ++++++ 19 files changed, 454 insertions(+), 78 deletions(-) create mode 100644 src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index b665604be34..8f5321fb7a1 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -62,6 +62,6 @@ The scope permits callbacks associated with that call chain until it is disposed Grain timer callbacks participate in scheduling. Callbacks don't interleave by default; configure `GrainTimerCreationOptions.Interleave` when intentional. - runs a callback as a request on the activation, from any thread. The request follows the grain's reentrancy rules, passes through grain call filters, starts a new call chain, and keeps the activation alive. Use it to apply results from background work, such as stream or messaging handlers, under the same scheduling as grain calls. On a non-reentrant grain, awaiting the request from within another request on the same activation deadlocks. + runs a callback as a request on the activation, from any thread. The request follows the grain's reentrancy rules, passes through grain call filters, starts a new call chain, and keeps the activation alive. Pass to let the request interleave with other requests, to run it without restarting the idle period, or to give call filters an interface name, method name, and arguments that describe it. Use it to apply results from background work, such as stream or messaging handlers, under the same scheduling as grain calls. On a non-reentrant grain, awaiting the request from within another request on the same activation deadlocks. Use only to isolate unavoidable synchronous blocking or CPU work from the Orleans scheduler. Don't access grain state from the thread-pool delegate. Await it and update grain state after execution resumes on the activation scheduler. See [External tasks and grains](external-tasks-and-grains.md). diff --git a/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs new file mode 100644 index 00000000000..c78af916bbc --- /dev/null +++ b/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs @@ -0,0 +1,77 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Concurrency; + +namespace Orleans.Runtime; + +/// +/// Options for requests which a grain runs on its own activation using +/// . +/// +[SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "These per-call options carry a caller-owned arguments array rather than representing a comparable value.")] +public readonly struct GrainRequestOptions +{ + private readonly bool _suppressKeepAlive; + + /// + /// Initializes a new instance with the default options. + /// + public GrainRequestOptions() + { + } + + /// + /// Gets a value indicating whether the request extends the lifetime of the grain activation. + /// Defaults to . + /// + /// + /// A request which keeps the activation alive restarts the activation's idle collection period when it completes, + /// the same as a grain call. A request which does not keep the activation alive leaves the idle collection period unchanged. + /// + public bool KeepAlive + { + get => !_suppressKeepAlive; + init => _suppressKeepAlive = !value; + } + + /// + /// Gets a value indicating whether the request can interleave with other requests on the activation. + /// Defaults to . + /// + /// + /// If this value is , the request follows the grain's reentrancy rules, the same as a grain call. + /// If this value is , the request interleaves with other requests, the same as a call to a method with + /// the applied. + /// + public bool Interleave { get; init; } + + /// + /// Gets the interface name which grain call filters observe for the request. + /// Defaults to IGrainRequestInvoker. + /// + /// + /// Call filters observe this value as , and tracing uses it in the request's activity name. + /// + public string? InterfaceName { get; init; } + + /// + /// Gets the method name which grain call filters observe for the request. + /// Defaults to RunRequestAsync. + /// + /// + /// Call filters observe this value as , and tracing uses it in the request's activity name. + /// + public string? MethodName { get; init; } + + /// + /// Gets the arguments which grain call filters observe for the request. + /// Defaults to no arguments. + /// + /// + /// The arguments describe the request to call filters, which access them through . + /// The callback receives its state rather than these arguments. A call filter which sets an argument writes to this array. + /// + public object?[]? Arguments { get; init; } +} diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 7ed01ec637a..2cfab1a11ff 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -62,13 +62,28 @@ public static void DeactivateOnIdle(this IGrainBase grain) => /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, default(GrainRequestOptions), cancellationToken); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, GrainRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return GetRequestRunner(grain).RunRequestAsync, object?>( static (callback, cancellationToken) => callback(cancellationToken), callback, + options, cancellationToken); } @@ -83,11 +98,27 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, state, default(GrainRequestOptions), cancellationToken); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the parameter. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, GrainRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync(callback, state, cancellationToken); + return GetRequestRunner(grain).RunRequestAsync(callback, state, options, cancellationToken); } /// @@ -100,13 +131,29 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, default(GrainRequestOptions), cancellationToken); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the callback's result. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, GrainRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return GetRequestRunner(grain).RunRequestAsync>, TResult>( static (callback, cancellationToken) => callback(cancellationToken), callback, + options, cancellationToken); } @@ -122,12 +169,29 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// The type of the parameter. /// The type of the callback's result. /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, state, default(GrainRequestOptions), cancellationToken); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// The type of the parameter. + /// The type of the callback's result. + /// A which completes with the outcome of the callback. /// /// - /// The request is scheduled with the activation's other requests and follows the grain's reentrancy rules: + /// The request is scheduled with the activation's other requests. By default, it follows the grain's reentrancy rules: /// on a non-reentrant grain, the callback runs exclusively, the same as a grain method call. - /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync, - /// starts a new call chain, and keeps the activation alive. + /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync, or with the names and arguments set in + /// . It starts a new call chain and, by default, keeps the activation alive. /// A call filter which fails or sets the result without invoking the request completes the returned task with that outcome. /// /// @@ -142,10 +206,10 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// . /// /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, GrainRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync(callback, state, cancellationToken); + return GetRequestRunner(grain).RunRequestAsync(callback, state, options, cancellationToken); } private static IGrainRequestRunner GetRequestRunner(IGrainBase grain) => diff --git a/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs b/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs index c3bddecb079..4b216d11fc0 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs @@ -19,5 +19,6 @@ internal interface IGrainRequestRunner Task RunRequestAsync( Func callback, TState state, + in GrainRequestOptions options, CancellationToken cancellationToken); } diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index 25f2fdfc282..935f0e79103 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -52,6 +52,28 @@ private CorrelationId GetNextCorrelationId() return new CorrelationId(unchecked((long)id)); } + /// + /// Creates a one-way request which queues on itself. + /// + /// + /// The request starts a new call chain, never expires, and completes on or is rejected with it. + /// + public Message CreateLocalRequest(IGrainContext target, object body, GrainInterfaceType interfaceType, bool keepAlive, bool interleave) + { + var siloAddress = target.Address.SiloAddress; + var message = CreateMessage(body, InvokeMethodOptions.OneWay, requestContextData: null); + message.SetInfiniteTimeToLive(); + message.SendingGrain = target.GrainId; + message.TargetGrain = target.GrainId; + message.SendingSilo = siloAddress; + message.TargetSilo = siloAddress; + message.InterfaceType = interfaceType; + message.IsKeepAlive = keepAlive; + message.IsAlwaysInterleave = interleave; + message.IsLocalOnly = true; + return message; + } + public Message CreateResponseMessage(Message request) { var response = new Message diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index eb06251aae4..8bd72d63402 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -574,28 +574,15 @@ void IActivationKeepAlive.ReleaseKeepAlive() } } - Task IGrainRequestRunner.RunRequestAsync(Func callback, TState state, CancellationToken cancellationToken) + Task IGrainRequestRunner.RunRequestAsync(Func callback, TState state, in GrainRequestOptions options, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return Task.FromCanceled(cancellationToken); } - var request = new GrainRequest(callback, state, cancellationToken); - - // Requests start a new call chain, matching timer ticks. - var message = _shared.InternalRuntime.MessageFactory.CreateMessage(body: request, options: CodeGeneration.InvokeMethodOptions.OneWay, requestContextData: null); - message.SetInfiniteTimeToLive(); - message.SendingGrain = GrainId; - message.TargetGrain = GrainId; - message.SendingSilo = Address.SiloAddress; - message.TargetSilo = Address.SiloAddress; - message.InterfaceType = IGrainRequestInvoker.InterfaceType; - message.IsKeepAlive = true; - - // The request completes on this activation or is rejected with it. - message.IsLocalOnly = true; - + var request = GrainRequest.Create(callback, state, options, cancellationToken); + var message = _shared.InternalRuntime.MessageFactory.CreateLocalRequest(this, request, IGrainRequestInvoker.InterfaceType, options.KeepAlive, options.Interleave); ReceiveMessage(message); return request.Task; } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 4a62131c166..66f9c196e08 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -339,7 +339,7 @@ public async Task Invoke(IGrainContext target, Message message) } // Local requests report the response's exception through their tasks unless a filter failed after the callback completed. - var reportedByRequest = message.IsLocalOnly && message.BodyObject is ILocalRequest localRequest && localRequest.OnInvoked(response); + var reportedByRequest = message.IsLocalOnly && message.BodyObject is ILocalRequest localRequest && localRequest.Complete(response); if (response.Exception is { } invocationException) { diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Runtime/Timers/GrainRequest.cs index 361ba10e598..be25a46dfb6 100644 --- a/src/Orleans.Runtime/Timers/GrainRequest.cs +++ b/src/Orleans.Runtime/Timers/GrainRequest.cs @@ -10,8 +10,10 @@ namespace Orleans.Runtime; /// The request is its own completion source, so queuing a request allocates only this object and its message. /// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. /// When a call filter completes the invocation without running the callback, the task reflects the filter's outcome. +/// Call filters observe the request as unless it is created with +/// and named by . /// -internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest +internal class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest { private readonly Func _callback; private readonly TState _state; @@ -27,6 +29,18 @@ public GrainRequest(Func callback, TState state this); } + /// + /// Creates a request, allocating storage for the names and arguments in only when they are set. + /// + public static GrainRequest Create( + Func callback, + TState state, + in GrainRequestOptions options, + CancellationToken cancellationToken) => + options.InterfaceName is null && options.MethodName is null && options.Arguments is null + ? new GrainRequest(callback, state, cancellationToken) + : new NamedGrainRequest(callback, state, options, cancellationToken); + public ValueTask Invoke() { var cancellationToken = _registration.Token; @@ -85,13 +99,7 @@ private async ValueTask AwaitCallbackAsync(Task task) private static TResult GetResult(Task task) => task is Task typed ? typed.Result : default!; - void ILocalRequest.Reject(Exception exception) - { - _registration.Dispose(); - TrySetException(exception); - } - - bool ILocalRequest.OnInvoked(Response response) + bool ILocalRequest.Complete(Response response) { var task = Task; if (task.IsCompleted) @@ -123,17 +131,17 @@ public void SetTarget(ITargetHolder holder) // Declared for call filters, which inspect the interface and implementation methods. Task IGrainRequestInvoker.RunRequestAsync(CancellationToken cancellationToken) => throw new InvalidOperationException(); - public int GetArgumentCount() => 0; + public virtual int GetArgumentCount() => 0; - public object? GetArgument(int index) => throw new InvalidOperationException(); + public virtual object? GetArgument(int index) => throw new ArgumentOutOfRangeException(nameof(index)); - public void SetArgument(int index, object? value) => throw new InvalidOperationException(); + public virtual void SetArgument(int index, object? value) => throw new ArgumentOutOfRangeException(nameof(index)); - public string GetMethodName() => nameof(IGrainRequestInvoker.RunRequestAsync); + public virtual string GetMethodName() => nameof(IGrainRequestInvoker.RunRequestAsync); - public string GetInterfaceName() => nameof(IGrainRequestInvoker); + public virtual string GetInterfaceName() => nameof(IGrainRequestInvoker); - public string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{nameof(IGrainRequestInvoker.RunRequestAsync)}"; + public virtual string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{nameof(IGrainRequestInvoker.RunRequestAsync)}"; public MethodInfo GetMethod() => IGrainRequestInvoker.MethodInfo; @@ -148,6 +156,47 @@ public void Dispose() public override string ToString() => $"Request '{_callback.Target}.{_callback.Method}'"; } +/// +/// A which describes itself to call filters using the names and arguments +/// in . Each unset name falls back to 's. +/// +internal sealed class NamedGrainRequest : GrainRequest +{ + private readonly string? _interfaceName; + private readonly string? _methodName; + private readonly object?[]? _arguments; + + public NamedGrainRequest(Func callback, TState state, in GrainRequestOptions options, CancellationToken cancellationToken) + : base(callback, state, cancellationToken) + { + _interfaceName = options.InterfaceName; + _methodName = options.MethodName; + _arguments = options.Arguments; + } + + public override int GetArgumentCount() => _arguments?.Length ?? 0; + + public override object? GetArgument(int index) => _arguments is { } arguments ? arguments[index] : base.GetArgument(index); + + public override void SetArgument(int index, object? value) + { + if (_arguments is { } arguments) + { + arguments[index] = value; + } + else + { + base.SetArgument(index, value); + } + } + + public override string GetMethodName() => _methodName ?? base.GetMethodName(); + + public override string GetInterfaceName() => _interfaceName ?? base.GetInterfaceName(); + + public override string GetActivityName() => $"{GetInterfaceName()}/{GetMethodName()}"; +} + /// /// The interface which call filters observe for requests queued by . /// @@ -169,13 +218,13 @@ internal interface IGrainRequestInvoker : IAddressable internal interface ILocalRequest { /// - /// Completes the request with . + /// Completes the request with if the request has not already completed. /// - void Reject(Exception exception); + /// if the request's task reports the response's exception, if any. + bool Complete(Response response); /// - /// Completes the request with the invocation's if the invocation did not run the request. + /// Completes the request with if the request has not already completed. /// - /// if the request's task reports the response's exception, if any. - bool OnInvoked(Response response); + void Reject(Exception exception) => Complete(Response.FromException(exception)); } diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index c707fdc10d4..6deacb665dc 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -2,7 +2,6 @@ using System.Diagnostics.CodeAnalysis; using System.Reflection; using Microsoft.Extensions.Logging; -using Orleans.CodeGeneration; using Orleans.Runtime.Internal; using Orleans.Serialization.Invocation; using Orleans.Timers; @@ -107,20 +106,7 @@ private void QueueTickOnActivation() { try { - // Timer requests start a new call chain, including ticks queued immediately during registration. - var msg = _shared.MessageFactory.CreateMessage(body: _invoker, options: InvokeMethodOptions.OneWay, requestContextData: null); - msg.SetInfiniteTimeToLive(); - msg.SendingGrain = _grainContext.GrainId; - msg.TargetGrain = _grainContext.GrainId; - msg.SendingSilo = _shared.LocalSiloDetails.SiloAddress; - msg.TargetSilo = _shared.LocalSiloDetails.SiloAddress; - msg.InterfaceType = InvokableInterfaceType; - msg.IsKeepAlive = _keepAlive; - msg.IsAlwaysInterleave = _interleave; - - // Prevent the message from being forwarded in the case of deactivation. - msg.IsLocalOnly = true; - + var msg = _shared.MessageFactory.CreateLocalRequest(_grainContext, _invoker, InvokableInterfaceType, _keepAlive, _interleave); _grainContext.ReceiveMessage(msg); } catch (Exception exception) diff --git a/src/Orleans.Runtime/Timers/TimerRegistry.cs b/src/Orleans.Runtime/Timers/TimerRegistry.cs index d7f5f26c38a..f7183a04eff 100644 --- a/src/Orleans.Runtime/Timers/TimerRegistry.cs +++ b/src/Orleans.Runtime/Timers/TimerRegistry.cs @@ -5,12 +5,11 @@ namespace Orleans.Timers; -internal class TimerRegistry(ILoggerFactory loggerFactory, [FromKeyedServices(TimeProviderNames.Grains)] TimeProvider timeProvider, MessageFactory messageFactory, ILocalSiloDetails localSiloDetails) : ITimerRegistry +internal class TimerRegistry(ILoggerFactory loggerFactory, [FromKeyedServices(TimeProviderNames.Grains)] TimeProvider timeProvider, MessageFactory messageFactory) : ITimerRegistry { public ILogger TimerLogger { get; } = loggerFactory.CreateLogger(); public TimeProvider TimeProvider { get; } = timeProvider; public MessageFactory MessageFactory { get; } = messageFactory; - public ILocalSiloDetails LocalSiloDetails { get; } = localSiloDetails; public IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period) { diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 5b7e6cec207..5b7569af37f 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -1119,12 +1119,20 @@ public static void MigrateOnIdle(this IGrainBase grain) { } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, System.TimeSpan dueTime, System.TimeSpan period) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } } @@ -2353,6 +2361,21 @@ public GrainReferenceShared(GrainType grainType, GrainInterfaceType grainInterfa public System.IServiceProvider ServiceProvider { get { throw null; } } } + public readonly partial struct GrainRequestOptions + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public object?[]? Arguments { get { throw null; } init { } } + + public string? InterfaceName { get { throw null; } init { } } + + public bool Interleave { get { throw null; } init { } } + + public bool KeepAlive { get { throw null; } init { } } + + public string? MethodName { get { throw null; } init { } } + } + public readonly partial struct GrainTimerCreationOptions : System.IEquatable { private readonly int _dummyPrimitive; diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index b7bb8bc9cfa..1eac2575772 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -6,6 +6,10 @@ public interface IRunRequestGrain : IGrainWithGuidKey { Task StartRequestDuringCall(); + Task StartInterleavingRequestDuringCall(); + + Task StartNamedRequest(int orderId); + Task StartBlockingRequest(); [AlwaysInterleave] diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index f7e5087c9f6..6af014449d2 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -13,6 +13,8 @@ public abstract class RunRequestGrainBase : Grain, IRunRequestGrain, IIncomingGr private Task _pending = Task.CompletedTask; private RequestFilterMode _requestFilterMode; + public const string NamedRequestInterfaceName = "IOrderProcessor"; + public async Task Invoke(IIncomingGrainCallContext context) { _filteredCalls.Add($"{context.InterfaceName}.{context.MethodName}"); @@ -31,6 +33,11 @@ public async Task Invoke(IIncomingGrainCallContext context) } } + else if (context.InterfaceName == NamedRequestInterfaceName) + { + _log.Add($"argument:{context.Request.GetArgument(0)}"); + } + await context.Invoke(); } @@ -53,6 +60,28 @@ public async Task StartRequestDuringCall() _log.Add("call-end"); } + public async Task StartInterleavingRequestDuringCall() + { + _log.Add("call-start"); + _pending = this.RunRequestAsync(_ => + { + _log.Add("request"); + return Task.CompletedTask; + }, new GrainRequestOptions { Interleave = true }); + + await Task.Delay(200); + _log.Add("call-end"); + } + + public Task StartNamedRequest(int orderId) + { + _pending = this.RunRequestAsync( + static (orderId, _) => Task.FromResult($"order-{orderId}"), + orderId, + new GrainRequestOptions { InterfaceName = NamedRequestInterfaceName, MethodName = "ProcessOrder", Arguments = [orderId] }); + return Task.CompletedTask; + } + public Task StartBlockingRequest() { var gate = _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs b/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs index 5ec8e7e19b8..498e99e7293 100644 --- a/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs +++ b/test/Grains/TestInternalGrainInterfaces/ActivationGCTestGrainInterfaces.cs @@ -46,5 +46,6 @@ public interface IKeepAliveActivationGcTestGrain : IGrainWithGuidKey Task CancelKeepAlive(); Task AcquireKeepAliveHold(); Task ReleaseKeepAliveHold(); + Task QueueRequest(TimeSpan duration, bool keepAlive); } } diff --git a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs index eef58398611..54fa280ef38 100644 --- a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs +++ b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs @@ -200,5 +200,11 @@ public Task ReleaseKeepAliveHold() ((IActivationKeepAlive)GrainContext).ReleaseKeepAlive(); return Task.CompletedTask; } + + public Task QueueRequest(TimeSpan duration, bool keepAlive) + { + _ = this.RunRequestAsync(static (duration, cancellationToken) => Task.Delay(duration, cancellationToken), duration, new GrainRequestOptions { KeepAlive = keepAlive }); + return Task.CompletedTask; + } } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index d3e7b7548a2..f83c98ba2b3 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -47,30 +47,30 @@ public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRun } [Fact] - public async Task OnInvoked_CallbackFailure_IsReportedByRequest() + public async Task Complete_CallbackFailure_IsReportedByRequest() { var exception = new InvalidOperationException("Request failed."); var request = new GrainRequest((_, _) => Task.FromException(exception), null, default); var response = await request.Invoke(); - Assert.True(((ILocalRequest)request).OnInvoked(response)); + Assert.True(((ILocalRequest)request).Complete(response)); Assert.Same(exception, await Assert.ThrowsAsync(() => request.Task)); } [Fact] - public async Task OnInvoked_FilterFailureAfterCallback_IsNotReportedByRequest() + public async Task Complete_FilterFailureAfterCallback_IsNotReportedByRequest() { var request = new GrainRequest((_, _) => Task.FromResult("result"), null, default); await request.Invoke(); - Assert.False(((ILocalRequest)request).OnInvoked(Response.FromException(new InvalidOperationException("Filter failed.")))); + Assert.False(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); Assert.Equal("result", await request.Task); } [Fact] - public async Task OnInvoked_FilterFailureWithoutCallback_IsReportedByRequest() + public async Task Complete_FilterFailureWithoutCallback_IsReportedByRequest() { var callbackRan = false; var request = new GrainRequest( @@ -82,13 +82,13 @@ public async Task OnInvoked_FilterFailureWithoutCallback_IsReportedByRequest() null, default); - Assert.True(((ILocalRequest)request).OnInvoked(Response.FromException(new InvalidOperationException("Filter failed.")))); + Assert.True(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); Assert.False(callbackRan); await Assert.ThrowsAsync(() => request.Task); } [Fact] - public async Task OnInvoked_FilterFailureRacingCancellation_ReportsOnlyFailuresTheTaskCarries() + public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTheTaskCarries() { var filterException = new InvalidOperationException("Filter failed."); var response = Response.FromException(filterException); @@ -104,11 +104,88 @@ public async Task OnInvoked_FilterFailureRacingCancellation_ReportsOnlyFailuresT }, TestContext.Current.CancellationToken); start.SignalAndWait(TestContext.Current.CancellationToken); - var reported = ((ILocalRequest)request).OnInvoked(response); + var reported = ((ILocalRequest)request).Complete(response); await cancel; // The request reports the filter's failure exactly when its task carries that failure. Assert.Equal(request.Task.IsFaulted && ReferenceEquals(request.Task.Exception!.InnerException, filterException), reported); } } + + [Fact] + public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() + { + using var cancellation = new CancellationTokenSource(); + var callbackRan = false; + var request = new GrainRequest( + (_, _) => + { + callbackRan = true; + return Task.CompletedTask; + }, + null, + cancellation.Token); + var exception = new OrleansMessageRejectionException("Rejected."); + + ((ILocalRequest)request).Reject(exception); + cancellation.Cancel(); + await request.Invoke(); + + Assert.False(callbackRan); + Assert.Same(exception, await Assert.ThrowsAsync(() => request.Task)); + } + + [Fact] + public void Create_WithoutNames_DescribesRequestAsGrainRequestInvoker() + { + var request = GrainRequest.Create((_, _) => Task.CompletedTask, null, new GrainRequestOptions { KeepAlive = false, Interleave = true }, TestContext.Current.CancellationToken); + + Assert.IsNotType>(request); + Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); + Assert.Equal("RunRequestAsync", request.GetMethodName()); + Assert.Equal("IGrainRequestInvoker/RunRequestAsync", request.GetActivityName()); + Assert.Equal(0, request.GetArgumentCount()); + Assert.Throws(() => request.GetArgument(0)); + } + + [Fact] + public void Create_WithNamesAndArguments_DescribesRequestWithThem() + { + var arguments = new object?[] { 1, "two" }; + var request = GrainRequest.Create( + (_, _) => Task.CompletedTask, + null, + new GrainRequestOptions { InterfaceName = "IOrderProcessor", MethodName = "ProcessOrder", Arguments = arguments }, + TestContext.Current.CancellationToken); + + Assert.Equal("IOrderProcessor", request.GetInterfaceName()); + Assert.Equal("ProcessOrder", request.GetMethodName()); + Assert.Equal("IOrderProcessor/ProcessOrder", request.GetActivityName()); + Assert.Equal(2, request.GetArgumentCount()); + Assert.Equal(1, request.GetArgument(0)); + Assert.Equal("two", request.GetArgument(1)); + + request.SetArgument(1, "three"); + Assert.Equal("three", arguments[1]); + } + + [Fact] + public void Create_WithMethodNameOnly_FallsBackToDefaultInterfaceName() + { + var request = GrainRequest.Create((_, _) => Task.CompletedTask, null, new GrainRequestOptions { MethodName = "Drain" }, TestContext.Current.CancellationToken); + + Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); + Assert.Equal("Drain", request.GetMethodName()); + Assert.Equal("IGrainRequestInvoker/Drain", request.GetActivityName()); + Assert.Equal(0, request.GetArgumentCount()); + } + + [Fact] + public void GrainRequestOptions_Default_KeepsActivationAlive() + { + Assert.True(default(GrainRequestOptions).KeepAlive); + Assert.True(new GrainRequestOptions().KeepAlive); + Assert.False(new GrainRequestOptions { KeepAlive = false }.KeepAlive); + Assert.False(default(GrainRequestOptions).Interleave); + } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 9a0f8970423..30ea749f102 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -498,16 +498,16 @@ public TimerFixture() _logger.IsEnabled(LogLevel.Error).Returns(true); var loggerFactory = Substitute.For(); loggerFactory.CreateLogger(Arg.Any()).Returns(_logger); - Grain.GrainId.Returns(GrainId.Create("timer-test", "one")); + var grainId = GrainId.Create("timer-test", "one"); + Grain.GrainId.Returns(grainId); Grain.When(context => context.ReceiveMessage(Arg.Any())).Do(call => { BeforeReceive?.Invoke(); Messages.Add((Message)call[0]); }); - var details = Substitute.For(); - details.SiloAddress.Returns(SiloAddress.New(System.Net.IPAddress.Loopback, 11111, 1)); + Grain.Address.Returns(GrainAddress.GetAddress(SiloAddress.New(System.Net.IPAddress.Loopback, 11111, 1), grainId, ActivationId.NewId())); MessageFactory = new(_services.GetRequiredService(), NullLogger.Instance, null!); - Registry = new(loggerFactory, Time, MessageFactory, details); + Registry = new(loggerFactory, Time, MessageFactory); _subscription = GrainTimerEvents.AllEvents.Subscribe(this); } diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 07a089e0876..06a752b3f9e 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -131,6 +131,29 @@ public async Task RunRequest_PassesThroughGrainCallFilter() Assert.Contains("IGrainRequestInvoker.RunRequestAsync", await grain.GetFilteredCalls()); } + [Fact] + public async Task NamedRequest_DescribesItselfToGrainCallFilter() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartNamedRequest(42); + + Assert.Equal("order-42", await grain.AwaitPendingRequest()); + Assert.Contains($"{RunRequestGrainBase.NamedRequestInterfaceName}.ProcessOrder", await grain.GetFilteredCalls()); + Assert.Equal(["argument:42"], await grain.GetLog()); + } + + [Fact] + public async Task NonReentrantGrain_InterleavingRequestRunsDuringCurrentCall() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartInterleavingRequestDuringCall(); + await grain.AwaitPendingRequest(); + + Assert.Equal(["call-start", "request", "call-end"], await grain.GetLog()); + } + [Fact] public async Task RunRequest_FilterThrows_FaultsRequest() { diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs index 4ac0c474ec8..9b394371638 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs @@ -816,6 +816,34 @@ public async Task ActivationCollectorShouldRestartIdlePeriodWhenLastKeepAliveHol await WaitForActivationCountToConverge(fullGrainTypeName, expectedCount: 0, cancellationToken); } + [TestSuite("Functional")] + [TestProvider("None")] + [Theory, TestCategory("ActivationCollector"), TestCategory("Functional")] + [InlineData(true)] + [InlineData(false)] + public async Task RunRequestKeepAliveOptionControlsIdlePeriod(bool keepAlive) + { + var cancellationToken = TestContext.Current.CancellationToken; + await Initialize(DEFAULT_IDLE_TIMEOUT, cancellationToken); + + var fullGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(KeepAliveActivationGcTestGrain)); + var grain = this.testCluster.GrainFactory!.GetGrain(Guid.NewGuid()); + var requestDuration = TimeSpan.FromSeconds(2); + await grain.QueueRequest(requestDuration, keepAlive); + + var directory = ((InProcessSiloHandle)this.testCluster.Primary!).SiloHost.Services.GetRequiredService(); + var activation = Assert.IsType(directory.FindTarget(((GrainReference)grain).GrainId)); + while (!activation.IsInactive) + { + await Task.Delay(10, cancellationToken); + } + + // The idle period began when the grain call completed, before the request ran, unless the request restarted it. + var mgmtGrain = this.testCluster.GrainFactory!.GetGrain(0); + await mgmtGrain.ForceActivationCollection(TimeSpan.FromSeconds(1), cancellationToken); + Assert.Equal(keepAlive ? 1 : 0, await TestUtils.GetActivationCount(this.testCluster.GrainFactory!, fullGrainTypeName)); + } + [TestSuite("SlowBVT")] [TestProvider("None")] [Fact, TestCategory("SlowBVT"), TestCategory("Timers")] From dbbcb5829c515f6723c105dba9e449247838ecc7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 10:48:18 -0700 Subject: [PATCH 07/31] docs: add RunRequestAsync usage examples --- .../docs/grains/external-tasks-and-grains.md | 4 +- .../content/docs/grains/request-scheduling.md | 35 ++++- docs/site/src/content/docs/grains/timers.md | 4 +- .../Grains/RequestsAndVersioningSnippets.cs | 147 ++++++++++++++++++ 4 files changed, 185 insertions(+), 5 deletions(-) diff --git a/docs/site/src/content/docs/grains/external-tasks-and-grains.md b/docs/site/src/content/docs/grains/external-tasks-and-grains.md index e8c0bbcc090..7003cbc494a 100644 --- a/docs/site/src/content/docs/grains/external-tasks-and-grains.md +++ b/docs/site/src/content/docs/grains/external-tasks-and-grains.md @@ -1,7 +1,7 @@ --- title: External tasks and grain scheduling description: Safely use asynchronous libraries, CPU work, and blocking APIs from Orleans grains. -ms.date: 08/21/2026 +ms.date: 10/15/2026 ms.topic: concept-article --- @@ -41,6 +41,8 @@ Don't use `ConfigureAwait(false)` directly in grain methods. It can resume the c If an advanced integration passes an async delegate to , unwrap the nested task: :::code language="csharp" source="../snippets/compiled/Grains/GeneralSnippets.cs" id="start_async_worker"::: +To apply the result of a background callback to grain state, use . It runs the callback as a request on the activation. See [Run background work as a grain request](request-scheduling.md#run-background-work-as-a-grain-request). + Keep background work under an observed runtime mechanism. Use [grain timers](timers.md), [reminders](reminders.md), a durable job abstraction, or a hosted service according to the required lifetime and reliability. ## Reentrancy still applies diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index 8f5321fb7a1..5d980c16b86 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -1,7 +1,7 @@ --- title: Grain request scheduling description: Understand Orleans turn-based execution, interleaving, and reentrancy. -ms.date: 08/02/2026 +ms.date: 10/15/2026 ms.topic: concept-article --- @@ -62,6 +62,37 @@ The scope permits callbacks associated with that call chain until it is disposed Grain timer callbacks participate in scheduling. Callbacks don't interleave by default; configure `GrainTimerCreationOptions.Interleave` when intentional. - runs a callback as a request on the activation, from any thread. The request follows the grain's reentrancy rules, passes through grain call filters, starts a new call chain, and keeps the activation alive. Pass to let the request interleave with other requests, to run it without restarting the idle period, or to give call filters an interface name, method name, and arguments that describe it. Use it to apply results from background work, such as stream or messaging handlers, under the same scheduling as grain calls. On a non-reentrant grain, awaiting the request from within another request on the same activation deadlocks. +To run work from another thread under the activation's scheduling, see [Run background work as a grain request](#run-background-work-as-a-grain-request). Use only to isolate unavoidable synchronous blocking or CPU work from the Orleans scheduler. Don't access grain state from the thread-pool delegate. Await it and update grain state after execution resumes on the activation scheduler. See [External tasks and grains](external-tasks-and-grains.md). + +## Run background work as a grain request + + runs a callback as a request on the activation. Call it from any thread, such as a stream, messaging, or library callback, to apply results to grain state under the same scheduling as grain calls: + +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_from_callback"::: +The request: + +- Follows the grain's reentrancy rules and queues behind the current request on a non-reentrant grain. +- Passes through incoming grain call filters. By default, filters see it as `IGrainRequestInvoker.RunRequestAsync` with no arguments. +- Starts a new call chain. values from the calling thread don't flow into the callback. +- Restarts the activation's idle collection period. + +The returned task completes with the callback's result or exception. The callback receives the cancellation token passed to `RunRequestAsync`, and cancelling it while the request is queued cancels the request. If the activation deactivates before the request runs, the task faults with . + +Pass the grain and other values through the `state` argument and use a `static` lambda, so the call doesn't allocate a closure. On a non-reentrant grain, awaiting the returned task from another request on the same activation deadlocks, because the queued request waits for the current one to complete. + +### Configure the request + +Pass to change how the request is scheduled. Set to `false` for frequent background work that shouldn't keep an otherwise idle activation in memory. Set to `true` to run the request between the turns of other requests, as with an `[AlwaysInterleave]` method: + +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_options"::: +### Describe the request to call filters + +Set , , and to identify the work in call filters, logs, and traces. Tracing names the request's activity `InterfaceName/MethodName`. The callback returns a result through the returned task: + +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_named"::: +Call filters read the description through the call context: + +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_filter"::: +The arguments describe the request to filters; the callback receives its `state`. A filter that calls `SetArgument` writes into the array assigned to `Arguments`. diff --git a/docs/site/src/content/docs/grains/timers.md b/docs/site/src/content/docs/grains/timers.md index 2efa9eb20a5..350f5b366d8 100644 --- a/docs/site/src/content/docs/grains/timers.md +++ b/docs/site/src/content/docs/grains/timers.md @@ -1,7 +1,7 @@ --- title: Grain timers description: Schedule activation-scoped periodic work as Orleans grain turns. -ms.date: 08/21/2026 +ms.date: 10/15/2026 ms.topic: concept-article --- @@ -11,7 +11,7 @@ A grain timer schedules periodic work for one grain activation. Each callback ex Use a grain timer for frequent work whose lifetime and state belong to the current activation. Use a [reminder](reminders.md) when the schedule belongs to the logical grain and must survive activation or cluster lifecycle changes. -To run a callback once as a grain request, use . See [Request scheduling](request-scheduling.md#timers-and-external-work). +To run a callback once as a grain request, use . See [Request scheduling](request-scheduling.md#run-background-work-as-a-grain-request). ## Register a grain timer diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index b92cb0dbca5..2fcb38a7a52 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -225,3 +225,150 @@ public async Task JoinRoom(string roomName) // } } + +namespace Documentation.Grains.RequestScheduling.BackgroundWork +{ + public interface IPriceFeed + { + IDisposable Subscribe(string symbol, Func onPrice); + } + + public interface IQuoteGrain : IGrainWithStringKey + { + ValueTask GetPrice(); + } + + // +public sealed class QuoteGrain(IPriceFeed feed) : Grain, IQuoteGrain +{ + private IDisposable? _subscription; + private decimal _price; + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + // The feed invokes OnPrice on its own threads. + _subscription = feed.Subscribe(this.GetPrimaryKeyString(), OnPrice); + return Task.CompletedTask; + } + + public override Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) + { + _subscription?.Dispose(); + return Task.CompletedTask; + } + + public ValueTask GetPrice() => ValueTask.FromResult(_price); + + // Runs the update as a request on this activation, where it can safely access grain state. + private Task OnPrice(decimal price, CancellationToken cancellationToken) => + this.RunRequestAsync( + static (state, _) => + { + state.Grain._price = state.Price; + return Task.CompletedTask; + }, + (Grain: this, Price: price), + cancellationToken); +} + // + + public interface IConnectionMonitor + { + IDisposable Subscribe(string sessionId, Func onHeartbeat); + } + + public interface ISessionGrain : IGrainWithStringKey + { + ValueTask GetLastSeen(); + } + + public sealed class SessionGrain(IConnectionMonitor monitor) : Grain, ISessionGrain + { + private IDisposable? _subscription; + private DateTimeOffset _lastSeen; + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + _subscription = monitor.Subscribe(this.GetPrimaryKeyString(), OnHeartbeat); + return Task.CompletedTask; + } + + public override Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) + { + _subscription?.Dispose(); + return Task.CompletedTask; + } + + public ValueTask GetLastSeen() => ValueTask.FromResult(_lastSeen); + + // +// Heartbeats don't extend the activation's lifetime, and they run between the turns of other requests. +private Task OnHeartbeat(DateTimeOffset timestamp, CancellationToken cancellationToken) => + this.RunRequestAsync( + static (state, _) => + { + state.Grain._lastSeen = state.Timestamp; + return Task.CompletedTask; + }, + (Grain: this, Timestamp: timestamp), + new GrainRequestOptions { KeepAlive = false, Interleave = true }, + cancellationToken); + // + } + + [GenerateSerializer] + public sealed record Order([property: Id(0)] string Id, [property: Id(1)] decimal Total); + + public interface IOrderBookGrain : IGrainWithStringKey + { + ValueTask GetTotal(); + } + + public sealed class OrderBookGrain : Grain, IOrderBookGrain + { + private decimal _total; + + public ValueTask GetTotal() => ValueTask.FromResult(_total); + + private int Apply(IReadOnlyList orders) + { + foreach (var order in orders) + { + _total += order.Total; + } + + return orders.Count; + } + + // +// Call filters and tracing observe this request as IOrderFeed.ApplyOrders with one argument. +internal Task ApplyOrders(IReadOnlyList orders, CancellationToken cancellationToken) => + this.RunRequestAsync( + static (state, _) => Task.FromResult(state.Grain.Apply(state.Orders)), + (Grain: this, Orders: orders), + new GrainRequestOptions + { + InterfaceName = "IOrderFeed", + MethodName = "ApplyOrders", + Arguments = [orders.Count], + }, + cancellationToken); + // + } + + // +public sealed class RequestLoggingFilter(ILogger logger) : IIncomingGrainCallFilter +{ + public async Task Invoke(IIncomingGrainCallContext context) + { + logger.LogInformation( + "Running {InterfaceName}.{MethodName} with {ArgumentCount} arguments", + context.InterfaceName, + context.MethodName, + context.Request.GetArgumentCount()); + + await context.Invoke(); + } +} + // +} From e0290ade6da062fdee6ff76ac32e8510933230f1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 12:19:29 -0700 Subject: [PATCH 08/31] refactor(runtime): add IGrainContext.ReceiveLocalRequest and LocalRequestOptions Replace IGrainRequestRunner and GrainRequestOptions with a public IGrainContext.ReceiveLocalRequest primitive and LocalRequestOptions (KeepAlive, Interleave, EnforceQueueLimit). Grain timers deliver ticks through the same primitive. RunRequestAsync names requests after a method-group callback or an explicit methodName, and local requests bypass the activation queue limit unless EnforceQueueLimit is set. --- .../content/docs/grains/request-scheduling.md | 20 ++- .../Grains/RequestsAndVersioningSnippets.cs | 88 ++++++++++-- .../Core/GrainRequestOptions.cs | 77 ----------- .../Core/IGrainBase.cs | 124 ++++++++++++----- .../Core/IGrainContext.cs | 27 ++++ .../Core/IGrainRequestRunner.cs | 24 ---- .../Core/LocalRequestOptions.cs | 58 ++++++++ .../Runtime}/GrainRequest.cs | 91 +++---------- src/Orleans.Core/Messaging/MessageFactory.cs | 6 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 30 ++--- .../Core/InsideRuntimeClient.cs | 21 ++- .../Core/InternalGrainRuntime.cs | 4 +- .../Core/LocalRequestCompletion.cs | 55 ++++++++ src/Orleans.Runtime/Core/SystemTarget.cs | 4 + src/Orleans.Runtime/OrleansContracts.txt | 3 - src/Orleans.Runtime/Timers/GrainTimer.cs | 10 +- src/Orleans.Runtime/Timers/TimerRegistry.cs | 3 +- .../Orleans.Core.Abstractions.cs | 42 +++--- src/api/Orleans.Runtime/Orleans.Runtime.cs | 2 + .../TestGrainInterfaces/IRunRequestGrain.cs | 6 + test/Grains/TestGrains/RunRequestGrain.cs | 78 +++++++++-- .../ActivationGCTestGrains.cs | 2 +- .../Runtime/GrainRequestTests.cs | 63 ++++----- .../Runtime/GrainTimerTests.cs | 125 +++++++++++------- .../RunRequestTests.cs | 45 ++++++- 25 files changed, 625 insertions(+), 383 deletions(-) delete mode 100644 src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs delete mode 100644 src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs create mode 100644 src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs rename src/{Orleans.Runtime/Timers => Orleans.Core.Abstractions/Runtime}/GrainRequest.cs (62%) create mode 100644 src/Orleans.Runtime/Core/LocalRequestCompletion.cs diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index 5d980c16b86..ff04408564e 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -74,9 +74,10 @@ Use only to isolate unavoidable synchron The request: - Follows the grain's reentrancy rules and queues behind the current request on a non-reentrant grain. -- Passes through incoming grain call filters. By default, filters see it as `IGrainRequestInvoker.RunRequestAsync` with no arguments. +- Passes through incoming grain call filters, which see it as a method of `IGrainRequestInvoker` with no arguments. - Starts a new call chain. values from the calling thread don't flow into the callback. - Restarts the activation's idle collection period. +- Is admitted regardless of , like a grain timer tick. The returned task completes with the callback's result or exception. The callback receives the cancellation token passed to `RunRequestAsync`, and cancelling it while the request is queued cancels the request. If the activation deactivates before the request runs, the task faults with . @@ -84,15 +85,22 @@ Pass the grain and other values through the `state` argument and use a `static` ### Configure the request -Pass to change how the request is scheduled. Set to `false` for frequent background work that shouldn't keep an otherwise idle activation in memory. Set to `true` to run the request between the turns of other requests, as with an `[AlwaysInterleave]` method: +Pass to change how the request is scheduled. Set to `false` for frequent background work that shouldn't keep an otherwise idle activation in memory. Set to `true` to run the request between the turns of other requests, as with an `[AlwaysInterleave]` method: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_options"::: -### Describe the request to call filters +Set to `true` to apply backpressure to a producer. The request then counts against , and the returned task faults with when the activation's queue is full: -Set , , and to identify the work in call filters, logs, and traces. Tracing names the request's activity `InterfaceName/MethodName`. The callback returns a result through the returned task: +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_queue_limit"::: +### Name the request + +Call filters, logs, and traces observe the request as `IGrainRequestInvoker.{methodName}`, and tracing names its activity `IGrainRequestInvoker/{methodName}`. When the callback is a method group, such as `this.RunRequestAsync(RefreshAsync)`, the method's name is used. Otherwise, the request is named `RunRequestAsync`. Pass `methodName` to choose the name: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_named"::: -Call filters read the description through the call context: +Call filters read the name through the call context: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_filter"::: -The arguments describe the request to filters; the callback receives its `state`. A filter that calls `SetArgument` writes into the array assigned to `Arguments`. +### Deliver a custom request + + delivers any to the activation with the same scheduling as `RunRequestAsync`. Call filters observe the request's own interface and method names. The activation completes the supplied with the request's response, or with an exception if the activation rejects the request: + +:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="receive_local_request"::: diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index 2fcb38a7a52..e460939159b 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -311,7 +311,7 @@ private Task OnHeartbeat(DateTimeOffset timestamp, CancellationToken cancellatio return Task.CompletedTask; }, (Grain: this, Timestamp: timestamp), - new GrainRequestOptions { KeepAlive = false, Interleave = true }, + new LocalRequestOptions { KeepAlive = false, Interleave = true }, cancellationToken); // } @@ -341,18 +341,13 @@ private int Apply(IReadOnlyList orders) } // -// Call filters and tracing observe this request as IOrderFeed.ApplyOrders with one argument. +// Call filters and tracing observe this request as IGrainRequestInvoker.ApplyOrders. internal Task ApplyOrders(IReadOnlyList orders, CancellationToken cancellationToken) => this.RunRequestAsync( static (state, _) => Task.FromResult(state.Grain.Apply(state.Orders)), (Grain: this, Orders: orders), - new GrainRequestOptions - { - InterfaceName = "IOrderFeed", - MethodName = "ApplyOrders", - Arguments = [orders.Count], - }, - cancellationToken); + cancellationToken, + methodName: nameof(ApplyOrders)); // } @@ -362,13 +357,82 @@ public sealed class RequestLoggingFilter(ILogger logger) : public async Task Invoke(IIncomingGrainCallContext context) { logger.LogInformation( - "Running {InterfaceName}.{MethodName} with {ArgumentCount} arguments", + "Running {InterfaceName}.{MethodName}", context.InterfaceName, - context.MethodName, - context.Request.GetArgumentCount()); + context.MethodName); await context.Invoke(); } } // + + public interface IIngestGrain : IGrainWithStringKey + { + ValueTask GetCount(); + } + + public sealed class IngestGrain : Grain, IIngestGrain + { + private int _count; + + public ValueTask GetCount() => ValueTask.FromResult(_count); + + // +// Applies backpressure to the producer: the request faults with LimitExceededException +// when the activation's queue exceeds SiloMessagingOptions.MaxEnqueuedRequestsHardLimit. +internal Task Ingest(int items, CancellationToken cancellationToken) => + this.RunRequestAsync( + static (state, _) => + { + state.Grain._count += state.Items; + return Task.CompletedTask; + }, + (Grain: this, Items: items), + new LocalRequestOptions { EnforceQueueLimit = true }, + cancellationToken); + // + + internal void Increment() => _count++; + } + + // +// A request which increments IngestGrain's counter. Call filters observe it as IIngestRequests.Increment. +internal sealed class IncrementRequest : Request +{ + private IngestGrain? _grain; + + public override object GetTarget() => _grain!; + + public override void SetTarget(ITargetHolder holder) => _grain = (IngestGrain?)holder.GetTarget(); + + protected override ValueTask InvokeInner() + { + _grain!.Increment(); + return default; + } + + public override string GetInterfaceName() => "IIngestRequests"; + + public override string GetMethodName() => "Increment"; + + public override string GetActivityName() => "IIngestRequests/Increment"; + + public override Type GetInterfaceType() => typeof(IIngestGrain); + + public override System.Reflection.MethodInfo GetMethod() => typeof(IIngestGrain).GetMethod(nameof(IIngestGrain.GetCount))!; + + public override void Dispose() { } +} + +public static class IngestRequests +{ + // Delivers the request to the activation and completes the returned task with its response. + public static ValueTask Increment(IGrainContext activation) + { + var completion = new ResponseCompletionSource(); + activation.ReceiveLocalRequest(new IncrementRequest(), completion, default); + return completion.AsVoidValueTask(); + } +} + // } diff --git a/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs deleted file mode 100644 index c78af916bbc..00000000000 --- a/src/Orleans.Core.Abstractions/Core/GrainRequestOptions.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Orleans.Concurrency; - -namespace Orleans.Runtime; - -/// -/// Options for requests which a grain runs on its own activation using -/// . -/// -[SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "These per-call options carry a caller-owned arguments array rather than representing a comparable value.")] -public readonly struct GrainRequestOptions -{ - private readonly bool _suppressKeepAlive; - - /// - /// Initializes a new instance with the default options. - /// - public GrainRequestOptions() - { - } - - /// - /// Gets a value indicating whether the request extends the lifetime of the grain activation. - /// Defaults to . - /// - /// - /// A request which keeps the activation alive restarts the activation's idle collection period when it completes, - /// the same as a grain call. A request which does not keep the activation alive leaves the idle collection period unchanged. - /// - public bool KeepAlive - { - get => !_suppressKeepAlive; - init => _suppressKeepAlive = !value; - } - - /// - /// Gets a value indicating whether the request can interleave with other requests on the activation. - /// Defaults to . - /// - /// - /// If this value is , the request follows the grain's reentrancy rules, the same as a grain call. - /// If this value is , the request interleaves with other requests, the same as a call to a method with - /// the applied. - /// - public bool Interleave { get; init; } - - /// - /// Gets the interface name which grain call filters observe for the request. - /// Defaults to IGrainRequestInvoker. - /// - /// - /// Call filters observe this value as , and tracing uses it in the request's activity name. - /// - public string? InterfaceName { get; init; } - - /// - /// Gets the method name which grain call filters observe for the request. - /// Defaults to RunRequestAsync. - /// - /// - /// Call filters observe this value as , and tracing uses it in the request's activity name. - /// - public string? MethodName { get; init; } - - /// - /// Gets the arguments which grain call filters observe for the request. - /// Defaults to no arguments. - /// - /// - /// The arguments describe the request to call filters, which access them through . - /// The callback receives its state rather than these arguments. A call filter which sets an argument writes to this array. - /// - public object?[]? Arguments { get; init; } -} diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 2cfab1a11ff..fcf6ff0cdd8 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -1,4 +1,6 @@ using System; +using System.Buffers; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -61,10 +63,14 @@ public static void DeactivateOnIdle(this IGrainBase grain) => /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default) => - RunRequestAsync(grain, callback, default(GrainRequestOptions), cancellationToken); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); /// /// Runs as a request on this grain activation. @@ -75,16 +81,22 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, GrainRequestOptions options, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync, object?>( + return RunRequestCore, object?>( + grain.GrainContext, static (callback, cancellationToken) => callback(cancellationToken), callback, options, - cancellationToken); + cancellationToken, + methodName); } /// @@ -96,11 +108,15 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default) => - RunRequestAsync(grain, callback, state, default(GrainRequestOptions), cancellationToken); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); /// /// Runs as a request on this grain activation. @@ -112,13 +128,17 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, GrainRequestOptions options, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync(callback, state, options, cancellationToken); + return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken, methodName); } /// @@ -129,11 +149,15 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default) => - RunRequestAsync(grain, callback, default(GrainRequestOptions), cancellationToken); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); /// /// Runs as a request on this grain activation. @@ -144,17 +168,23 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, GrainRequestOptions options, CancellationToken cancellationToken = default) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync>, TResult>( + return RunRequestCore>, TResult>( + grain.GrainContext, static (callback, cancellationToken) => callback(cancellationToken), callback, options, - cancellationToken); + cancellationToken, + methodName); } /// @@ -166,12 +196,16 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the parameter. /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) => - RunRequestAsync(grain, callback, state, default(GrainRequestOptions), cancellationToken); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); /// /// Runs as a request on this grain activation. @@ -183,6 +217,10 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// /// The type of the parameter. /// The type of the callback's result. /// A which completes with the outcome of the callback. @@ -190,8 +228,8 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// The request is scheduled with the activation's other requests. By default, it follows the grain's reentrancy rules: /// on a non-reentrant grain, the callback runs exclusively, the same as a grain method call. - /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync, or with the names and arguments set in - /// . It starts a new call chain and, by default, keeps the activation alive. + /// The request passes through grain call filters as a method of IGrainRequestInvoker named by . + /// It starts a new call chain and, by default, keeps the activation alive. /// A call filter which fails or sets the result without invoking the request completes the returned task with that outcome. /// /// @@ -202,18 +240,42 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// If the activation deactivates before the request starts, the returned task faults with /// . - /// If the activation's queue exceeds the configured maximum number of enqueued requests, the returned task faults with - /// . + /// If is set and the activation's queue exceeds the configured maximum number of enqueued requests, + /// the returned task faults with . + /// To queue a custom , use . /// /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, GrainRequestOptions options, CancellationToken cancellationToken = default) + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) { ArgumentNullException.ThrowIfNull(callback); - return GetRequestRunner(grain).RunRequestAsync(callback, state, options, cancellationToken); + return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken, methodName); } - private static IGrainRequestRunner GetRequestRunner(IGrainBase grain) => - grain.GrainContext as IGrainRequestRunner ?? throw new NotSupportedException($"{grain.GrainContext.GetType()} does not support running grain requests."); + private static Task RunRequestCore( + IGrainContext grainContext, + Func callback, + TState state, + LocalRequestOptions options, + CancellationToken cancellationToken, + string? methodName) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + // The activation owns the request once it is queued and disposes it after invocation. +#pragma warning disable CA2000 // Dispose objects before losing scope + var request = new GrainRequest(callback, state, IsMemberName(methodName) ? methodName : null, cancellationToken); +#pragma warning restore CA2000 // Dispose objects before losing scope + grainContext.ReceiveLocalRequest(request, request, options); + return request.Task; + } + + // A simple method group, such as ProcessAsync, names the request. Other expressions, such as lambdas, use the default name. + private static bool IsMemberName(string? name) => name is { Length: > 0 } && !name.AsSpan().ContainsAnyExcept(MemberNameChars); + + private static readonly SearchValues MemberNameChars = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"); /// /// Creates a grain timer. diff --git a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs index 6ebb35ddd00..a72c391c2b2 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs @@ -69,6 +69,33 @@ public interface IGrainContext : ITargetHolder, IEquatable /// The message. void ReceiveMessage(object message); + /// + /// Queues as a one-way request on this instance. + /// + /// + /// The request to invoke. Its method must return an interface type. + /// + /// + /// The completion source which receives the request's response, or if no caller observes the response. + /// + /// The options for the request. + /// + /// + /// The request is scheduled with the instance's other requests and passes through grain call filters, which observe + /// . It starts a new call chain and never leaves this instance. + /// + /// + /// receives the response after the request and its call filters complete. + /// If this instance deactivates before the request starts, receives an + /// . If is set and the + /// instance's queue exceeds the configured maximum number of enqueued requests, receives a + /// . + /// + /// + /// This instance does not support local requests. + void ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + => throw new NotSupportedException($"{GetType()} does not support local requests."); + /// /// Start activating this instance. /// diff --git a/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs b/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs deleted file mode 100644 index 4b216d11fc0..00000000000 --- a/src/Orleans.Core.Abstractions/Core/IGrainRequestRunner.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace Orleans.Runtime; - -/// -/// A grain context which runs callbacks as requests on its activation. -/// -internal interface IGrainRequestRunner -{ - /// - /// Queues as a request on this activation. - /// - /// - /// The returned task completes with the result of the returned by the callback, or with - /// when the callback returns a non-generic . - /// - Task RunRequestAsync( - Func callback, - TState state, - in GrainRequestOptions options, - CancellationToken cancellationToken); -} diff --git a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs new file mode 100644 index 00000000000..0d434a5c64c --- /dev/null +++ b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs @@ -0,0 +1,58 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Orleans.Concurrency; + +namespace Orleans.Runtime; + +/// +/// Options for a request which a grain queues on its own activation, using +/// or 's RunRequestAsync methods. +/// +[SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "Per-call options are passed to the runtime rather than compared.")] +public readonly struct LocalRequestOptions +{ + private readonly bool _suppressKeepAlive; + + /// + /// Initializes a new instance with the default options. + /// + public LocalRequestOptions() + { + } + + /// + /// Gets a value indicating whether the request extends the lifetime of the grain activation. + /// Defaults to . + /// + /// + /// A request which keeps the activation alive restarts the activation's idle collection period when it completes, + /// the same as a grain call. A request which does not keep the activation alive leaves the idle collection period unchanged. + /// + public bool KeepAlive + { + get => !_suppressKeepAlive; + init => _suppressKeepAlive = !value; + } + + /// + /// Gets a value indicating whether the request can interleave with other requests on the activation. + /// Defaults to . + /// + /// + /// If this value is , the request follows the grain's reentrancy rules, the same as a grain call. + /// If this value is , the request interleaves with other requests, the same as a call to a method with + /// the applied. + /// + public bool Interleave { get; init; } + + /// + /// Gets a value indicating whether the activation rejects the request when its queue is full. + /// Defaults to . + /// + /// + /// If this value is and the activation's queue exceeds + /// the configured maximum number of enqueued requests (SiloMessagingOptions.MaxEnqueuedRequestsHardLimit), the request completes with a . + /// If this value is , the activation queues the request regardless of the number of enqueued requests. + /// + public bool EnforceQueueLimit { get; init; } +} diff --git a/src/Orleans.Runtime/Timers/GrainRequest.cs b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs similarity index 62% rename from src/Orleans.Runtime/Timers/GrainRequest.cs rename to src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs index be25a46dfb6..e4915ee468f 100644 --- a/src/Orleans.Runtime/Timers/GrainRequest.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs @@ -1,46 +1,38 @@ +using System; using System.Reflection; +using System.Threading; +using System.Threading.Tasks; using Orleans.Serialization.Invocation; namespace Orleans.Runtime; /// -/// A request queued on an activation by . +/// A request queued on an activation by . /// /// /// The request is its own completion source, so queuing a request allocates only this object and its message. /// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. /// When a call filter completes the invocation without running the callback, the task reflects the filter's outcome. -/// Call filters observe the request as unless it is created with -/// and named by . +/// Call filters observe the request as , with the method name supplied by the caller, if any. /// -internal class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest +internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest, IResponseCompletionSource { private readonly Func _callback; private readonly TState _state; + private readonly string? _methodName; private readonly CancellationTokenRegistration _registration; - public GrainRequest(Func callback, TState state, CancellationToken cancellationToken) + public GrainRequest(Func callback, TState state, string? methodName, CancellationToken cancellationToken) : base(TaskCreationOptions.RunContinuationsAsynchronously) { _callback = callback; _state = state; + _methodName = methodName; _registration = cancellationToken.UnsafeRegister( static (state, cancellationToken) => ((GrainRequest)state!).TrySetCanceled(cancellationToken), this); } - /// - /// Creates a request, allocating storage for the names and arguments in only when they are set. - /// - public static GrainRequest Create( - Func callback, - TState state, - in GrainRequestOptions options, - CancellationToken cancellationToken) => - options.InterfaceName is null && options.MethodName is null && options.Arguments is null - ? new GrainRequest(callback, state, cancellationToken) - : new NamedGrainRequest(callback, state, options, cancellationToken); - public ValueTask Invoke() { var cancellationToken = _registration.Token; @@ -122,6 +114,10 @@ bool ILocalRequest.Complete(Response response) return true; } + void IResponseCompletionSource.Complete(Response value) => ((ILocalRequest)this).Complete(value); + + void IResponseCompletionSource.Complete() => ((ILocalRequest)this).Complete(Response.Completed); + public object? GetTarget() => this; public void SetTarget(ITargetHolder holder) @@ -131,17 +127,17 @@ public void SetTarget(ITargetHolder holder) // Declared for call filters, which inspect the interface and implementation methods. Task IGrainRequestInvoker.RunRequestAsync(CancellationToken cancellationToken) => throw new InvalidOperationException(); - public virtual int GetArgumentCount() => 0; + public int GetArgumentCount() => 0; - public virtual object? GetArgument(int index) => throw new ArgumentOutOfRangeException(nameof(index)); + public object? GetArgument(int index) => throw new ArgumentOutOfRangeException(nameof(index)); - public virtual void SetArgument(int index, object? value) => throw new ArgumentOutOfRangeException(nameof(index)); + public void SetArgument(int index, object? value) => throw new ArgumentOutOfRangeException(nameof(index)); - public virtual string GetMethodName() => nameof(IGrainRequestInvoker.RunRequestAsync); + public string GetMethodName() => _methodName ?? nameof(IGrainRequestInvoker.RunRequestAsync); - public virtual string GetInterfaceName() => nameof(IGrainRequestInvoker); + public string GetInterfaceName() => nameof(IGrainRequestInvoker); - public virtual string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{nameof(IGrainRequestInvoker.RunRequestAsync)}"; + public string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{GetMethodName()}"; public MethodInfo GetMethod() => IGrainRequestInvoker.MethodInfo; @@ -153,62 +149,19 @@ public void Dispose() { } - public override string ToString() => $"Request '{_callback.Target}.{_callback.Method}'"; -} - -/// -/// A which describes itself to call filters using the names and arguments -/// in . Each unset name falls back to 's. -/// -internal sealed class NamedGrainRequest : GrainRequest -{ - private readonly string? _interfaceName; - private readonly string? _methodName; - private readonly object?[]? _arguments; - - public NamedGrainRequest(Func callback, TState state, in GrainRequestOptions options, CancellationToken cancellationToken) - : base(callback, state, cancellationToken) - { - _interfaceName = options.InterfaceName; - _methodName = options.MethodName; - _arguments = options.Arguments; - } - - public override int GetArgumentCount() => _arguments?.Length ?? 0; - - public override object? GetArgument(int index) => _arguments is { } arguments ? arguments[index] : base.GetArgument(index); - - public override void SetArgument(int index, object? value) - { - if (_arguments is { } arguments) - { - arguments[index] = value; - } - else - { - base.SetArgument(index, value); - } - } - - public override string GetMethodName() => _methodName ?? base.GetMethodName(); - - public override string GetInterfaceName() => _interfaceName ?? base.GetInterfaceName(); - - public override string GetActivityName() => $"{GetInterfaceName()}/{GetMethodName()}"; + public override string ToString() => $"Request '{GetActivityName()}'"; } /// -/// The interface which call filters observe for requests queued by . +/// The interface which call filters observe for requests queued by . /// -internal interface IGrainRequestInvoker : IAddressable +internal interface IGrainRequestInvoker { - internal static readonly GrainInterfaceType InterfaceType = GrainInterfaceType.Create("Orleans.Runtime.IGrainRequestInvoker"); internal static readonly MethodInfo MethodInfo = typeof(IGrainRequestInvoker).GetMethod(nameof(RunRequestAsync), BindingFlags.Instance | BindingFlags.Public)!; /// /// Runs the request callback. /// - [Alias("6D1E58B4")] Task RunRequestAsync(CancellationToken cancellationToken = default); } diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index 935f0e79103..6dde9a424a3 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -58,7 +58,7 @@ private CorrelationId GetNextCorrelationId() /// /// The request starts a new call chain, never expires, and completes on or is rejected with it. /// - public Message CreateLocalRequest(IGrainContext target, object body, GrainInterfaceType interfaceType, bool keepAlive, bool interleave) + public Message CreateLocalRequest(IGrainContext target, object body, GrainInterfaceType interfaceType, LocalRequestOptions options) { var siloAddress = target.Address.SiloAddress; var message = CreateMessage(body, InvokeMethodOptions.OneWay, requestContextData: null); @@ -68,8 +68,8 @@ public Message CreateLocalRequest(IGrainContext target, object body, GrainInterf message.SendingSilo = siloAddress; message.TargetSilo = siloAddress; message.InterfaceType = interfaceType; - message.IsKeepAlive = keepAlive; - message.IsAlwaysInterleave = interleave; + message.IsKeepAlive = options.KeepAlive; + message.IsAlwaysInterleave = options.Interleave; message.IsLocalOnly = true; return message; } diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 8bd72d63402..ac4f11999d9 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -30,7 +30,6 @@ internal sealed partial class ActivationData : IGrainContext, ICollectibleGrainContext, IActivationKeepAlive, - IGrainRequestRunner, IGrainExtensionBinder, IActivationWorkingSetMember, IGrainTimerRegistry, @@ -574,19 +573,6 @@ void IActivationKeepAlive.ReleaseKeepAlive() } } - Task IGrainRequestRunner.RunRequestAsync(Func callback, TState state, in GrainRequestOptions options, CancellationToken cancellationToken) - { - if (cancellationToken.IsCancellationRequested) - { - return Task.FromCanceled(cancellationToken); - } - - var request = GrainRequest.Create(callback, state, options, cancellationToken); - var message = _shared.InternalRuntime.MessageFactory.CreateLocalRequest(this, request, IGrainRequestInvoker.InterfaceType, options.KeepAlive, options.Interleave); - ReceiveMessage(message); - return request.Task; - } - public void ResetKeepAliveRequest() { lock (_lock) @@ -1674,10 +1660,17 @@ public void ReceiveMessage(Message message) } else // Request or OneWay { - ReceiveRequest(message); + ReceiveRequest(message, enforceQueueLimit: !message.IsLocalOnly); } } + public void ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + { + var message = _shared.InternalRuntime.RuntimeClient.CreateLocalRequest(this, request, completion, options); + _shared.InternalRuntime.MessagingTrace.OnDispatcherReceiveMessage(message); + ReceiveRequest(message, options.EnforceQueueLimit); + } + private void ReceiveResponse(Message message) { var state = State; @@ -1694,15 +1687,14 @@ private void ReceiveResponse(Message message) _shared.InternalRuntime.RuntimeClient.ReceiveResponse(message); } - private void ReceiveRequest(Message message) + private void ReceiveRequest(Message message, bool enforceQueueLimit) { var options = _shared.MessagingOptions; var (hardLimit, softLimit) = IsStatelessWorker ? (options.MaxEnqueuedRequestsHardLimit_StatelessWorker, options.MaxEnqueuedRequestsSoftLimit_StatelessWorker) : (options.MaxEnqueuedRequestsHardLimit, options.MaxEnqueuedRequestsSoftLimit); - // Requests queued by RunRequestAsync honor the hard limit. Grain timer ticks are exempt. - var enforceHardLimit = hardLimit > 0 && (!message.IsLocalOnly || message.BodyObject is ILocalRequest); + var enforceHardLimit = hardLimit > 0 && enforceQueueLimit; int count; bool admitted; lock (_lock) @@ -1723,7 +1715,7 @@ private void ReceiveRequest(Message message) var overloadException = new LimitExceededException(limitName, count, hardLimit, ToString()); if (message.IsLocalOnly) { - ((ILocalRequest)message.BodyObject!).Reject(overloadException); + (message.BodyObject as ILocalRequest)?.Reject(overloadException); } else { diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 66f9c196e08..82d6d055901 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -48,6 +48,7 @@ internal sealed partial class InsideRuntimeClient : IRuntimeClient, ILifecyclePa private HostedClient HostedClient => this.hostedClient; private readonly MessageFactory messageFactory; + private readonly GrainInterfaceTypeResolver interfaceTypeResolver; private IGrainReferenceRuntime grainReferenceRuntime = null!; private Task? callbackTimerTask; private readonly MessagingTrace messagingTrace; @@ -76,6 +77,7 @@ public InsideRuntimeClient( this.MySilo = siloDetails.SiloAddress; this.callbacks = new ConcurrentDictionary<(GrainId, CorrelationId), CallbackData>(); this.messageFactory = messageFactory; + this.interfaceTypeResolver = interfaceIdResolver; this.ConcreteGrainFactory = new GrainFactory(this, referenceActivator, interfaceIdResolver, interfaceToTypeResolver); this.logger = loggerFactory.CreateLogger(); this.invokeExceptionLogger = loggerFactory.CreateLogger($"{typeof(Grain).FullName}.InvokeException"); @@ -134,6 +136,21 @@ internal void ConsumeServices() systemSharedCallbackData.CancellationManager = _cancellationManager; } + /// + /// Creates the message which queues on as a local request. + /// + public Message CreateLocalRequest(IGrainContext target, IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + { + ArgumentNullException.ThrowIfNull(request); + var interfaceType = interfaceTypeResolver.GetGrainInterfaceType(request.GetInterfaceType()); + + // A request which is its own completion source reports its response directly. + object body = completion is null || ReferenceEquals(completion, request) && request is ILocalRequest + ? request + : new LocalRequestCompletion(request, completion); + return messageFactory.CreateLocalRequest(target, body, interfaceType, options); + } + public void SendRequest( GrainReference target, IInvokable request, @@ -308,8 +325,10 @@ public async Task Invoke(IGrainContext target, Message message) { switch (message.BodyObject) { - case IInvokable invokable: + case IInvokable body: { + // Call filters observe the request rather than its completion envelope. + var invokable = body is LocalRequestCompletion envelope ? envelope.Request : body; invokable.SetTarget(target); CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); diff --git a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs index 8bada07015f..3fb20d38c2f 100644 --- a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs +++ b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs @@ -20,8 +20,7 @@ internal class InternalGrainRuntime( IOptions collectionOptions, ILocalGrainDirectory localGrainDirectory, IActivationWorkingSet activationWorkingSet, - ActivationCollector activationCollector, - MessageFactory messageFactory) + ActivationCollector activationCollector) { public InsideRuntimeClient RuntimeClient { get; } = catalog.RuntimeClient; public MessageCenter MessageCenter { get; } = messageCenter; @@ -34,6 +33,5 @@ internal class InternalGrainRuntime( public ILocalGrainDirectory LocalGrainDirectory { get; } = localGrainDirectory; public IActivationWorkingSet ActivationWorkingSet { get; } = activationWorkingSet; public ActivationCollector ActivationCollector { get; } = activationCollector; - public MessageFactory MessageFactory { get; } = messageFactory; } } diff --git a/src/Orleans.Runtime/Core/LocalRequestCompletion.cs b/src/Orleans.Runtime/Core/LocalRequestCompletion.cs new file mode 100644 index 00000000000..89ca94aeb5e --- /dev/null +++ b/src/Orleans.Runtime/Core/LocalRequestCompletion.cs @@ -0,0 +1,55 @@ +using System.Reflection; +using Orleans.Serialization.Invocation; + +namespace Orleans.Runtime; + +/// +/// The body of a local request whose response is reported to a completion source other than the request itself. +/// +/// +/// Message processing observes this envelope as the request, and grain call filters observe . +/// +internal sealed class LocalRequestCompletion(IInvokable request, IResponseCompletionSource completion) : IInvokable, ILocalRequest +{ + public IInvokable Request { get; } = request; + + public bool Complete(Response response) + { + completion.Complete(response); + return true; + } + + public object? GetTarget() => Request.GetTarget(); + + public void SetTarget(ITargetHolder holder) => Request.SetTarget(holder); + + public ValueTask Invoke() => Request.Invoke(); + + public int GetArgumentCount() => Request.GetArgumentCount(); + + public object? GetArgument(int index) => Request.GetArgument(index); + + public void SetArgument(int index, object? value) => Request.SetArgument(index, value!); + + public string GetMethodName() => Request.GetMethodName(); + + public string GetInterfaceName() => Request.GetInterfaceName(); + + public string GetActivityName() => Request.GetActivityName(); + + public MethodInfo GetMethod() => Request.GetMethod(); + + public Type GetInterfaceType() => Request.GetInterfaceType(); + + public TimeSpan? GetDefaultResponseTimeout() => Request.GetDefaultResponseTimeout(); + + public CancellationToken GetCancellationToken() => Request.GetCancellationToken(); + + public bool TryCancel() => Request.TryCancel(); + + public bool IsCancellable => Request.IsCancellable; + + public void Dispose() => Request.Dispose(); + + public override string? ToString() => Request.ToString(); +} diff --git a/src/Orleans.Runtime/Core/SystemTarget.cs b/src/Orleans.Runtime/Core/SystemTarget.cs index b2f7d798c1c..fbf9b3f47d5 100644 --- a/src/Orleans.Runtime/Core/SystemTarget.cs +++ b/src/Orleans.Runtime/Core/SystemTarget.cs @@ -374,6 +374,10 @@ public void ReceiveMessage(object message) } } + /// + void IGrainContext.ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + => ReceiveMessage(RuntimeClient.CreateLocalRequest(this, request, completion, options)); + /// public object? GetTarget() => this; diff --git a/src/Orleans.Runtime/OrleansContracts.txt b/src/Orleans.Runtime/OrleansContracts.txt index ed58edddb4b..3dd8082bb7d 100644 --- a/src/Orleans.Runtime/OrleansContracts.txt +++ b/src/Orleans.Runtime/OrleansContracts.txt @@ -70,9 +70,6 @@ interface [GrainInterfaceType("Orleans.Runtime.ICatalog")] Orleans.Runtime.ICata interface [GrainInterfaceType("Orleans.Runtime.IGrainCallCancellationManagerSystemTarget")] Orleans.Runtime.IGrainCallCancellationManagerSystemTarget [Version(0)] AF79F3FA: CancelCallsAsync(System.Collections.Generic.List, System.Threading.CancellationToken) -> ValueTask -interface [GrainInterfaceType("Orleans.Runtime.IGrainRequestInvoker")] Orleans.Runtime.IGrainRequestInvoker [Version(0)] - 6D1E58B4: RunRequestAsync(System.Threading.CancellationToken) -> Task - interface [GrainInterfaceType("Orleans.Runtime.IGrainTimerInvoker")] Orleans.Runtime.IGrainTimerInvoker [Version(0)] 3F6C2672: InvokeCallbackAsync(System.Threading.CancellationToken) -> Task diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index 6deacb665dc..f8f81e5aeca 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -11,15 +11,13 @@ namespace Orleans.Runtime; internal abstract partial class GrainTimer : IGrainTimer { - protected static readonly GrainInterfaceType InvokableInterfaceType = GrainInterfaceType.Create("Orleans.Runtime.IGrainTimerInvoker"); protected static readonly TimerCallback TimerCallback = (state) => ((GrainTimer)state!).ScheduleTickOnActivation(); protected static readonly MethodInfo InvokableMethodInfo = typeof(IGrainTimerInvoker).GetMethod(nameof(IGrainTimerInvoker.InvokeCallbackAsync), BindingFlags.Instance | BindingFlags.Public)!; private readonly CancellationTokenSource _cts = new(); private readonly ITimer _timer; private readonly IGrainContext _grainContext; private readonly TimerRegistry _shared; - private readonly bool _interleave; - private readonly bool _keepAlive; + private readonly LocalRequestOptions _options; private readonly TimerTickInvoker _invoker; private TimerState _state; // The current arm while idle; the next delay once a tick is queued. Change replaces it. @@ -38,8 +36,7 @@ public GrainTimer(TimerRegistry shared, IGrainContext grainContext, bool interle ArgumentNullException.ThrowIfNull(shared); ArgumentNullException.ThrowIfNull(grainContext); - _interleave = interleave; - _keepAlive = keepAlive; + _options = new() { Interleave = interleave, KeepAlive = keepAlive }; _shared = shared; _grainContext = grainContext; _dueTime = Timeout.InfiniteTimeSpan; @@ -106,8 +103,7 @@ private void QueueTickOnActivation() { try { - var msg = _shared.MessageFactory.CreateLocalRequest(_grainContext, _invoker, InvokableInterfaceType, _keepAlive, _interleave); - _grainContext.ReceiveMessage(msg); + _grainContext.ReceiveLocalRequest(_invoker, completion: null, _options); } catch (Exception exception) { diff --git a/src/Orleans.Runtime/Timers/TimerRegistry.cs b/src/Orleans.Runtime/Timers/TimerRegistry.cs index f7183a04eff..acd2679f08b 100644 --- a/src/Orleans.Runtime/Timers/TimerRegistry.cs +++ b/src/Orleans.Runtime/Timers/TimerRegistry.cs @@ -5,11 +5,10 @@ namespace Orleans.Timers; -internal class TimerRegistry(ILoggerFactory loggerFactory, [FromKeyedServices(TimeProviderNames.Grains)] TimeProvider timeProvider, MessageFactory messageFactory) : ITimerRegistry +internal class TimerRegistry(ILoggerFactory loggerFactory, [FromKeyedServices(TimeProviderNames.Grains)] TimeProvider timeProvider) : ITimerRegistry { public ILogger TimerLogger { get; } = loggerFactory.CreateLogger(); public TimeProvider TimeProvider { get; } = timeProvider; - public MessageFactory MessageFactory { get; } = messageFactory; public IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period) { diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 5b7569af37f..d4cfcdbb3af 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -1119,21 +1119,21 @@ public static void MigrateOnIdle(this IGrainBase grain) { } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, System.TimeSpan dueTime, System.TimeSpan period) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, Runtime.GrainRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } } public delegate System.Threading.Tasks.Task GrainCallFilterDelegate(IGrainCallContext context); @@ -2361,21 +2361,6 @@ public GrainReferenceShared(GrainType grainType, GrainInterfaceType grainInterfa public System.IServiceProvider ServiceProvider { get { throw null; } } } - public readonly partial struct GrainRequestOptions - { - private readonly object _dummy; - private readonly int _dummyPrimitive; - public object?[]? Arguments { get { throw null; } init { } } - - public string? InterfaceName { get { throw null; } init { } } - - public bool Interleave { get { throw null; } init { } } - - public bool KeepAlive { get { throw null; } init { } } - - public string? MethodName { get { throw null; } init { } } - } - public readonly partial struct GrainTimerCreationOptions : System.IEquatable { private readonly int _dummyPrimitive; @@ -2642,6 +2627,7 @@ public partial interface IGrainContext : Orleans.Serialization.Invocation.ITarge void Activate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); void Deactivate(DeactivationReason deactivationReason, System.Threading.CancellationToken cancellationToken = default); void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); + void ReceiveLocalRequest(Orleans.Serialization.Invocation.IInvokable request, Orleans.Serialization.Invocation.IResponseCompletionSource? completion, LocalRequestOptions options); void ReceiveMessage(object message); void Rehydrate(IRehydrationContext context); void SetComponent(TComponent? value) @@ -2821,6 +2807,16 @@ public LimitExceededException(string limitName, int current, int threshold, obje public LimitExceededException(string message) { } } + public readonly partial struct LocalRequestOptions + { + private readonly int _dummyPrimitive; + public bool EnforceQueueLimit { get { throw null; } init { } } + + public bool Interleave { get { throw null; } init { } } + + public bool KeepAlive { get { throw null; } init { } } + } + public static partial class LogFormatter { public const int MAX_LOG_MESSAGE_SIZE = 20000; diff --git a/src/api/Orleans.Runtime/Orleans.Runtime.cs b/src/api/Orleans.Runtime/Orleans.Runtime.cs index edaf0437875..46d8d4e5bbc 100644 --- a/src/api/Orleans.Runtime/Orleans.Runtime.cs +++ b/src/api/Orleans.Runtime/Orleans.Runtime.cs @@ -991,6 +991,8 @@ public TExtensionInterface GetExtension() public void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken) { } + void IGrainContext.ReceiveLocalRequest(Orleans.Serialization.Invocation.IInvokable request, Orleans.Serialization.Invocation.IResponseCompletionSource? completion, LocalRequestOptions options) { } + object? Orleans.Serialization.Invocation.ITargetHolder.GetComponent(System.Type componentType) { throw null; } public void ReceiveMessage(object message) { } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 1eac2575772..3fc60641815 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -10,6 +10,12 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartNamedRequest(int orderId); + Task StartMethodGroupRequest(); + + Task StartCustomInvokableRequest(string value); + + Task QueueUnlimitedRequests(int count); + Task StartBlockingRequest(); [AlwaysInterleave] diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 6af014449d2..71b6ea09dbf 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -1,5 +1,7 @@ using System.Collections.Concurrent; +using System.Reflection; using Orleans.Concurrency; +using Orleans.Serialization.Invocation; using UnitTests.GrainInterfaces; namespace UnitTests.Grains; @@ -13,7 +15,6 @@ public abstract class RunRequestGrainBase : Grain, IRunRequestGrain, IIncomingGr private Task _pending = Task.CompletedTask; private RequestFilterMode _requestFilterMode; - public const string NamedRequestInterfaceName = "IOrderProcessor"; public async Task Invoke(IIncomingGrainCallContext context) { @@ -33,11 +34,6 @@ public async Task Invoke(IIncomingGrainCallContext context) } } - else if (context.InterfaceName == NamedRequestInterfaceName) - { - _log.Add($"argument:{context.Request.GetArgument(0)}"); - } - await context.Invoke(); } @@ -67,7 +63,7 @@ public async Task StartInterleavingRequestDuringCall() { _log.Add("request"); return Task.CompletedTask; - }, new GrainRequestOptions { Interleave = true }); + }, new LocalRequestOptions { Interleave = true }); await Task.Delay(200); _log.Add("call-end"); @@ -78,7 +74,35 @@ public Task StartNamedRequest(int orderId) _pending = this.RunRequestAsync( static (orderId, _) => Task.FromResult($"order-{orderId}"), orderId, - new GrainRequestOptions { InterfaceName = NamedRequestInterfaceName, MethodName = "ProcessOrder", Arguments = [orderId] }); + methodName: "ProcessOrder"); + return Task.CompletedTask; + } + + public Task StartMethodGroupRequest() + { + _pending = this.RunRequestAsync(ProcessOrders); + return Task.CompletedTask; + } + + private Task ProcessOrders(CancellationToken cancellationToken) => Task.FromResult("processed"); + + public Task QueueUnlimitedRequests(int count) + { + var admitted = new List(count); + for (var i = 0; i < count; i++) + { + admitted.Add(QueueLoggedRequest($"request-{i}", default)); + } + + _pending = Task.WhenAll(admitted); + return Task.FromResult(admitted.Count(request => !request.IsFaulted)); + } + + public Task StartCustomInvokableRequest(string value) + { + var completion = new ResponseCompletionSource(); + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CustomRequest(value), completion, default); + _pending = completion.AsValueTask().AsTask(); return Task.CompletedTask; } @@ -119,7 +143,7 @@ public Task> QueueRequests(int count) var admitted = new List(count); for (var i = 0; i < count; i++) { - var request = QueueLoggedRequest($"request-{i}"); + var request = QueueLoggedRequest($"request-{i}", EnforceQueueLimit); if (request.IsFaulted) { outcomes.Add(request.Exception!.InnerException!.GetType().Name); @@ -143,7 +167,7 @@ public async Task QueueRequestsConcurrently(int count) () => { barrier.SignalAndWait(); - return QueueLoggedRequest($"request-{i}"); + return QueueLoggedRequest($"request-{i}", EnforceQueueLimit); }, CancellationToken.None, TaskCreationOptions.LongRunning, @@ -159,12 +183,14 @@ public async Task QueueRequestsConcurrently(int count) return admitted.Count; } - private Task QueueLoggedRequest(string entry) => + private static readonly LocalRequestOptions EnforceQueueLimit = new() { EnforceQueueLimit = true }; + + private Task QueueLoggedRequest(string entry, LocalRequestOptions options) => this.RunRequestAsync(static (state, _) => { state.Grain._log.Add(state.Entry); return Task.CompletedTask; - }, (Grain: this, Entry: entry)); + }, (Grain: this, Entry: entry), options); public Task StartRequestContextRequest(string value) { @@ -238,6 +264,34 @@ public Task DeactivateAndRetainInstance() RetainedInstances[key]); } +/// +/// A hand-written request delivered through . +/// +internal sealed class CustomRequest(string value) : Request +{ + public const string InterfaceName = "ICustomRequests"; + public const string MethodName = "Apply"; + private RunRequestGrainBase? _grain; + + public override object GetTarget() => _grain!; + + public override void SetTarget(ITargetHolder holder) => _grain = (RunRequestGrainBase?)holder.GetTarget(); + + protected override ValueTask InvokeInner() => new($"{value}:{_grain is not null}"); + + public override void Dispose() { } + + public override string GetMethodName() => MethodName; + + public override string GetInterfaceName() => InterfaceName; + + public override string GetActivityName() => $"{InterfaceName}/{MethodName}"; + + public override Type GetInterfaceType() => typeof(IRunRequestGrain); + + public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; +} + public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReentrantRunRequestGrain { } diff --git a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs index 54fa280ef38..f799bd01d26 100644 --- a/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs +++ b/test/Grains/TestInternalGrains/ActivationGCTestGrains.cs @@ -203,7 +203,7 @@ public Task ReleaseKeepAliveHold() public Task QueueRequest(TimeSpan duration, bool keepAlive) { - _ = this.RunRequestAsync(static (duration, cancellationToken) => Task.Delay(duration, cancellationToken), duration, new GrainRequestOptions { KeepAlive = keepAlive }); + _ = this.RunRequestAsync(static (duration, cancellationToken) => Task.Delay(duration, cancellationToken), duration, new LocalRequestOptions { KeepAlive = keepAlive }); return Task.CompletedTask; } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index f83c98ba2b3..44733e0cec9 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -5,7 +5,7 @@ namespace NonSilo.Tests.Runtime; /// -/// Tests for the request which queues. +/// Tests for the request which queues. /// [TestCategory("BVT"), TestCategory("Scheduler")] public class GrainRequestTests @@ -24,6 +24,7 @@ public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRun return Task.CompletedTask; }, null, + null, cancellation.Token); // Cancellation runs callbacks in reverse registration order, so this one runs before the request's. @@ -50,7 +51,7 @@ public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRun public async Task Complete_CallbackFailure_IsReportedByRequest() { var exception = new InvalidOperationException("Request failed."); - var request = new GrainRequest((_, _) => Task.FromException(exception), null, default); + var request = new GrainRequest((_, _) => Task.FromException(exception), null, null, default); var response = await request.Invoke(); @@ -61,7 +62,7 @@ public async Task Complete_CallbackFailure_IsReportedByRequest() [Fact] public async Task Complete_FilterFailureAfterCallback_IsNotReportedByRequest() { - var request = new GrainRequest((_, _) => Task.FromResult("result"), null, default); + var request = new GrainRequest((_, _) => Task.FromResult("result"), null, null, default); await request.Invoke(); @@ -80,6 +81,7 @@ public async Task Complete_FilterFailureWithoutCallback_IsReportedByRequest() return Task.CompletedTask; }, null, + null, default); Assert.True(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); @@ -95,7 +97,7 @@ public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTh for (var i = 0; i < 2_000; i++) { using var cancellation = new CancellationTokenSource(); - var request = new GrainRequest((_, _) => Task.CompletedTask, null, cancellation.Token); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, null, cancellation.Token); using var start = new Barrier(2); var cancel = Task.Run(() => { @@ -124,6 +126,7 @@ public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() return Task.CompletedTask; }, null, + null, cancellation.Token); var exception = new OrleansMessageRejectionException("Rejected."); @@ -136,43 +139,22 @@ public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() } [Fact] - public void Create_WithoutNames_DescribesRequestAsGrainRequestInvoker() + public void Describe_WithoutMethodName_DescribesRequestAsGrainRequestInvoker() { - var request = GrainRequest.Create((_, _) => Task.CompletedTask, null, new GrainRequestOptions { KeepAlive = false, Interleave = true }, TestContext.Current.CancellationToken); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, null, TestContext.Current.CancellationToken); - Assert.IsNotType>(request); Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); Assert.Equal("RunRequestAsync", request.GetMethodName()); Assert.Equal("IGrainRequestInvoker/RunRequestAsync", request.GetActivityName()); + Assert.Equal(typeof(IGrainRequestInvoker), request.GetInterfaceType()); Assert.Equal(0, request.GetArgumentCount()); Assert.Throws(() => request.GetArgument(0)); } [Fact] - public void Create_WithNamesAndArguments_DescribesRequestWithThem() - { - var arguments = new object?[] { 1, "two" }; - var request = GrainRequest.Create( - (_, _) => Task.CompletedTask, - null, - new GrainRequestOptions { InterfaceName = "IOrderProcessor", MethodName = "ProcessOrder", Arguments = arguments }, - TestContext.Current.CancellationToken); - - Assert.Equal("IOrderProcessor", request.GetInterfaceName()); - Assert.Equal("ProcessOrder", request.GetMethodName()); - Assert.Equal("IOrderProcessor/ProcessOrder", request.GetActivityName()); - Assert.Equal(2, request.GetArgumentCount()); - Assert.Equal(1, request.GetArgument(0)); - Assert.Equal("two", request.GetArgument(1)); - - request.SetArgument(1, "three"); - Assert.Equal("three", arguments[1]); - } - - [Fact] - public void Create_WithMethodNameOnly_FallsBackToDefaultInterfaceName() + public void Describe_WithMethodName_UsesMethodName() { - var request = GrainRequest.Create((_, _) => Task.CompletedTask, null, new GrainRequestOptions { MethodName = "Drain" }, TestContext.Current.CancellationToken); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, "Drain", TestContext.Current.CancellationToken); Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); Assert.Equal("Drain", request.GetMethodName()); @@ -181,11 +163,22 @@ public void Create_WithMethodNameOnly_FallsBackToDefaultInterfaceName() } [Fact] - public void GrainRequestOptions_Default_KeepsActivationAlive() + public async Task CompletionSource_Complete_CompletesTask() + { + var request = new GrainRequest((_, _) => Task.FromResult("unused"), null, null, TestContext.Current.CancellationToken); + + ((IResponseCompletionSource)request).Complete(Response.FromResult("filtered")); + + Assert.Equal("filtered", await request.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + + [Fact] + public void LocalRequestOptions_Default_KeepsActivationAliveWithoutInterleavingOrQueueLimit() { - Assert.True(default(GrainRequestOptions).KeepAlive); - Assert.True(new GrainRequestOptions().KeepAlive); - Assert.False(new GrainRequestOptions { KeepAlive = false }.KeepAlive); - Assert.False(default(GrainRequestOptions).Interleave); + Assert.True(default(LocalRequestOptions).KeepAlive); + Assert.True(new LocalRequestOptions().KeepAlive); + Assert.False(new LocalRequestOptions { KeepAlive = false }.KeepAlive); + Assert.False(default(LocalRequestOptions).Interleave); + Assert.False(default(LocalRequestOptions).EnforceQueueLimit); } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 30ea749f102..cdd740155e1 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -42,23 +42,49 @@ public async Task ImmediateTick_UsesOrdinaryActivationAdmission(bool interleave, return Task.CompletedTask; }, state, TimeSpan.Zero, Timeout.InfiniteTimeSpan, interleave, keepAlive); - var message = Assert.Single(fixture.Messages); + var request = Assert.Single(fixture.Requests); Assert.Equal(0, calls); Assert.Equal(1, fixture.Time.TimerCreations); + Assert.Equal(interleave, request.Options.Interleave); + Assert.Equal(keepAlive, request.Options.KeepAlive); + Assert.False(request.Options.EnforceQueueLimit); + Assert.Same(parent, RequestContext.Get("parent")); + await fixture.InvokeAsync(request); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(1, calls); + Assert.Single(fixture.Requests); + } + finally + { + RequestContext.Remove("parent"); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, true)] + public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool interleave, bool keepAlive) + { + using var fixture = new TimerFixture(); + var interfaceType = GrainInterfaceType.Create("local-request"); + RequestContext.Set("parent", new UncopyableContextValue()); + try + { + var body = new object(); + var message = fixture.MessageFactory.CreateLocalRequest(fixture.Grain, body, interfaceType, new() { Interleave = interleave, KeepAlive = keepAlive }); + + Assert.Same(body, message.BodyObject); Assert.Equal(Message.Directions.OneWay, message.Direction); Assert.Equal(interleave, message.IsAlwaysInterleave); Assert.Equal(keepAlive, message.IsKeepAlive); Assert.True(message.IsLocalOnly); Assert.Null(message.TimeToLive); Assert.Null(message.RequestContextData); - Assert.Same(parent, RequestContext.Get("parent")); + Assert.Equal(interfaceType, message.InterfaceType); Assert.Equal(fixture.Grain.GrainId, message.TargetGrain); Assert.Equal(message.SendingGrain, message.TargetGrain); + Assert.Equal(fixture.Grain.Address.SiloAddress, message.TargetSilo); Assert.Equal(message.SendingSilo, message.TargetSilo); - await fixture.InvokeAsync(message); - fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Equal(1, calls); - Assert.Single(fixture.Messages); } finally { @@ -110,8 +136,8 @@ public async Task ChangeWhileQueued_PreservesTickAndUsesLatestFollowingSchedule( timer.Change(TimeSpan.Zero, TimeSpan.Zero); timer.Change(TimeSpan.FromMilliseconds(milliseconds), Timeout.InfiniteTimeSpan); fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Single(fixture.Messages); - await fixture.InvokeAsync(fixture.Messages[0]); + Assert.Single(fixture.Requests); + await fixture.InvokeAsync(fixture.Requests[0]); Assert.Equal(1, calls); await VerifyFollowingTickAsync(fixture, milliseconds); Assert.Equal(milliseconds < 0 ? 1 : 2, calls); @@ -135,7 +161,7 @@ public async Task ChangeDuringCallback_DefersFollowingTickUntilCompletion(int mi await release.Task; } }, 0, TimeSpan.Zero, TimeSpan.Zero, interleave: true); - var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages)); + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Requests)); try { await started.Task; @@ -144,7 +170,7 @@ public async Task ChangeDuringCallback_DefersFollowingTickUntilCompletion(int mi fixture.Time.Advance(TimeSpan.FromDays(1)); fixture.Time.FireDispatchedCallback(); Assert.Equal(1, calls); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); } finally { @@ -162,19 +188,19 @@ public async Task ZeroPeriod_QueuesSuccessiveTurnsAndDisposalDrainsLastTurn() var calls = 0; using var timer = fixture.Register((_, _) => { - Assert.Equal(++calls, fixture.Messages.Count); + Assert.Equal(++calls, fixture.Requests.Count); return Task.CompletedTask; }, 0, TimeSpan.Zero, TimeSpan.Zero); for (var i = 0; i < 8; i++) { - await fixture.InvokeAsync(fixture.Messages[i]); - Assert.Equal(i + 2, fixture.Messages.Count); + await fixture.InvokeAsync(fixture.Requests[i]); + Assert.Equal(i + 2, fixture.Requests.Count); } timer.Dispose(); - await fixture.InvokeAsync(fixture.Messages[8]); + await fixture.InvokeAsync(fixture.Requests[8]); fixture.Time.Advance(TimeSpan.FromDays(1)); Assert.Equal(8, calls); - Assert.Equal(9, fixture.Messages.Count); + Assert.Equal(9, fixture.Requests.Count); Assert.Equal(1, fixture.Time.TimerCreations); } @@ -194,10 +220,10 @@ public async Task DelayedTick_UsesProviderTimingAndResumesPeriod(double millisec for (var i = 0; i < 2; i++) { fixture.Time.Advance(delay - TimeSpan.FromTicks(1)); - Assert.Equal(i, fixture.Messages.Count); + Assert.Equal(i, fixture.Requests.Count); fixture.Time.Advance(TimeSpan.FromTicks(1)); - Assert.Equal(i + 1, fixture.Messages.Count); - await fixture.InvokeAsync(fixture.Messages[i]); + Assert.Equal(i + 1, fixture.Requests.Count); + await fixture.InvokeAsync(fixture.Requests[i]); Assert.Equal(i + 1, calls); } Assert.Equal(1, fixture.Time.TimerCreations); @@ -217,11 +243,11 @@ public async Task DispatchedTick_RacingDelayedChange_IsAdmittedOnce() fixture.Time.FireDispatchedCallback(); fixture.Time.FireDispatchedCallback(); Assert.Equal(0, calls); - await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + await fixture.InvokeAsync(Assert.Single(fixture.Requests)); fixture.Time.Advance(TimeSpan.FromSeconds(5) - TimeSpan.FromTicks(1)); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); fixture.Time.Advance(TimeSpan.FromTicks(1)); - await fixture.InvokeAsync(fixture.Messages[1]); + await fixture.InvokeAsync(fixture.Requests[1]); Assert.Equal(2, calls); } @@ -234,7 +260,7 @@ public async Task PausedTimer_DispatchedTickLeavesSchedulePaused() timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); fixture.Time.FireDispatchedCallback(); fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Empty(fixture.Messages); + Assert.Empty(fixture.Requests); } [Theory] @@ -258,10 +284,10 @@ public async Task DisposedTimer_QueuedTickDrains(bool delayed, bool interleave) timer.Dispose(); timer.Change(TimeSpan.Zero, TimeSpan.Zero); fixture.Time.FireDispatchedCallback(); - await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + await fixture.InvokeAsync(Assert.Single(fixture.Requests)); fixture.Time.Advance(TimeSpan.FromDays(1)); Assert.Equal(0, calls); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); Assert.Collection(fixture.Events, evt => Assert.IsType(evt), evt => Assert.IsType(evt)); @@ -292,7 +318,7 @@ await Task.Run(timer.Dispose, TestContext.Current.CancellationToken) release.SetResult(); await admission; } - await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + await fixture.InvokeAsync(Assert.Single(fixture.Requests)); Assert.Empty(fixture.Events.OfType()); } @@ -310,7 +336,7 @@ public async Task DisposeWithBlockedCancellation_QueuedTickDrainsBeforeCancellat token = value; return Task.CompletedTask; }, 0, TimeSpan.Zero, Timeout.InfiniteTimeSpan, interleave: true); - await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + await fixture.InvokeAsync(Assert.Single(fixture.Requests)); using var registration = token.Register(() => { entered.SetResult(); @@ -321,7 +347,7 @@ public async Task DisposeWithBlockedCancellation_QueuedTickDrainsBeforeCancellat try { await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - await Task.Run(() => fixture.InvokeAsync(fixture.Messages[1]), TestContext.Current.CancellationToken) + await Task.Run(() => fixture.InvokeAsync(fixture.Requests[1]), TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.True(token.IsCancellationRequested); Assert.False(disposal.IsCompleted); @@ -361,7 +387,7 @@ public async Task DisposeDuringCallback_ReentrantFailingCancellationCompletesDis }, 0, TimeSpan.Zero, TimeSpan.Zero); using (timer) { - var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages)); + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Requests)); try { await started.Task; @@ -376,7 +402,7 @@ public async Task DisposeDuringCallback_ReentrantFailingCancellationCompletesDis Assert.Same(exception, Assert.Single(cancellationError.InnerExceptions)); registry.Received(1).OnTimerDisposed(timer); fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); Assert.Single(fixture.Events.OfType()); Assert.Single(fixture.Events.OfType()); } @@ -404,14 +430,14 @@ public async Task CallbackFailure_PairsDiagnosticsAndSchedulesNextPeriod(bool as } throw exception; }, 0, TimeSpan.Zero, TimeSpan.FromSeconds(2)); - var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages), exception); + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Requests), exception); try { if (asynchronous) { await started.Task; fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); } } finally @@ -420,7 +446,7 @@ public async Task CallbackFailure_PairsDiagnosticsAndSchedulesNextPeriod(bool as await invocation; } fixture.Time.Advance(TimeSpan.FromSeconds(2)); - await fixture.InvokeAsync(fixture.Messages[1]); + await fixture.InvokeAsync(fixture.Requests[1]); Assert.Equal(2, calls); Assert.Collection(fixture.Events, evt => Assert.IsType(evt), @@ -447,15 +473,15 @@ public async Task LegacyTimer_UsesInterleavingTurnsAndStopsAfterDisposal() calls++; return Task.CompletedTask; }, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan); - var message = Assert.Single(fixture.Messages); - Assert.True(message.IsAlwaysInterleave); - Assert.False(message.IsKeepAlive); - await fixture.InvokeAsync(message); + var request = Assert.Single(fixture.Requests); + Assert.True(request.Options.Interleave); + Assert.False(request.Options.KeepAlive); + await fixture.InvokeAsync(request); timer.Dispose(); fixture.Time.FireDispatchedCallback(); fixture.Time.Advance(TimeSpan.FromDays(1)); Assert.Equal(1, calls); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); } private static TaskCompletionSource NewCompletion() => new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -466,18 +492,18 @@ private static async Task VerifyFollowingTickAsync(TimerFixture fixture, int mil { if (milliseconds > 0) { - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); fixture.Time.Advance(TimeSpan.FromMilliseconds(milliseconds) - TimeSpan.FromTicks(1)); - Assert.Single(fixture.Messages); + Assert.Single(fixture.Requests); fixture.Time.Advance(TimeSpan.FromTicks(1)); } if (milliseconds >= 0) { - Assert.Equal(2, fixture.Messages.Count); - await fixture.InvokeAsync(fixture.Messages[1]); + Assert.Equal(2, fixture.Requests.Count); + await fixture.InvokeAsync(fixture.Requests[1]); } fixture.Time.Advance(TimeSpan.FromDays(1)); - Assert.Equal(milliseconds < 0 ? 1 : 2, fixture.Messages.Count); + Assert.Equal(milliseconds < 0 ? 1 : 2, fixture.Requests.Count); } private sealed class TimerFixture : IObserver, IDisposable @@ -487,7 +513,7 @@ private sealed class TimerFixture : IObserver, IDis private readonly ILogger _logger = Substitute.For(); public TrackingTimeProvider Time { get; } = new(); public IGrainContext Grain { get; } = Substitute.For(); - public List Messages { get; } = []; + public List<(IInvokable Request, LocalRequestOptions Options)> Requests { get; } = []; public List Events { get; } = []; public Action? BeforeReceive { get; set; } public MessageFactory MessageFactory { get; } @@ -500,14 +526,15 @@ public TimerFixture() loggerFactory.CreateLogger(Arg.Any()).Returns(_logger); var grainId = GrainId.Create("timer-test", "one"); Grain.GrainId.Returns(grainId); - Grain.When(context => context.ReceiveMessage(Arg.Any())).Do(call => + Grain.When(context => context.ReceiveLocalRequest(Arg.Any(), Arg.Any(), Arg.Any())).Do(call => { + Assert.Null(call[1]); BeforeReceive?.Invoke(); - Messages.Add((Message)call[0]); + Requests.Add(((IInvokable)call[0], (LocalRequestOptions)call[2])); }); Grain.Address.Returns(GrainAddress.GetAddress(SiloAddress.New(System.Net.IPAddress.Loopback, 11111, 1), grainId, ActivationId.NewId())); MessageFactory = new(_services.GetRequiredService(), NullLogger.Instance, null!); - Registry = new(loggerFactory, Time, MessageFactory); + Registry = new(loggerFactory, Time); _subscription = GrainTimerEvents.AllEvents.Subscribe(this); } @@ -515,10 +542,10 @@ public IGrainTimer Register(Func callback, T stat TimeSpan period, bool interleave = false, bool keepAlive = false) => Registry.RegisterGrainTimer(Grain, callback, state, new(dueTime, period) { Interleave = interleave, KeepAlive = keepAlive }); - public async Task InvokeAsync(Message message, Exception? expectedException = null) + public async Task InvokeAsync((IInvokable Request, LocalRequestOptions Options) request, Exception? expectedException = null) { - RequestContextExtensions.Import(message.RequestContextData); - var invokable = Assert.IsAssignableFrom(message.BodyObject); + RequestContextExtensions.Import(null); + var invokable = request.Request; invokable.SetTarget(Grain); using var response = await invokable.Invoke(); Assert.Same(expectedException, response.Exception); diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 06a752b3f9e..b96dd26b557 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -9,7 +9,7 @@ namespace DefaultCluster.Tests.General; /// -/// Tests for , +/// Tests for , /// which runs a delegate as a request on the grain's activation. /// [TestCategory("BVT"), TestCategory("Scheduler")] @@ -139,8 +139,29 @@ public async Task NamedRequest_DescribesItselfToGrainCallFilter() await grain.StartNamedRequest(42); Assert.Equal("order-42", await grain.AwaitPendingRequest()); - Assert.Contains($"{RunRequestGrainBase.NamedRequestInterfaceName}.ProcessOrder", await grain.GetFilteredCalls()); - Assert.Equal(["argument:42"], await grain.GetLog()); + Assert.Contains("IGrainRequestInvoker.ProcessOrder", await grain.GetFilteredCalls()); + } + + [Fact] + public async Task MethodGroupRequest_IsNamedAfterMethod() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartMethodGroupRequest(); + + Assert.Equal("processed", await grain.AwaitPendingRequest()); + Assert.Contains("IGrainRequestInvoker.ProcessOrders", await grain.GetFilteredCalls()); + } + + [Fact] + public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletionSource() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + await grain.StartCustomInvokableRequest("custom"); + + Assert.Equal("custom:True", await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } [Fact] @@ -234,8 +255,8 @@ private static async Task WaitForLogEntryAsync(IRunRequestGrain grain, string en } /// -/// Tests that requests queued by -/// honor . +/// Tests how requests queued by +/// interact with . /// [TestCategory("BVT"), TestCategory("Scheduler")] [TestSuite("BVT")] @@ -252,7 +273,7 @@ protected override void ConfigureTestCluster(TestClusterBuilder builder) } [Fact] - public async Task RunRequest_BeyondHardLimit_FaultsWithLimitExceeded() + public async Task RunRequest_EnforcingQueueLimit_BeyondHardLimit_FaultsWithLimitExceeded() { var grain = GrainFactory.GetGrain(Guid.NewGuid()); @@ -264,6 +285,18 @@ public async Task RunRequest_BeyondHardLimit_FaultsWithLimitExceeded() Assert.Equal(["request-0", "request-1", "request-2"], await grain.GetLog()); } + [Fact] + public async Task RunRequest_ByDefault_IgnoresHardLimit() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + var admitted = await grain.QueueUnlimitedRequests(HardLimit * 4); + await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(HardLimit * 4, admitted); + Assert.Equal(HardLimit * 4, (await grain.GetLog()).Count); + } + [Fact] public async Task RunRequest_ConcurrentProducers_StayWithinHardLimit() { From 5bddd50682625a2a41fda002eecd02572f77d9fb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 12:22:50 -0700 Subject: [PATCH 09/31] docs(runtime): scope RunRequestAsync behavior list to default options --- docs/site/src/content/docs/grains/request-scheduling.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index ff04408564e..e10cfd8a318 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -71,13 +71,13 @@ Use only to isolate unavoidable synchron runs a callback as a request on the activation. Call it from any thread, such as a stream, messaging, or library callback, to apply results to grain state under the same scheduling as grain calls: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_from_callback"::: -The request: +With the default , the request: - Follows the grain's reentrancy rules and queues behind the current request on a non-reentrant grain. - Passes through incoming grain call filters, which see it as a method of `IGrainRequestInvoker` with no arguments. - Starts a new call chain. values from the calling thread don't flow into the callback. - Restarts the activation's idle collection period. -- Is admitted regardless of , like a grain timer tick. +- Bypasses . The returned task completes with the callback's result or exception. The callback receives the cancellation token passed to `RunRequestAsync`, and cancelling it while the request is queued cancels the request. If the activation deactivates before the request runs, the task faults with . From 1e37ad6eb71ef29607f1b6258a9e24fffd8b8f2b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 12:33:24 -0700 Subject: [PATCH 10/31] feat(runtime): add synchronous RunRequestAsync overloads --- .../content/docs/grains/request-scheduling.md | 2 +- .../Grains/RequestsAndVersioningSnippets.cs | 18 +--- .../Core/IGrainBase.cs | 100 ++++++++++++++++++ .../Orleans.Core.Abstractions.cs | 8 ++ .../TestGrainInterfaces/IRunRequestGrain.cs | 2 +- test/Grains/TestGrains/RunRequestGrain.cs | 40 +++---- .../RunRequestTests.cs | 8 +- 7 files changed, 132 insertions(+), 46 deletions(-) diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index e10cfd8a318..79a7bd4bdac 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -79,7 +79,7 @@ With the default , the request: - Restarts the activation's idle collection period. - Bypasses . -The returned task completes with the callback's result or exception. The callback receives the cancellation token passed to `RunRequestAsync`, and cancelling it while the request is queued cancels the request. If the activation deactivates before the request runs, the task faults with . +The callback can be synchronous, as in this example, or return a `Task` or `Task`. The returned task completes with the callback's result or exception. The callback receives the cancellation token passed to `RunRequestAsync`, and cancelling it while the request is queued cancels the request. If the activation deactivates before the request runs, the task faults with . Pass the grain and other values through the `state` argument and use a `static` lambda, so the call doesn't allocate a closure. On a non-reentrant grain, awaiting the returned task from another request on the same activation deadlocks, because the queued request waits for the current one to complete. diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index e460939159b..7ad7ecddbbf 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -262,11 +262,7 @@ public override Task OnDeactivateAsync(DeactivationReason reason, CancellationTo // Runs the update as a request on this activation, where it can safely access grain state. private Task OnPrice(decimal price, CancellationToken cancellationToken) => this.RunRequestAsync( - static (state, _) => - { - state.Grain._price = state.Price; - return Task.CompletedTask; - }, + static (state, _) => state.Grain._price = state.Price, (Grain: this, Price: price), cancellationToken); } @@ -305,11 +301,7 @@ public override Task OnDeactivateAsync(DeactivationReason reason, CancellationTo // Heartbeats don't extend the activation's lifetime, and they run between the turns of other requests. private Task OnHeartbeat(DateTimeOffset timestamp, CancellationToken cancellationToken) => this.RunRequestAsync( - static (state, _) => - { - state.Grain._lastSeen = state.Timestamp; - return Task.CompletedTask; - }, + static (state, _) => state.Grain._lastSeen = state.Timestamp, (Grain: this, Timestamp: timestamp), new LocalRequestOptions { KeepAlive = false, Interleave = true }, cancellationToken); @@ -382,11 +374,7 @@ public sealed class IngestGrain : Grain, IIngestGrain // when the activation's queue exceeds SiloMessagingOptions.MaxEnqueuedRequestsHardLimit. internal Task Ingest(int items, CancellationToken cancellationToken) => this.RunRequestAsync( - static (state, _) => - { - state.Grain._count += state.Items; - return Task.CompletedTask; - }, + static (state, _) => state.Grain._count += state.Items, (Grain: this, Items: items), new LocalRequestOptions { EnforceQueueLimit = true }, cancellationToken); diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index fcf6ff0cdd8..f3a9ebf23ae 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -141,6 +141,106 @@ public static Task RunRequestAsync(this IGrainBase grain, Func(grain.GrainContext, callback, state, options, cancellationToken, methodName); } + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + { + ArgumentNullException.ThrowIfNull(callback); + return RunRequestCore, object?>( + grain.GrainContext, + static (callback, cancellationToken) => + { + callback(cancellationToken); + return Task.CompletedTask; + }, + callback, + options, + cancellationToken, + methodName); + } + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// + /// The type of the parameter. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); + + /// + /// Runs as a request on this grain activation. + /// + /// The grain instance. + /// The callback to run. + /// The state passed to the callback. + /// The options for the request. + /// + /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. + /// + /// + /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . + /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. + /// + /// The type of the parameter. + /// A which completes with the outcome of the callback. + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + { + ArgumentNullException.ThrowIfNull(callback); + return RunRequestCore<(Action Callback, TState State), object?>( + grain.GrainContext, + static (state, cancellationToken) => + { + state.Callback(state.State, cancellationToken); + return Task.CompletedTask; + }, + (callback, state), + options, + cancellationToken, + methodName); + } + /// /// Runs as a request on this grain activation. /// diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index d4cfcdbb3af..0a9a5a849ce 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -1119,10 +1119,18 @@ public static void MigrateOnIdle(this IGrainBase grain) { } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, System.TimeSpan dueTime, System.TimeSpan period) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 3fc60641815..b485033e1ed 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -33,7 +33,7 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartRequestContextRequest(string value); - Task StartFaultingRequest(); + Task StartFaultingRequest(bool synchronous); Task StartCanceledRequest(bool cancelBeforeStart); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 71b6ea09dbf..b0fc7288245 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -46,11 +46,7 @@ public Task SetRequestFilterMode(RequestFilterMode mode) public async Task StartRequestDuringCall() { _log.Add("call-start"); - _pending = this.RunRequestAsync(_ => - { - _log.Add("request"); - return Task.CompletedTask; - }); + _pending = this.RunRequestAsync(_ => _log.Add("request")); await Task.Delay(200); _log.Add("call-end"); @@ -59,11 +55,7 @@ public async Task StartRequestDuringCall() public async Task StartInterleavingRequestDuringCall() { _log.Add("call-start"); - _pending = this.RunRequestAsync(_ => - { - _log.Add("request"); - return Task.CompletedTask; - }, new LocalRequestOptions { Interleave = true }); + _pending = this.RunRequestAsync(_ => _log.Add("request"), new LocalRequestOptions { Interleave = true }); await Task.Delay(200); _log.Add("call-end"); @@ -186,11 +178,7 @@ public async Task QueueRequestsConcurrently(int count) private static readonly LocalRequestOptions EnforceQueueLimit = new() { EnforceQueueLimit = true }; private Task QueueLoggedRequest(string entry, LocalRequestOptions options) => - this.RunRequestAsync(static (state, _) => - { - state.Grain._log.Add(state.Entry); - return Task.CompletedTask; - }, (Grain: this, Entry: entry), options); + this.RunRequestAsync(static (state, _) => state.Grain._log.Add(state.Entry), (Grain: this, Entry: entry), options); public Task StartRequestContextRequest(string value) { @@ -199,17 +187,21 @@ public Task StartRequestContextRequest(string value) return Task.CompletedTask; } - public Task StartFaultingRequest() + public Task StartFaultingRequest(bool synchronous) { - _pending = this.RunRequestAsync(static async _ => - { - await Task.Yield(); - throw new InvalidOperationException("Request failed."); - }); + _pending = synchronous + ? this.RunRequestAsync(Fail) + : this.RunRequestAsync(static async _ => + { + await Task.Yield(); + throw new InvalidOperationException("Request failed."); + }); return Task.CompletedTask; } + private static void Fail(CancellationToken cancellationToken) => throw new InvalidOperationException("Request failed."); + public Task StartCanceledRequest(bool cancelBeforeStart) { using var cancellation = new CancellationTokenSource(); @@ -218,11 +210,7 @@ public Task StartCanceledRequest(bool cancelBeforeStart) cancellation.Cancel(); } - _pending = this.RunRequestAsync(_ => - { - _log.Add("canceled-request"); - return Task.CompletedTask; - }, cancellation.Token); + _pending = this.RunRequestAsync(_ => _log.Add("canceled-request"), cancellation.Token); cancellation.Cancel(); return Task.CompletedTask; diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index b96dd26b557..113152ad4fb 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -96,12 +96,14 @@ public async Task RunRequest_StartsNewCallChain() Assert.Null(await grain.AwaitPendingRequest()); } - [Fact] - public async Task RunRequest_PropagatesCallbackException() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task RunRequest_PropagatesCallbackException(bool synchronous) { var grain = GrainFactory.GetGrain(Guid.NewGuid()); - await grain.StartFaultingRequest(); + await grain.StartFaultingRequest(synchronous); var exception = await Assert.ThrowsAsync(grain.AwaitPendingRequest); Assert.Equal("Request failed.", exception.Message); From 843dbadf922cc9682d1e32403c47590bf8213f61 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 12:50:15 -0700 Subject: [PATCH 11/31] refactor(runtime): remove RunRequestAsync methodName parameter --- .../content/docs/grains/request-scheduling.md | 12 +- .../Grains/RequestsAndVersioningSnippets.cs | 50 ------- .../Core/IGrainBase.cs | 139 +++++------------- .../Runtime/GrainRequest.cs | 14 +- .../Orleans.Core.Abstractions.cs | 24 +-- .../TestGrainInterfaces/IRunRequestGrain.cs | 3 - test/Grains/TestGrains/RunRequestGrain.cs | 16 -- .../Runtime/GrainRequestTests.cs | 28 +--- .../RunRequestTests.cs | 25 +--- 9 files changed, 69 insertions(+), 242 deletions(-) diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index 79a7bd4bdac..e5a044c3639 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -74,7 +74,7 @@ Use only to isolate unavoidable synchron With the default , the request: - Follows the grain's reentrancy rules and queues behind the current request on a non-reentrant grain. -- Passes through incoming grain call filters, which see it as a method of `IGrainRequestInvoker` with no arguments. +- Passes through incoming grain call filters, which observe it as `IGrainRequestInvoker.RunRequestAsync` with no arguments. - Starts a new call chain. values from the calling thread don't flow into the callback. - Restarts the activation's idle collection period. - Bypasses . @@ -91,16 +91,8 @@ Pass to change how the request is sch Set to `true` to apply backpressure to a producer. The request then counts against , and the returned task faults with when the activation's queue is full: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_queue_limit"::: -### Name the request - -Call filters, logs, and traces observe the request as `IGrainRequestInvoker.{methodName}`, and tracing names its activity `IGrainRequestInvoker/{methodName}`. When the callback is a method group, such as `this.RunRequestAsync(RefreshAsync)`, the method's name is used. Otherwise, the request is named `RunRequestAsync`. Pass `methodName` to choose the name: - -:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_named"::: -Call filters read the name through the call context: - -:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_filter"::: ### Deliver a custom request - delivers any to the activation with the same scheduling as `RunRequestAsync`. Call filters observe the request's own interface and method names. The activation completes the supplied with the request's response, or with an exception if the activation rejects the request: + delivers any to the activation with the same scheduling as `RunRequestAsync`. Call filters, logs, and traces observe the request's own interface name, method name, and arguments, so a custom request can describe the work to them. The activation completes the supplied with the request's response, or with an exception if the activation rejects the request: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="receive_local_request"::: diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index 7ad7ecddbbf..42bbad30e39 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -308,56 +308,6 @@ private Task OnHeartbeat(DateTimeOffset timestamp, CancellationToken cancellatio // } - [GenerateSerializer] - public sealed record Order([property: Id(0)] string Id, [property: Id(1)] decimal Total); - - public interface IOrderBookGrain : IGrainWithStringKey - { - ValueTask GetTotal(); - } - - public sealed class OrderBookGrain : Grain, IOrderBookGrain - { - private decimal _total; - - public ValueTask GetTotal() => ValueTask.FromResult(_total); - - private int Apply(IReadOnlyList orders) - { - foreach (var order in orders) - { - _total += order.Total; - } - - return orders.Count; - } - - // -// Call filters and tracing observe this request as IGrainRequestInvoker.ApplyOrders. -internal Task ApplyOrders(IReadOnlyList orders, CancellationToken cancellationToken) => - this.RunRequestAsync( - static (state, _) => Task.FromResult(state.Grain.Apply(state.Orders)), - (Grain: this, Orders: orders), - cancellationToken, - methodName: nameof(ApplyOrders)); - // - } - - // -public sealed class RequestLoggingFilter(ILogger logger) : IIncomingGrainCallFilter -{ - public async Task Invoke(IIncomingGrainCallContext context) - { - logger.LogInformation( - "Running {InterfaceName}.{MethodName}", - context.InterfaceName, - context.MethodName); - - await context.Invoke(); - } -} - // - public interface IIngestGrain : IGrainWithStringKey { ValueTask GetCount(); diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index f3a9ebf23ae..ee823b1c3ad 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -1,6 +1,4 @@ using System; -using System.Buffers; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -63,14 +61,10 @@ public static void DeactivateOnIdle(this IGrainBase grain) => /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -81,13 +75,9 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return RunRequestCore, object?>( @@ -95,8 +85,7 @@ public static Task RunRequestAsync(this IGrainBase grain, Func callback(cancellationToken), callback, options, - cancellationToken, - methodName); + cancellationToken); } /// @@ -108,15 +97,11 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -128,17 +113,13 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); - return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken, methodName); + return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken); } /// @@ -149,14 +130,10 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Action callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -167,13 +144,9 @@ public static Task RunRequestAsync(this IGrainBase grain, Action /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Action callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return RunRequestCore, object?>( @@ -185,8 +158,7 @@ public static Task RunRequestAsync(this IGrainBase grain, Action @@ -198,15 +170,11 @@ public static Task RunRequestAsync(this IGrainBase grain, Action /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -218,14 +186,10 @@ public static Task RunRequestAsync(this IGrainBase grain, Action /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + /// + public static Task RunRequestAsync(this IGrainBase grain, Action callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return RunRequestCore<(Action Callback, TState State), object?>( @@ -237,8 +201,7 @@ public static Task RunRequestAsync(this IGrainBase grain, Action @@ -249,15 +212,11 @@ public static Task RunRequestAsync(this IGrainBase grain, Action /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -268,14 +227,10 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); return RunRequestCore>, TResult>( @@ -283,8 +238,7 @@ public static Task RunRequestAsync(this IGrainBase grain, Func static (callback, cancellationToken) => callback(cancellationToken), callback, options, - cancellationToken, - methodName); + cancellationToken); } /// @@ -296,16 +250,12 @@ public static Task RunRequestAsync(this IGrainBase grain, Func /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// The type of the callback's result. /// A which completes with the outcome of the callback. - /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) => - RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken, methodName); + /// + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, CancellationToken cancellationToken = default) => + RunRequestAsync(grain, callback, state, default(LocalRequestOptions), cancellationToken); /// /// Runs as a request on this grain activation. @@ -317,10 +267,6 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// Cancels the request while it is queued. Once the callback starts, the token is passed to the callback. /// - /// - /// The method name which grain call filters observe for the request. Defaults to the name of the method passed as . - /// A value which is not a simple identifier, such as a lambda expression, is replaced by RunRequestAsync. - /// /// The type of the parameter. /// The type of the callback's result. /// A which completes with the outcome of the callback. @@ -328,7 +274,7 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// The request is scheduled with the activation's other requests. By default, it follows the grain's reentrancy rules: /// on a non-reentrant grain, the callback runs exclusively, the same as a grain method call. - /// The request passes through grain call filters as a method of IGrainRequestInvoker named by . + /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync. /// It starts a new call chain and, by default, keeps the activation alive. /// A call filter which fails or sets the result without invoking the request completes the returned task with that outcome. /// @@ -342,13 +288,14 @@ public static Task RunRequestAsync(this IGrainBase gra /// . /// If is set and the activation's queue exceeds the configured maximum number of enqueued requests, /// the returned task faults with . - /// To queue a custom , use . + /// To present the request to call filters and tracing with a different interface name, method name, or arguments, + /// queue a custom using . /// /// - public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default, [CallerArgumentExpression(nameof(callback))] string? methodName = null) + public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(callback); - return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken, methodName); + return RunRequestCore(grain.GrainContext, callback, state, options, cancellationToken); } private static Task RunRequestCore( @@ -356,8 +303,7 @@ private static Task RunRequestCore( Func callback, TState state, LocalRequestOptions options, - CancellationToken cancellationToken, - string? methodName) + CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { @@ -366,17 +312,12 @@ private static Task RunRequestCore( // The activation owns the request once it is queued and disposes it after invocation. #pragma warning disable CA2000 // Dispose objects before losing scope - var request = new GrainRequest(callback, state, IsMemberName(methodName) ? methodName : null, cancellationToken); + var request = new GrainRequest(callback, state, cancellationToken); #pragma warning restore CA2000 // Dispose objects before losing scope grainContext.ReceiveLocalRequest(request, request, options); return request.Task; } - // A simple method group, such as ProcessAsync, names the request. Other expressions, such as lambdas, use the default name. - private static bool IsMemberName(string? name) => name is { Length: > 0 } && !name.AsSpan().ContainsAnyExcept(MemberNameChars); - - private static readonly SearchValues MemberNameChars = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"); - /// /// Creates a grain timer. /// diff --git a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs index e4915ee468f..0ce6ed07072 100644 --- a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs @@ -7,27 +7,25 @@ namespace Orleans.Runtime; /// -/// A request queued on an activation by . +/// A request queued on an activation by . /// /// /// The request is its own completion source, so queuing a request allocates only this object and its message. /// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. /// When a call filter completes the invocation without running the callback, the task reflects the filter's outcome. -/// Call filters observe the request as , with the method name supplied by the caller, if any. +/// Call filters observe the request as . /// internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest, IResponseCompletionSource { private readonly Func _callback; private readonly TState _state; - private readonly string? _methodName; private readonly CancellationTokenRegistration _registration; - public GrainRequest(Func callback, TState state, string? methodName, CancellationToken cancellationToken) + public GrainRequest(Func callback, TState state, CancellationToken cancellationToken) : base(TaskCreationOptions.RunContinuationsAsynchronously) { _callback = callback; _state = state; - _methodName = methodName; _registration = cancellationToken.UnsafeRegister( static (state, cancellationToken) => ((GrainRequest)state!).TrySetCanceled(cancellationToken), this); @@ -133,11 +131,11 @@ public void SetTarget(ITargetHolder holder) public void SetArgument(int index, object? value) => throw new ArgumentOutOfRangeException(nameof(index)); - public string GetMethodName() => _methodName ?? nameof(IGrainRequestInvoker.RunRequestAsync); + public string GetMethodName() => nameof(IGrainRequestInvoker.RunRequestAsync); public string GetInterfaceName() => nameof(IGrainRequestInvoker); - public string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{GetMethodName()}"; + public string GetActivityName() => $"{nameof(IGrainRequestInvoker)}/{nameof(IGrainRequestInvoker.RunRequestAsync)}"; public MethodInfo GetMethod() => IGrainRequestInvoker.MethodInfo; @@ -153,7 +151,7 @@ public void Dispose() } /// -/// The interface which call filters observe for requests queued by . +/// The interface which call filters observe for requests queued by . /// internal interface IGrainRequestInvoker { diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 0a9a5a849ce..cf9e01d0816 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -1119,29 +1119,29 @@ public static void MigrateOnIdle(this IGrainBase grain) { } public static Runtime.IGrainTimer RegisterGrainTimer(this IGrainBase grain, System.Func callback, TState state, System.TimeSpan dueTime, System.TimeSpan period) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Action callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, Runtime.LocalRequestOptions options, System.Threading.CancellationToken cancellationToken = default) { throw null; } - public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default, string? methodName = null) { throw null; } + public static System.Threading.Tasks.Task RunRequestAsync(this IGrainBase grain, System.Func> callback, TState state, System.Threading.CancellationToken cancellationToken = default) { throw null; } } public delegate System.Threading.Tasks.Task GrainCallFilterDelegate(IGrainCallContext context); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index b485033e1ed..33dc4ec26f5 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -8,9 +8,6 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartInterleavingRequestDuringCall(); - Task StartNamedRequest(int orderId); - - Task StartMethodGroupRequest(); Task StartCustomInvokableRequest(string value); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index b0fc7288245..7b023aaf967 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -61,22 +61,6 @@ public async Task StartInterleavingRequestDuringCall() _log.Add("call-end"); } - public Task StartNamedRequest(int orderId) - { - _pending = this.RunRequestAsync( - static (orderId, _) => Task.FromResult($"order-{orderId}"), - orderId, - methodName: "ProcessOrder"); - return Task.CompletedTask; - } - - public Task StartMethodGroupRequest() - { - _pending = this.RunRequestAsync(ProcessOrders); - return Task.CompletedTask; - } - - private Task ProcessOrders(CancellationToken cancellationToken) => Task.FromResult("processed"); public Task QueueUnlimitedRequests(int count) { diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index 44733e0cec9..55f317127f6 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -5,7 +5,7 @@ namespace NonSilo.Tests.Runtime; /// -/// Tests for the request which queues. +/// Tests for the request which queues. /// [TestCategory("BVT"), TestCategory("Scheduler")] public class GrainRequestTests @@ -24,7 +24,6 @@ public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRun return Task.CompletedTask; }, null, - null, cancellation.Token); // Cancellation runs callbacks in reverse registration order, so this one runs before the request's. @@ -51,7 +50,7 @@ public async Task Invoke_CanceledWhileEarlierCancellationCallbackRuns_DoesNotRun public async Task Complete_CallbackFailure_IsReportedByRequest() { var exception = new InvalidOperationException("Request failed."); - var request = new GrainRequest((_, _) => Task.FromException(exception), null, null, default); + var request = new GrainRequest((_, _) => Task.FromException(exception), null, default); var response = await request.Invoke(); @@ -62,7 +61,7 @@ public async Task Complete_CallbackFailure_IsReportedByRequest() [Fact] public async Task Complete_FilterFailureAfterCallback_IsNotReportedByRequest() { - var request = new GrainRequest((_, _) => Task.FromResult("result"), null, null, default); + var request = new GrainRequest((_, _) => Task.FromResult("result"), null, default); await request.Invoke(); @@ -81,7 +80,6 @@ public async Task Complete_FilterFailureWithoutCallback_IsReportedByRequest() return Task.CompletedTask; }, null, - null, default); Assert.True(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); @@ -97,7 +95,7 @@ public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTh for (var i = 0; i < 2_000; i++) { using var cancellation = new CancellationTokenSource(); - var request = new GrainRequest((_, _) => Task.CompletedTask, null, null, cancellation.Token); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, cancellation.Token); using var start = new Barrier(2); var cancel = Task.Run(() => { @@ -126,7 +124,6 @@ public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() return Task.CompletedTask; }, null, - null, cancellation.Token); var exception = new OrleansMessageRejectionException("Rejected."); @@ -139,9 +136,9 @@ public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() } [Fact] - public void Describe_WithoutMethodName_DescribesRequestAsGrainRequestInvoker() + public void Describe_DescribesRequestAsGrainRequestInvoker() { - var request = new GrainRequest((_, _) => Task.CompletedTask, null, null, TestContext.Current.CancellationToken); + var request = new GrainRequest((_, _) => Task.CompletedTask, null, TestContext.Current.CancellationToken); Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); Assert.Equal("RunRequestAsync", request.GetMethodName()); @@ -151,21 +148,10 @@ public void Describe_WithoutMethodName_DescribesRequestAsGrainRequestInvoker() Assert.Throws(() => request.GetArgument(0)); } - [Fact] - public void Describe_WithMethodName_UsesMethodName() - { - var request = new GrainRequest((_, _) => Task.CompletedTask, null, "Drain", TestContext.Current.CancellationToken); - - Assert.Equal("IGrainRequestInvoker", request.GetInterfaceName()); - Assert.Equal("Drain", request.GetMethodName()); - Assert.Equal("IGrainRequestInvoker/Drain", request.GetActivityName()); - Assert.Equal(0, request.GetArgumentCount()); - } - [Fact] public async Task CompletionSource_Complete_CompletesTask() { - var request = new GrainRequest((_, _) => Task.FromResult("unused"), null, null, TestContext.Current.CancellationToken); + var request = new GrainRequest((_, _) => Task.FromResult("unused"), null, TestContext.Current.CancellationToken); ((IResponseCompletionSource)request).Complete(Response.FromResult("filtered")); diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 113152ad4fb..1f35a15e677 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -9,7 +9,7 @@ namespace DefaultCluster.Tests.General; /// -/// Tests for , +/// Tests for , /// which runs a delegate as a request on the grain's activation. /// [TestCategory("BVT"), TestCategory("Scheduler")] @@ -133,27 +133,6 @@ public async Task RunRequest_PassesThroughGrainCallFilter() Assert.Contains("IGrainRequestInvoker.RunRequestAsync", await grain.GetFilteredCalls()); } - [Fact] - public async Task NamedRequest_DescribesItselfToGrainCallFilter() - { - var grain = GrainFactory.GetGrain(Guid.NewGuid()); - - await grain.StartNamedRequest(42); - - Assert.Equal("order-42", await grain.AwaitPendingRequest()); - Assert.Contains("IGrainRequestInvoker.ProcessOrder", await grain.GetFilteredCalls()); - } - - [Fact] - public async Task MethodGroupRequest_IsNamedAfterMethod() - { - var grain = GrainFactory.GetGrain(Guid.NewGuid()); - - await grain.StartMethodGroupRequest(); - - Assert.Equal("processed", await grain.AwaitPendingRequest()); - Assert.Contains("IGrainRequestInvoker.ProcessOrders", await grain.GetFilteredCalls()); - } [Fact] public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletionSource() @@ -257,7 +236,7 @@ private static async Task WaitForLogEntryAsync(IRunRequestGrain grain, string en } /// -/// Tests how requests queued by +/// Tests how requests queued by /// interact with . /// [TestCategory("BVT"), TestCategory("Scheduler")] From 97fe83fcf64d58f888016df103571ea8472c32ba Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 14:39:12 -0700 Subject: [PATCH 12/31] fix(runtime): complete rejected local requests outside the activation lock --- .../Core/IGrainBase.cs | 11 +++- .../Activation/IGrainContextActivator.cs | 4 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 48 ++++++++++---- .../Core/InsideRuntimeClient.cs | 2 +- .../Core/LocalRequestCompletion.cs | 7 +- .../TestGrainInterfaces/IRunRequestGrain.cs | 12 ++++ test/Grains/TestGrains/RunRequestGrain.cs | 66 ++++++++++++++++++- .../Runtime/GrainRequestTests.cs | 40 +++++++++++ .../RunRequestTests.cs | 35 ++++++++++ 9 files changed, 207 insertions(+), 18 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index ee823b1c3ad..ae206e18373 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -314,7 +314,16 @@ private static Task RunRequestCore( #pragma warning disable CA2000 // Dispose objects before losing scope var request = new GrainRequest(callback, state, cancellationToken); #pragma warning restore CA2000 // Dispose objects before losing scope - grainContext.ReceiveLocalRequest(request, request, options); + try + { + grainContext.ReceiveLocalRequest(request, request, options); + } + catch (Exception exception) + { + // Rejecting the request releases its cancellation registration and reports the failure through the returned task. + ((ILocalRequest)request).Reject(exception); + } + return request.Task; } diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 00be05eab51..94b215115c6 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -374,9 +374,11 @@ internal class GrainCanInterleave public List MayInterleavePredicates { get; } = new List(); public bool MayInterleave(object? instance, Message message) { + // Predicates observe the request rather than its completion envelope. + var request = LocalRequestCompletion.GetRequest(message.BodyObject); foreach (var predicate in this.MayInterleavePredicates) { - if (predicate!.Invoke(instance!, (IInvokable)message.BodyObject!)) return true; + if (predicate!.Invoke(instance!, request)) return true; } return false; diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index ac4f11999d9..2a2f6f14c6e 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -488,7 +488,7 @@ internal int GetRequestCount() } } - internal List DequeueAllWaitingRequests() + private List DequeueAllWaitingRequests(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) { lock (_lock) { @@ -498,7 +498,11 @@ internal List DequeueAllWaitingRequests() // Local-only messages are not allowed to escape the activation. if (message.IsLocalOnly) { - (message.BodyObject as ILocalRequest)?.Reject(new OrleansMessageRejectionException($"Activation {this} is no longer valid.")); + if (message.BodyObject is ILocalRequest request) + { + (rejectedLocalRequests ??= []).Add((request, new OrleansMessageRejectionException($"Activation {this} is no longer valid."))); + } + continue; } @@ -1139,6 +1143,26 @@ private async Task RunMessageLoop() } void ProcessPendingRequests() + { + List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests = null; + try + { + ProcessPendingRequestsCore(ref rejectedLocalRequests); + } + finally + { + // Rejection can run application code which queues further requests, so it runs once the queue is consistent and unlocked. + if (rejectedLocalRequests is not null) + { + foreach (var (request, exception) in rejectedLocalRequests) + { + request.Reject(exception); + } + } + } + } + + void ProcessPendingRequestsCore(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) { var i = 0; @@ -1158,7 +1182,7 @@ void ProcessPendingRequests() // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - ProcessRequestsToInvalidActivation(); + ProcessRequestsToInvalidActivation(ref rejectedLocalRequests); break; } @@ -1218,9 +1242,9 @@ void ProcessPendingRequests() { _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); } - else + else if (message.BodyObject is ILocalRequest request) { - (message.BodyObject as ILocalRequest)?.Reject(exception); + (rejectedLocalRequests ??= []).Add((request, exception)); } _waitingRequests.RemoveAt(i); @@ -1253,7 +1277,7 @@ void RecordRunning(Message message, bool isInterleavable) _busyDuration = stopwatch; } - void ProcessRequestsToInvalidActivation() + void ProcessRequestsToInvalidActivation(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) { if (State is ActivationState.Creating or ActivationState.Activating) { @@ -1289,12 +1313,12 @@ void ProcessRequestsToInvalidActivation() { // Either this was a duplicate activation or it was at some point successfully activated // Forward all pending messages - RerouteAllQueuedMessages(); + RerouteAllQueuedMessages(ref rejectedLocalRequests); } else { // Reject all pending messages - RejectAllQueuedMessages(); + RejectAllQueuedMessages(ref rejectedLocalRequests); } } @@ -1737,11 +1761,11 @@ private void ReceiveRequest(Message message, bool enforceQueueLimit) /// /// Rejects all messages enqueued for the provided activation. /// - private void RejectAllQueuedMessages() + private void RejectAllQueuedMessages(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) { lock (_lock) { - List msgs = DequeueAllWaitingRequests(); + List msgs = DequeueAllWaitingRequests(ref rejectedLocalRequests); if (msgs == null || msgs.Count <= 0) return; LogRejectAllQueuedMessages(_shared.Logger, msgs.Count, this); @@ -1756,11 +1780,11 @@ private void RejectAllQueuedMessages() } } - private void RerouteAllQueuedMessages() + private void RerouteAllQueuedMessages(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) { lock (_lock) { - List msgs = DequeueAllWaitingRequests(); + List msgs = DequeueAllWaitingRequests(ref rejectedLocalRequests); if (msgs is not { Count: > 0 }) { return; diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 82d6d055901..f1c1f46945c 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -328,7 +328,7 @@ public async Task Invoke(IGrainContext target, Message message) case IInvokable body: { // Call filters observe the request rather than its completion envelope. - var invokable = body is LocalRequestCompletion envelope ? envelope.Request : body; + var invokable = LocalRequestCompletion.GetRequest(body); invokable.SetTarget(target); CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); diff --git a/src/Orleans.Runtime/Core/LocalRequestCompletion.cs b/src/Orleans.Runtime/Core/LocalRequestCompletion.cs index 89ca94aeb5e..207b589c290 100644 --- a/src/Orleans.Runtime/Core/LocalRequestCompletion.cs +++ b/src/Orleans.Runtime/Core/LocalRequestCompletion.cs @@ -7,12 +7,17 @@ namespace Orleans.Runtime; /// The body of a local request whose response is reported to a completion source other than the request itself. /// /// -/// Message processing observes this envelope as the request, and grain call filters observe . +/// Message processing observes this envelope as the request. Grain call filters and [MayInterleave] predicates observe . /// internal sealed class LocalRequestCompletion(IInvokable request, IResponseCompletionSource completion) : IInvokable, ILocalRequest { public IInvokable Request { get; } = request; + /// + /// Returns the request carried by a message body, unwrapping the completion envelope if present. + /// + public static IInvokable GetRequest(object? body) => body is LocalRequestCompletion envelope ? envelope.Request : (IInvokable)body!; + public bool Complete(Response response) { completion.Complete(response); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 33dc4ec26f5..aaf972ef875 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -56,6 +56,18 @@ public interface IReentrantRunRequestGrain : IRunRequestGrain { } +/// +/// A non-reentrant grain whose [MayInterleave] predicate admits custom local requests. +/// +public interface IMayInterleaveRunRequestGrain : IRunRequestGrain +{ + /// + /// Queues a custom request and returns its result once it runs, which requires interleaving with a blocking request. + /// + [AlwaysInterleave] + Task RunCustomRequestDuringBlockingRequest(bool observeCompletion); +} + /// /// How the grain's call filter handles requests queued by RunRequestAsync. /// diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 7b023aaf967..04f2141aef1 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -234,12 +234,53 @@ public Task DeactivateAndRetainInstance() return Task.FromResult("ran"); }, RetainedInstances[key]); + + /// + /// Queues a custom request on the retained instance whose rejection queues a second request from within the completion. + /// + /// The number of times the first request completed and the second request's response. + public static async Task<(int FirstCompletions, Response SecondResponse)> QueueChainedRequestsOnRetainedInstance(Guid key) + { + var context = ((IGrainBase)RetainedInstances[key]).GrainContext; + var second = new ChainingCompletion(context, next: null); + var first = new ChainingCompletion(context, second); + context.ReceiveLocalRequest(new CustomRequest("first"), first, default); + var secondResponse = await second.Response; + await first.Response; + return (first.Completions, secondResponse); + } +} + +/// +/// A completion source which queues a request completed by each time it completes. +/// +internal sealed class ChainingCompletion(IGrainContext context, ChainingCompletion? next) : IResponseCompletionSource +{ + private readonly TaskCompletionSource _response = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _completions; + + public Task Response => _response.Task; + + public int Completions => Volatile.Read(ref _completions); + + public void Complete(Response value) + { + Interlocked.Increment(ref _completions); + if (next is not null) + { + context.ReceiveLocalRequest(new CustomRequest("next"), next, default); + } + + _response.TrySetResult(value); + } + + public void Complete() => Complete(Orleans.Serialization.Invocation.Response.Completed); } /// /// A hand-written request delivered through . /// -internal sealed class CustomRequest(string value) : Request +internal sealed class CustomRequest(string value, TaskCompletionSource? invoked = null) : Request { public const string InterfaceName = "ICustomRequests"; public const string MethodName = "Apply"; @@ -249,7 +290,12 @@ internal sealed class CustomRequest(string value) : Request public override void SetTarget(ITargetHolder holder) => _grain = (RunRequestGrainBase?)holder.GetTarget(); - protected override ValueTask InvokeInner() => new($"{value}:{_grain is not null}"); + protected override ValueTask InvokeInner() + { + var result = $"{value}:{_grain is not null}"; + invoked?.TrySetResult(result); + return new(result); + } public override void Dispose() { } @@ -272,3 +318,19 @@ public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReent public sealed class ReentrantRunRequestGrain : RunRequestGrainBase, IReentrantRunRequestGrain { } + +[MayInterleave(nameof(CanInterleave))] +public sealed class MayInterleaveRunRequestGrain : RunRequestGrainBase, IMayInterleaveRunRequestGrain +{ + public static bool CanInterleave(IInvokable request) => request is CustomRequest; + + public async Task RunCustomRequestDuringBlockingRequest(bool observeCompletion) + { + var invoked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completion = observeCompletion ? new ResponseCompletionSource() : null; + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CustomRequest("custom", invoked), completion, default); + + var result = await invoked.Task.WaitAsync(TimeSpan.FromSeconds(10)); + return completion is null ? result : await completion.AsValueTask(); + } +} diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index 55f317127f6..e5b116176f1 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -1,3 +1,5 @@ +using System.Reflection; +using System.Runtime.CompilerServices; using Orleans.Runtime; using Orleans.Serialization.Invocation; using Xunit; @@ -148,6 +150,44 @@ public void Describe_DescribesRequestAsGrainRequestInvoker() Assert.Throws(() => request.GetArgument(0)); } + [Fact] + public async Task RunRequestAsync_SubmissionFails_FaultsTaskAndReleasesRequest() + { + using var cancellation = new CancellationTokenSource(); + var grain = new ThrowingGrain(); + + var (task, state) = StartRequest(grain, cancellation.Token); + + await Assert.ThrowsAsync(() => task); + + // The uncanceled token no longer retains the request or its state. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Assert.False(state.IsAlive); + GC.KeepAlive(cancellation); + + [MethodImpl(MethodImplOptions.NoInlining)] + static (Task Task, WeakReference State) StartRequest(IGrainBase grain, CancellationToken cancellationToken) + { + var state = new object(); + return (grain.RunRequestAsync(static (_, _) => { }, state, cancellationToken), new WeakReference(state)); + } + } + + /// + /// A grain whose context rejects every local request. + /// + private sealed class ThrowingGrain : IGrainBase + { + public IGrainContext GrainContext { get; } = DispatchProxy.Create(); + } + + public class ThrowingContext : DispatchProxy + { + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) => throw new NotSupportedException(targetMethod?.Name); + } + [Fact] public async Task CompletionSource_Complete_CompletesTask() { diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 1f35a15e677..b59b0d25ad7 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -145,6 +145,21 @@ public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletio Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReceiveLocalRequest_MayInterleavePredicate_ObservesCustomRequest(bool observeCompletion) + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.StartBlockingRequest(); + await WaitForLogEntryAsync(grain, "request-start"); + + Assert.Equal("custom:True", await grain.RunCustomRequestDuringBlockingRequest(observeCompletion)); + + await grain.ReleaseBlockingRequest(); + await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + [Fact] public async Task NonReentrantGrain_InterleavingRequestRunsDuringCurrentCall() { @@ -209,6 +224,26 @@ public async Task RunRequest_OnDeactivatedActivation_IsRejected() await grain.DeactivateAndRetainInstance(); // Requests queued while the activation is still deactivating run; once it is invalid, they are rejected. + await WaitForRejectionAsync(key); + } + + [Fact] + public async Task ReceiveLocalRequest_RejectionCompletionQueuesRequest_RejectsEachRequestOnce() + { + var key = Guid.NewGuid(); + var grain = GrainFactory.GetGrain(key); + await grain.DeactivateAndRetainInstance(); + await WaitForRejectionAsync(key); + + // The first request's rejection queues the second request while the activation drains its queue. + var (firstCompletions, secondResponse) = await RunRequestGrainBase.QueueChainedRequestsOnRetainedInstance(key).WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal(1, firstCompletions); + Assert.IsType(secondResponse.Exception); + } + + private static async Task WaitForRejectionAsync(Guid key) + { using var timeout = new CancellationTokenSource(Timeout); while (true) { From fee59da5962ecdb2088331f345be594026ae3629 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 14:46:52 -0700 Subject: [PATCH 13/31] fix(runtime): keep caller grain cancellation tokens on local requests --- .../Core/InsideRuntimeClient.cs | 8 ++- .../TestGrainInterfaces/IRunRequestGrain.cs | 2 + test/Grains/TestGrains/RunRequestGrain.cs | 58 +++++++++++++++++++ .../RunRequestTests.cs | 8 +++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index f1c1f46945c..c9a4de39e4c 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -331,7 +331,13 @@ public async Task Invoke(IGrainContext target, Message message) var invokable = LocalRequestCompletion.GetRequest(body); invokable.SetTarget(target); - CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); + // A local request carries the caller's own tokens, which are already linked to their sources. + // Tokens deserialized from the wire are replaced with tokens registered on the activation. + if (!message.IsLocalOnly) + { + CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); + } + if (GrainCallFilters is { Count: > 0 } || target.GrainInstance is IIncomingGrainCallFilter) { var invoker = new GrainMethodInvoker(message, target, invokable, GrainCallFilters, this.interfaceToImplementationMapping, this.responseCopier); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index aaf972ef875..0a3ef726fc9 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -11,6 +11,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartCustomInvokableRequest(string value); + Task RunCancellableCustomRequest(); + Task QueueUnlimitedRequests(int count); Task StartBlockingRequest(); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 04f2141aef1..0384e47d5cf 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -82,6 +82,19 @@ public Task StartCustomInvokableRequest(string value) return Task.CompletedTask; } + public async Task RunCancellableCustomRequest() + { + using var source = new GrainCancellationTokenSource(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completion = new ResponseCompletionSource(); + + // The request interleaves so that it runs while this call waits for it. + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CancellableRequest(source.Token, started), completion, new LocalRequestOptions { Interleave = true }); + await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await source.Cancel(); + return await completion.AsValueTask().AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + } + public Task StartBlockingRequest() { var gate = _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -310,6 +323,51 @@ public override void Dispose() { } public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; } +/// +/// A hand-written request which waits for its argument to be canceled. +/// +internal sealed class CancellableRequest(GrainCancellationToken token, TaskCompletionSource started) : Request +{ + private GrainCancellationToken _token = token; + private RunRequestGrainBase? _grain; + + public override object GetTarget() => _grain!; + + public override void SetTarget(ITargetHolder holder) => _grain = (RunRequestGrainBase?)holder.GetTarget(); + + protected override async ValueTask InvokeInner() + { + started.TrySetResult(); + try + { + await Task.Delay(Timeout.Infinite, _token.CancellationToken); + return "completed"; + } + catch (OperationCanceledException) + { + return "canceled"; + } + } + + public override int GetArgumentCount() => 1; + + public override object GetArgument(int index) => index == 0 ? _token : throw new ArgumentOutOfRangeException(nameof(index)); + + public override void SetArgument(int index, object value) => _token = index == 0 ? (GrainCancellationToken)value : throw new ArgumentOutOfRangeException(nameof(index)); + + public override void Dispose() { } + + public override string GetMethodName() => "WaitForCancellation"; + + public override string GetInterfaceName() => CustomRequest.InterfaceName; + + public override string GetActivityName() => $"{CustomRequest.InterfaceName}/WaitForCancellation"; + + public override Type GetInterfaceType() => typeof(IRunRequestGrain); + + public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; +} + public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReentrantRunRequestGrain { } diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index b59b0d25ad7..91ea0a81ca3 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -145,6 +145,14 @@ public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletio Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } + [Fact] + public async Task ReceiveLocalRequest_CustomInvokable_ObservesCallerGrainCancellationToken() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + Assert.Equal("canceled", await grain.RunCancellableCustomRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + [Theory] [InlineData(true)] [InlineData(false)] From 05916de43239f76be19e7f518d872840f10e8338 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 14:49:01 -0700 Subject: [PATCH 14/31] Revert "fix(runtime): keep caller grain cancellation tokens on local requests" This reverts commit 145d9f7103fd638855430ef2035f9ec8bf2f8018. --- .../Core/InsideRuntimeClient.cs | 8 +-- .../TestGrainInterfaces/IRunRequestGrain.cs | 2 - test/Grains/TestGrains/RunRequestGrain.cs | 58 ------------------- .../RunRequestTests.cs | 8 --- 4 files changed, 1 insertion(+), 75 deletions(-) diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index c9a4de39e4c..f1c1f46945c 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -331,13 +331,7 @@ public async Task Invoke(IGrainContext target, Message message) var invokable = LocalRequestCompletion.GetRequest(body); invokable.SetTarget(target); - // A local request carries the caller's own tokens, which are already linked to their sources. - // Tokens deserialized from the wire are replaced with tokens registered on the activation. - if (!message.IsLocalOnly) - { - CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); - } - + CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); if (GrainCallFilters is { Count: > 0 } || target.GrainInstance is IIncomingGrainCallFilter) { var invoker = new GrainMethodInvoker(message, target, invokable, GrainCallFilters, this.interfaceToImplementationMapping, this.responseCopier); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 0a3ef726fc9..aaf972ef875 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -11,8 +11,6 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartCustomInvokableRequest(string value); - Task RunCancellableCustomRequest(); - Task QueueUnlimitedRequests(int count); Task StartBlockingRequest(); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 0384e47d5cf..04f2141aef1 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -82,19 +82,6 @@ public Task StartCustomInvokableRequest(string value) return Task.CompletedTask; } - public async Task RunCancellableCustomRequest() - { - using var source = new GrainCancellationTokenSource(); - var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var completion = new ResponseCompletionSource(); - - // The request interleaves so that it runs while this call waits for it. - ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CancellableRequest(source.Token, started), completion, new LocalRequestOptions { Interleave = true }); - await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); - await source.Cancel(); - return await completion.AsValueTask().AsTask().WaitAsync(TimeSpan.FromSeconds(10)); - } - public Task StartBlockingRequest() { var gate = _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -323,51 +310,6 @@ public override void Dispose() { } public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; } -/// -/// A hand-written request which waits for its argument to be canceled. -/// -internal sealed class CancellableRequest(GrainCancellationToken token, TaskCompletionSource started) : Request -{ - private GrainCancellationToken _token = token; - private RunRequestGrainBase? _grain; - - public override object GetTarget() => _grain!; - - public override void SetTarget(ITargetHolder holder) => _grain = (RunRequestGrainBase?)holder.GetTarget(); - - protected override async ValueTask InvokeInner() - { - started.TrySetResult(); - try - { - await Task.Delay(Timeout.Infinite, _token.CancellationToken); - return "completed"; - } - catch (OperationCanceledException) - { - return "canceled"; - } - } - - public override int GetArgumentCount() => 1; - - public override object GetArgument(int index) => index == 0 ? _token : throw new ArgumentOutOfRangeException(nameof(index)); - - public override void SetArgument(int index, object value) => _token = index == 0 ? (GrainCancellationToken)value : throw new ArgumentOutOfRangeException(nameof(index)); - - public override void Dispose() { } - - public override string GetMethodName() => "WaitForCancellation"; - - public override string GetInterfaceName() => CustomRequest.InterfaceName; - - public override string GetActivityName() => $"{CustomRequest.InterfaceName}/WaitForCancellation"; - - public override Type GetInterfaceType() => typeof(IRunRequestGrain); - - public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; -} - public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReentrantRunRequestGrain { } diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 91ea0a81ca3..b59b0d25ad7 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -145,14 +145,6 @@ public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletio Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } - [Fact] - public async Task ReceiveLocalRequest_CustomInvokable_ObservesCallerGrainCancellationToken() - { - var grain = GrainFactory.GetGrain(Guid.NewGuid()); - - Assert.Equal("canceled", await grain.RunCancellableCustomRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken)); - } - [Theory] [InlineData(true)] [InlineData(false)] From 1c7acdec53ae6dbe9021b8ed1c3f5bb051e5c227 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 15:15:42 -0700 Subject: [PATCH 15/31] refactor(runtime): make ILocalRequest the local request and its completion --- .../content/docs/grains/request-scheduling.md | 2 +- .../Grains/RequestsAndVersioningSnippets.cs | 19 +++-- .../Core/IGrainBase.cs | 9 ++- .../Core/IGrainContext.cs | 12 +-- .../Core/ILocalRequest.cs | 31 ++++++++ .../Runtime/GrainRequest.cs | 24 +----- src/Orleans.Core/Messaging/MessageFactory.cs | 5 +- .../Activation/IGrainContextActivator.cs | 4 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 18 ++--- .../Core/InsideRuntimeClient.cs | 31 ++++---- .../Core/LocalRequestCompletion.cs | 60 --------------- src/Orleans.Runtime/Core/SystemTarget.cs | 4 +- src/Orleans.Runtime/Timers/GrainTimer.cs | 7 +- .../Orleans.Core.Abstractions.cs | 7 +- src/api/Orleans.Runtime/Orleans.Runtime.cs | 2 +- .../TestGrainInterfaces/IRunRequestGrain.cs | 2 +- test/Grains/TestGrains/RunRequestGrain.cs | 76 ++++++++----------- .../Runtime/GrainRequestTests.cs | 8 +- .../Runtime/GrainTimerTests.cs | 11 ++- .../RunRequestTests.cs | 10 +-- 20 files changed, 142 insertions(+), 200 deletions(-) create mode 100644 src/Orleans.Core.Abstractions/Core/ILocalRequest.cs delete mode 100644 src/Orleans.Runtime/Core/LocalRequestCompletion.cs diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index e5a044c3639..1ef7241b013 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -93,6 +93,6 @@ Set to `true` to ap :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_queue_limit"::: ### Deliver a custom request - delivers any to the activation with the same scheduling as `RunRequestAsync`. Call filters, logs, and traces observe the request's own interface name, method name, and arguments, so a custom request can describe the work to them. The activation completes the supplied with the request's response, or with an exception if the activation rejects the request: + delivers an to the activation with the same scheduling as `RunRequestAsync`. Call filters, logs, and traces observe the request's own interface name, method name, and arguments, so a custom request can describe the work to them. The activation passes the request's response to , or an exception if the activation rejects the request: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="receive_local_request"::: diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index 42bbad30e39..f3f13e178a2 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -335,10 +335,17 @@ internal Task Ingest(int items, CancellationToken cancellationToken) => // // A request which increments IngestGrain's counter. Call filters observe it as IIngestRequests.Increment. -internal sealed class IncrementRequest : Request +internal sealed class IncrementRequest : Request, ILocalRequest { + private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously); private IngestGrain? _grain; + public Task Completion => _completion.Task; + + // Receives the outcome once the request and its call filters complete, or when the activation rejects the request. + public bool Complete(Response response) => + response.Exception is { } exception ? _completion.TrySetException(exception) : _completion.TrySetResult(); + public override object GetTarget() => _grain!; public override void SetTarget(ITargetHolder holder) => _grain = (IngestGrain?)holder.GetTarget(); @@ -364,12 +371,12 @@ public override void Dispose() { } public static class IngestRequests { - // Delivers the request to the activation and completes the returned task with its response. - public static ValueTask Increment(IGrainContext activation) + // Delivers the request to the activation and returns a task which completes with the request's outcome. + public static Task Increment(IGrainContext activation) { - var completion = new ResponseCompletionSource(); - activation.ReceiveLocalRequest(new IncrementRequest(), completion, default); - return completion.AsVoidValueTask(); + var request = new IncrementRequest(); + activation.ReceiveLocalRequest(request, default); + return request.Completion; } } // diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index ae206e18373..1ef7a98b7f1 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime; +using Orleans.Serialization.Invocation; using Orleans.Timers; namespace Orleans @@ -289,7 +290,7 @@ public static Task RunRequestAsync(this IGrainBase gra /// If is set and the activation's queue exceeds the configured maximum number of enqueued requests, /// the returned task faults with . /// To present the request to call filters and tracing with a different interface name, method name, or arguments, - /// queue a custom using . + /// queue a custom using . /// /// public static Task RunRequestAsync(this IGrainBase grain, Func> callback, TState state, LocalRequestOptions options, CancellationToken cancellationToken = default) @@ -316,12 +317,12 @@ private static Task RunRequestCore( #pragma warning restore CA2000 // Dispose objects before losing scope try { - grainContext.ReceiveLocalRequest(request, request, options); + grainContext.ReceiveLocalRequest(request, options); } catch (Exception exception) { - // Rejecting the request releases its cancellation registration and reports the failure through the returned task. - ((ILocalRequest)request).Reject(exception); + // Completing the request releases its cancellation registration and reports the failure through the returned task. + ((ILocalRequest)request).Complete(Response.FromException(exception)); } return request.Task; diff --git a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs index a72c391c2b2..45a536112c8 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs @@ -75,9 +75,6 @@ public interface IGrainContext : ITargetHolder, IEquatable /// /// The request to invoke. Its method must return an interface type. /// - /// - /// The completion source which receives the request's response, or if no caller observes the response. - /// /// The options for the request. /// /// @@ -85,15 +82,12 @@ public interface IGrainContext : ITargetHolder, IEquatable /// . It starts a new call chain and never leaves this instance. /// /// - /// receives the response after the request and its call filters complete. - /// If this instance deactivates before the request starts, receives an - /// . If is set and the - /// instance's queue exceeds the configured maximum number of enqueued requests, receives a - /// . + /// The instance passes the request's outcome to after the request and its call filters complete, + /// or when it rejects the request. /// /// /// This instance does not support local requests. - void ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + void ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions options) => throw new NotSupportedException($"{GetType()} does not support local requests."); /// diff --git a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs new file mode 100644 index 00000000000..f797e711520 --- /dev/null +++ b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs @@ -0,0 +1,31 @@ +using Orleans.Serialization.Invocation; + +namespace Orleans.Runtime; + +/// +/// A request which a grain context receives through and which receives its own response. +/// +/// +/// The grain context invokes the request on the activation, through its incoming grain call filters, and then passes the +/// response to . Call filters, logs, and traces observe the request's interface name, method name, and arguments. +/// +public interface ILocalRequest : IInvokable +{ + /// + /// Completes the request with its outcome. + /// + /// + /// The response produced by the request and its call filters, or an exception if the grain context rejects the request. + /// The grain context rejects a request with an if it deactivates before the request starts, + /// and with a if is set and its queue is full. + /// + /// + /// if the request reports the response's exception to an observer; otherwise, , + /// in which case the runtime logs a failure as a warning. + /// + /// + /// The grain context calls this method once for each time it receives the request. + /// It can be called from any thread, so implementations should complete quickly and without blocking. + /// + bool Complete(Response response); +} diff --git a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs index 0ce6ed07072..c8cb9b1bfd2 100644 --- a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs @@ -10,12 +10,12 @@ namespace Orleans.Runtime; /// A request queued on an activation by . /// /// -/// The request is its own completion source, so queuing a request allocates only this object and its message. +/// The request receives its own response, so queuing a request allocates only this object and its message. /// Cancellation applies while the request is queued. Once the callback starts, the task reflects the callback's outcome. /// When a call filter completes the invocation without running the callback, the task reflects the filter's outcome. /// Call filters observe the request as . /// -internal sealed class GrainRequest : TaskCompletionSource, IInvokable, IGrainRequestInvoker, ILocalRequest, IResponseCompletionSource +internal sealed class GrainRequest : TaskCompletionSource, IGrainRequestInvoker, ILocalRequest { private readonly Func _callback; private readonly TState _state; @@ -112,9 +112,6 @@ bool ILocalRequest.Complete(Response response) return true; } - void IResponseCompletionSource.Complete(Response value) => ((ILocalRequest)this).Complete(value); - - void IResponseCompletionSource.Complete() => ((ILocalRequest)this).Complete(Response.Completed); public object? GetTarget() => this; @@ -162,20 +159,3 @@ internal interface IGrainRequestInvoker /// Task RunRequestAsync(CancellationToken cancellationToken = default); } - -/// -/// A local-only request whose task reports its outcome to the code which queued it. -/// -internal interface ILocalRequest -{ - /// - /// Completes the request with if the request has not already completed. - /// - /// if the request's task reports the response's exception, if any. - bool Complete(Response response); - - /// - /// Completes the request with if the request has not already completed. - /// - void Reject(Exception exception) => Complete(Response.FromException(exception)); -} diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index 6dde9a424a3..426d9449337 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -57,11 +57,12 @@ private CorrelationId GetNextCorrelationId() /// /// /// The request starts a new call chain, never expires, and completes on or is rejected with it. + /// Local-only messages always carry an body. /// - public Message CreateLocalRequest(IGrainContext target, object body, GrainInterfaceType interfaceType, LocalRequestOptions options) + public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, GrainInterfaceType interfaceType, LocalRequestOptions options) { var siloAddress = target.Address.SiloAddress; - var message = CreateMessage(body, InvokeMethodOptions.OneWay, requestContextData: null); + var message = CreateMessage(request, InvokeMethodOptions.OneWay, requestContextData: null); message.SetInfiniteTimeToLive(); message.SendingGrain = target.GrainId; message.TargetGrain = target.GrainId; diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 94b215115c6..00be05eab51 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -374,11 +374,9 @@ internal class GrainCanInterleave public List MayInterleavePredicates { get; } = new List(); public bool MayInterleave(object? instance, Message message) { - // Predicates observe the request rather than its completion envelope. - var request = LocalRequestCompletion.GetRequest(message.BodyObject); foreach (var predicate in this.MayInterleavePredicates) { - if (predicate!.Invoke(instance!, request)) return true; + if (predicate!.Invoke(instance!, (IInvokable)message.BodyObject!)) return true; } return false; diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 2a2f6f14c6e..ca9eb79e9a0 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -498,11 +498,7 @@ private List DequeueAllWaitingRequests(ref List<(ILocalRequest Request, // Local-only messages are not allowed to escape the activation. if (message.IsLocalOnly) { - if (message.BodyObject is ILocalRequest request) - { - (rejectedLocalRequests ??= []).Add((request, new OrleansMessageRejectionException($"Activation {this} is no longer valid."))); - } - + (rejectedLocalRequests ??= []).Add(((ILocalRequest)message.BodyObject!, new OrleansMessageRejectionException($"Activation {this} is no longer valid."))); continue; } @@ -1156,7 +1152,7 @@ void ProcessPendingRequests() { foreach (var (request, exception) in rejectedLocalRequests) { - request.Reject(exception); + request.Complete(Response.FromException(exception)); } } } @@ -1242,9 +1238,9 @@ void ProcessPendingRequestsCore(ref List<(ILocalRequest Request, Exception Excep { _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); } - else if (message.BodyObject is ILocalRequest request) + else { - (rejectedLocalRequests ??= []).Add((request, exception)); + (rejectedLocalRequests ??= []).Add(((ILocalRequest)message.BodyObject!, exception)); } _waitingRequests.RemoveAt(i); @@ -1688,9 +1684,9 @@ public void ReceiveMessage(Message message) } } - public void ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + public void ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions options) { - var message = _shared.InternalRuntime.RuntimeClient.CreateLocalRequest(this, request, completion, options); + var message = _shared.InternalRuntime.RuntimeClient.CreateLocalRequest(this, request, options); _shared.InternalRuntime.MessagingTrace.OnDispatcherReceiveMessage(message); ReceiveRequest(message, options.EnforceQueueLimit); } @@ -1739,7 +1735,7 @@ private void ReceiveRequest(Message message, bool enforceQueueLimit) var overloadException = new LimitExceededException(limitName, count, hardLimit, ToString()); if (message.IsLocalOnly) { - (message.BodyObject as ILocalRequest)?.Reject(overloadException); + ((ILocalRequest)message.BodyObject!).Complete(Response.FromException(overloadException)); } else { diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index f1c1f46945c..e18cacbc481 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -139,16 +139,11 @@ internal void ConsumeServices() /// /// Creates the message which queues on as a local request. /// - public Message CreateLocalRequest(IGrainContext target, IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) + public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, LocalRequestOptions options) { ArgumentNullException.ThrowIfNull(request); var interfaceType = interfaceTypeResolver.GetGrainInterfaceType(request.GetInterfaceType()); - - // A request which is its own completion source reports its response directly. - object body = completion is null || ReferenceEquals(completion, request) && request is ILocalRequest - ? request - : new LocalRequestCompletion(request, completion); - return messageFactory.CreateLocalRequest(target, body, interfaceType, options); + return messageFactory.CreateLocalRequest(target, request, interfaceType, options); } public void SendRequest( @@ -306,6 +301,8 @@ public void SniffIncomingMessage(Message message) public async Task Invoke(IGrainContext target, Message message) { + // A local request is completed exactly once, even if completing it fails. + var localRequestCompleted = false; try { // Don't process messages that have already timed out @@ -325,10 +322,8 @@ public async Task Invoke(IGrainContext target, Message message) { switch (message.BodyObject) { - case IInvokable body: + case IInvokable invokable: { - // Call filters observe the request rather than its completion envelope. - var invokable = LocalRequestCompletion.GetRequest(body); invokable.SetTarget(target); CancellationSourcesExtension.RegisterCancellationTokens(target, invokable); @@ -357,12 +352,17 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } - // Local requests report the response's exception through their tasks unless a filter failed after the callback completed. - var reportedByRequest = message.IsLocalOnly && message.BodyObject is ILocalRequest localRequest && localRequest.Complete(response); + // A local request reports its outcome to its own observers. + var reportedByRequest = false; + if (message.IsLocalOnly) + { + localRequestCompleted = true; + reportedByRequest = ((ILocalRequest)message.BodyObject!).Complete(response); + } if (response.Exception is { } invocationException) { - // Callers observe two-way failures. + // Callers observe two-way failures and failures which local requests report. var logLevel = message.Direction != Message.Directions.OneWay || reportedByRequest ? LogLevel.Debug : LogLevel.Warning; LogGrainInvokeException(this.invokeExceptionLogger, logLevel, invocationException, message); @@ -396,7 +396,10 @@ public async Task Invoke(IGrainContext target, Message message) catch (Exception exc2) { LogWarningInvokeException(this.logger, exc2, message); - (message.BodyObject as ILocalRequest)?.Reject(exc2); + if (message.IsLocalOnly && !localRequestCompleted) + { + ((ILocalRequest)message.BodyObject!).Complete(Response.FromException(exc2)); + } if (message.Direction != Message.Directions.OneWay) { diff --git a/src/Orleans.Runtime/Core/LocalRequestCompletion.cs b/src/Orleans.Runtime/Core/LocalRequestCompletion.cs deleted file mode 100644 index 207b589c290..00000000000 --- a/src/Orleans.Runtime/Core/LocalRequestCompletion.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System.Reflection; -using Orleans.Serialization.Invocation; - -namespace Orleans.Runtime; - -/// -/// The body of a local request whose response is reported to a completion source other than the request itself. -/// -/// -/// Message processing observes this envelope as the request. Grain call filters and [MayInterleave] predicates observe . -/// -internal sealed class LocalRequestCompletion(IInvokable request, IResponseCompletionSource completion) : IInvokable, ILocalRequest -{ - public IInvokable Request { get; } = request; - - /// - /// Returns the request carried by a message body, unwrapping the completion envelope if present. - /// - public static IInvokable GetRequest(object? body) => body is LocalRequestCompletion envelope ? envelope.Request : (IInvokable)body!; - - public bool Complete(Response response) - { - completion.Complete(response); - return true; - } - - public object? GetTarget() => Request.GetTarget(); - - public void SetTarget(ITargetHolder holder) => Request.SetTarget(holder); - - public ValueTask Invoke() => Request.Invoke(); - - public int GetArgumentCount() => Request.GetArgumentCount(); - - public object? GetArgument(int index) => Request.GetArgument(index); - - public void SetArgument(int index, object? value) => Request.SetArgument(index, value!); - - public string GetMethodName() => Request.GetMethodName(); - - public string GetInterfaceName() => Request.GetInterfaceName(); - - public string GetActivityName() => Request.GetActivityName(); - - public MethodInfo GetMethod() => Request.GetMethod(); - - public Type GetInterfaceType() => Request.GetInterfaceType(); - - public TimeSpan? GetDefaultResponseTimeout() => Request.GetDefaultResponseTimeout(); - - public CancellationToken GetCancellationToken() => Request.GetCancellationToken(); - - public bool TryCancel() => Request.TryCancel(); - - public bool IsCancellable => Request.IsCancellable; - - public void Dispose() => Request.Dispose(); - - public override string? ToString() => Request.ToString(); -} diff --git a/src/Orleans.Runtime/Core/SystemTarget.cs b/src/Orleans.Runtime/Core/SystemTarget.cs index fbf9b3f47d5..6e73625bcdd 100644 --- a/src/Orleans.Runtime/Core/SystemTarget.cs +++ b/src/Orleans.Runtime/Core/SystemTarget.cs @@ -375,8 +375,8 @@ public void ReceiveMessage(object message) } /// - void IGrainContext.ReceiveLocalRequest(IInvokable request, IResponseCompletionSource? completion, LocalRequestOptions options) - => ReceiveMessage(RuntimeClient.CreateLocalRequest(this, request, completion, options)); + void IGrainContext.ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions options) + => ReceiveMessage(RuntimeClient.CreateLocalRequest(this, request, options)); /// public object? GetTarget() => this; diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index f8f81e5aeca..83f49bfcb73 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -103,7 +103,7 @@ private void QueueTickOnActivation() { try { - _grainContext.ReceiveLocalRequest(_invoker, completion: null, _options); + _grainContext.ReceiveLocalRequest(_invoker, _options); } catch (Exception exception) { @@ -289,7 +289,7 @@ public void Dispose() public override string ToString() => $"[{GetType()}] Grain: '{_grainContext}'"; - private sealed class TimerTickInvoker(GrainTimer timer) : IInvokable, IGrainTimerInvoker + private sealed class TimerTickInvoker(GrainTimer timer) : ILocalRequest, IGrainTimerInvoker { public object? GetTarget() => this; @@ -303,6 +303,9 @@ public void SetTarget(ITargetHolder holder) public ValueTask Invoke() => timer.InvokeGrainTimerCallbackAsync(); + // No caller observes a tick, so the runtime logs a failed tick. + public bool Complete(Response response) => false; + // This method is declared for the sake of IGrainTimerCore, but it is not intended to be called directly. // It exists for grain call interceptors which inspect the implementation method. Task IGrainTimerInvoker.InvokeCallbackAsync(CancellationToken cancellationToken) => throw new InvalidOperationException(); diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index cf9e01d0816..102c9c71e5b 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -2635,7 +2635,7 @@ public partial interface IGrainContext : Orleans.Serialization.Invocation.ITarge void Activate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); void Deactivate(DeactivationReason deactivationReason, System.Threading.CancellationToken cancellationToken = default); void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); - void ReceiveLocalRequest(Orleans.Serialization.Invocation.IInvokable request, Orleans.Serialization.Invocation.IResponseCompletionSource? completion, LocalRequestOptions options); + void ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions options); void ReceiveMessage(object message); void Rehydrate(IRehydrationContext context); void SetComponent(TComponent? value) @@ -2714,6 +2714,11 @@ public partial interface IGrainTimer : System.IDisposable void Change(System.TimeSpan dueTime, System.TimeSpan period); } + public partial interface ILocalRequest : Orleans.Serialization.Invocation.IInvokable, System.IDisposable + { + bool Complete(Orleans.Serialization.Invocation.Response response); + } + public partial interface IRehydrationContext { System.Collections.Generic.IEnumerable Keys { get; } diff --git a/src/api/Orleans.Runtime/Orleans.Runtime.cs b/src/api/Orleans.Runtime/Orleans.Runtime.cs index 46d8d4e5bbc..b65ccfdff6e 100644 --- a/src/api/Orleans.Runtime/Orleans.Runtime.cs +++ b/src/api/Orleans.Runtime/Orleans.Runtime.cs @@ -991,7 +991,7 @@ public TExtensionInterface GetExtension() public void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken) { } - void IGrainContext.ReceiveLocalRequest(Orleans.Serialization.Invocation.IInvokable request, Orleans.Serialization.Invocation.IResponseCompletionSource? completion, LocalRequestOptions options) { } + void IGrainContext.ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions options) { } object? Orleans.Serialization.Invocation.ITargetHolder.GetComponent(System.Type componentType) { throw null; } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index aaf972ef875..10de4de2bd0 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -65,7 +65,7 @@ public interface IMayInterleaveRunRequestGrain : IRunRequestGrain /// Queues a custom request and returns its result once it runs, which requires interleaving with a blocking request. /// [AlwaysInterleave] - Task RunCustomRequestDuringBlockingRequest(bool observeCompletion); + Task RunCustomRequestDuringBlockingRequest(); } /// diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 04f2141aef1..10f43eb4530 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -76,9 +76,9 @@ public Task QueueUnlimitedRequests(int count) public Task StartCustomInvokableRequest(string value) { - var completion = new ResponseCompletionSource(); - ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CustomRequest(value), completion, default); - _pending = completion.AsValueTask().AsTask(); + var request = new CustomRequest(value); + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(request, default); + _pending = request.GetResultAsync(); return Task.CompletedTask; } @@ -236,66 +236,55 @@ public Task DeactivateAndRetainInstance() RetainedInstances[key]); /// - /// Queues a custom request on the retained instance whose rejection queues a second request from within the completion. + /// Queues a custom request on the retained instance whose rejection queues a second request from within . /// /// The number of times the first request completed and the second request's response. public static async Task<(int FirstCompletions, Response SecondResponse)> QueueChainedRequestsOnRetainedInstance(Guid key) { var context = ((IGrainBase)RetainedInstances[key]).GrainContext; - var second = new ChainingCompletion(context, next: null); - var first = new ChainingCompletion(context, second); - context.ReceiveLocalRequest(new CustomRequest("first"), first, default); - var secondResponse = await second.Response; - await first.Response; + var second = new CustomRequest("second"); + var first = new CustomRequest("first", _ => context.ReceiveLocalRequest(second, default)); + context.ReceiveLocalRequest(first, default); + var secondResponse = await second.Outcome; + await first.Outcome; return (first.Completions, secondResponse); } } /// -/// A completion source which queues a request completed by each time it completes. +/// A hand-written request delivered through , which records its outcome. /// -internal sealed class ChainingCompletion(IGrainContext context, ChainingCompletion? next) : IResponseCompletionSource +internal sealed class CustomRequest(string value, Action? onComplete = null) : Request, ILocalRequest { - private readonly TaskCompletionSource _response = new(TaskCreationOptions.RunContinuationsAsynchronously); + public const string InterfaceName = "ICustomRequests"; + public const string MethodName = "Apply"; + private readonly TaskCompletionSource _outcome = new(TaskCreationOptions.RunContinuationsAsynchronously); + private RunRequestGrainBase? _grain; private int _completions; - public Task Response => _response.Task; + public Task Outcome => _outcome.Task; public int Completions => Volatile.Read(ref _completions); - public void Complete(Response value) + public async Task GetResultAsync() { - Interlocked.Increment(ref _completions); - if (next is not null) - { - context.ReceiveLocalRequest(new CustomRequest("next"), next, default); - } - - _response.TrySetResult(value); + var response = await Outcome; + return response.Exception is { } exception ? throw exception : (string?)response.Result; } - public void Complete() => Complete(Orleans.Serialization.Invocation.Response.Completed); -} - -/// -/// A hand-written request delivered through . -/// -internal sealed class CustomRequest(string value, TaskCompletionSource? invoked = null) : Request -{ - public const string InterfaceName = "ICustomRequests"; - public const string MethodName = "Apply"; - private RunRequestGrainBase? _grain; + public bool Complete(Response response) + { + Interlocked.Increment(ref _completions); + onComplete?.Invoke(response); + _outcome.TrySetResult(response); + return true; + } public override object GetTarget() => _grain!; public override void SetTarget(ITargetHolder holder) => _grain = (RunRequestGrainBase?)holder.GetTarget(); - protected override ValueTask InvokeInner() - { - var result = $"{value}:{_grain is not null}"; - invoked?.TrySetResult(result); - return new(result); - } + protected override ValueTask InvokeInner() => new($"{value}:{_grain is not null}"); public override void Dispose() { } @@ -324,13 +313,10 @@ public sealed class MayInterleaveRunRequestGrain : RunRequestGrainBase, IMayInte { public static bool CanInterleave(IInvokable request) => request is CustomRequest; - public async Task RunCustomRequestDuringBlockingRequest(bool observeCompletion) + public Task RunCustomRequestDuringBlockingRequest() { - var invoked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var completion = observeCompletion ? new ResponseCompletionSource() : null; - ((IGrainBase)this).GrainContext.ReceiveLocalRequest(new CustomRequest("custom", invoked), completion, default); - - var result = await invoked.Task.WaitAsync(TimeSpan.FromSeconds(10)); - return completion is null ? result : await completion.AsValueTask(); + var request = new CustomRequest("custom"); + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(request, default); + return request.GetResultAsync().WaitAsync(TimeSpan.FromSeconds(10)); } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index e5b116176f1..821d5dfdb35 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -115,7 +115,7 @@ public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTh } [Fact] - public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() + public async Task Complete_RejectedQueuedRequest_FaultsWithoutRunningCallback() { using var cancellation = new CancellationTokenSource(); var callbackRan = false; @@ -129,7 +129,7 @@ public async Task Reject_QueuedRequest_FaultsWithoutRunningCallback() cancellation.Token); var exception = new OrleansMessageRejectionException("Rejected."); - ((ILocalRequest)request).Reject(exception); + ((ILocalRequest)request).Complete(Response.FromException(exception)); cancellation.Cancel(); await request.Invoke(); @@ -189,11 +189,11 @@ public class ThrowingContext : DispatchProxy } [Fact] - public async Task CompletionSource_Complete_CompletesTask() + public async Task Complete_FilterResultWithoutCallback_CompletesTask() { var request = new GrainRequest((_, _) => Task.FromResult("unused"), null, TestContext.Current.CancellationToken); - ((IResponseCompletionSource)request).Complete(Response.FromResult("filtered")); + Assert.True(((ILocalRequest)request).Complete(Response.FromResult("filtered"))); Assert.Equal("filtered", await request.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken)); } diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index cdd740155e1..5fdd8a5397f 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -70,7 +70,7 @@ public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool int RequestContext.Set("parent", new UncopyableContextValue()); try { - var body = new object(); + var body = Substitute.For(); var message = fixture.MessageFactory.CreateLocalRequest(fixture.Grain, body, interfaceType, new() { Interleave = interleave, KeepAlive = keepAlive }); Assert.Same(body, message.BodyObject); @@ -513,7 +513,7 @@ private sealed class TimerFixture : IObserver, IDis private readonly ILogger _logger = Substitute.For(); public TrackingTimeProvider Time { get; } = new(); public IGrainContext Grain { get; } = Substitute.For(); - public List<(IInvokable Request, LocalRequestOptions Options)> Requests { get; } = []; + public List<(ILocalRequest Request, LocalRequestOptions Options)> Requests { get; } = []; public List Events { get; } = []; public Action? BeforeReceive { get; set; } public MessageFactory MessageFactory { get; } @@ -526,11 +526,10 @@ public TimerFixture() loggerFactory.CreateLogger(Arg.Any()).Returns(_logger); var grainId = GrainId.Create("timer-test", "one"); Grain.GrainId.Returns(grainId); - Grain.When(context => context.ReceiveLocalRequest(Arg.Any(), Arg.Any(), Arg.Any())).Do(call => + Grain.When(context => context.ReceiveLocalRequest(Arg.Any(), Arg.Any())).Do(call => { - Assert.Null(call[1]); BeforeReceive?.Invoke(); - Requests.Add(((IInvokable)call[0], (LocalRequestOptions)call[2])); + Requests.Add(((ILocalRequest)call[0], (LocalRequestOptions)call[1])); }); Grain.Address.Returns(GrainAddress.GetAddress(SiloAddress.New(System.Net.IPAddress.Loopback, 11111, 1), grainId, ActivationId.NewId())); MessageFactory = new(_services.GetRequiredService(), NullLogger.Instance, null!); @@ -542,7 +541,7 @@ public IGrainTimer Register(Func callback, T stat TimeSpan period, bool interleave = false, bool keepAlive = false) => Registry.RegisterGrainTimer(Grain, callback, state, new(dueTime, period) { Interleave = interleave, KeepAlive = keepAlive }); - public async Task InvokeAsync((IInvokable Request, LocalRequestOptions Options) request, Exception? expectedException = null) + public async Task InvokeAsync((ILocalRequest Request, LocalRequestOptions Options) request, Exception? expectedException = null) { RequestContextExtensions.Import(null); var invokable = request.Request; diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index b59b0d25ad7..7dc8ee5a7d4 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -135,7 +135,7 @@ public async Task RunRequest_PassesThroughGrainCallFilter() [Fact] - public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletionSource() + public async Task ReceiveLocalRequest_CustomRequest_ReceivesResponse() { var grain = GrainFactory.GetGrain(Guid.NewGuid()); @@ -145,16 +145,14 @@ public async Task ReceiveLocalRequest_CustomInvokable_CompletesSeparateCompletio Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task ReceiveLocalRequest_MayInterleavePredicate_ObservesCustomRequest(bool observeCompletion) + [Fact] + public async Task ReceiveLocalRequest_MayInterleavePredicate_ObservesCustomRequest() { var grain = GrainFactory.GetGrain(Guid.NewGuid()); await grain.StartBlockingRequest(); await WaitForLogEntryAsync(grain, "request-start"); - Assert.Equal("custom:True", await grain.RunCustomRequestDuringBlockingRequest(observeCompletion)); + Assert.Equal("custom:True", await grain.RunCustomRequestDuringBlockingRequest()); await grain.ReleaseBlockingRequest(); await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken); From 353f88bcac09a1e7a72e835adfd3e4eff4b96e22 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 15:42:44 -0700 Subject: [PATCH 16/31] refactor(runtime): make ILocalRequest.Complete void and remove EnforceQueueLimit --- .../content/docs/grains/request-scheduling.md | 5 +- .../Grains/RequestsAndVersioningSnippets.cs | 25 +++--- .../Core/IGrainBase.cs | 2 - .../Core/ILocalRequest.cs | 13 +-- .../Core/LocalRequestOptions.cs | 11 --- .../Runtime/GrainRequest.cs | 24 +++--- src/Orleans.Runtime/Catalog/ActivationData.cs | 83 ++++++++++--------- .../Core/InsideRuntimeClient.cs | 13 +-- src/Orleans.Runtime/Timers/GrainTimer.cs | 6 +- .../Orleans.Core.Abstractions.cs | 4 +- .../TestGrainInterfaces/IRunRequestGrain.cs | 5 +- test/Grains/TestGrains/RunRequestGrain.cs | 66 ++------------- .../Runtime/GrainRequestTests.cs | 21 +++-- .../Runtime/GrainTimerTests.cs | 1 - .../RunRequestTests.cs | 32 +------ 15 files changed, 101 insertions(+), 210 deletions(-) diff --git a/docs/site/src/content/docs/grains/request-scheduling.md b/docs/site/src/content/docs/grains/request-scheduling.md index 1ef7241b013..27ae43ffea3 100644 --- a/docs/site/src/content/docs/grains/request-scheduling.md +++ b/docs/site/src/content/docs/grains/request-scheduling.md @@ -88,11 +88,8 @@ Pass the grain and other values through the `state` argument and use a `static` Pass to change how the request is scheduled. Set to `false` for frequent background work that shouldn't keep an otherwise idle activation in memory. Set to `true` to run the request between the turns of other requests, as with an `[AlwaysInterleave]` method: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_options"::: -Set to `true` to apply backpressure to a producer. The request then counts against , and the returned task faults with when the activation's queue is full: - -:::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="run_request_queue_limit"::: ### Deliver a custom request - delivers an to the activation with the same scheduling as `RunRequestAsync`. Call filters, logs, and traces observe the request's own interface name, method name, and arguments, so a custom request can describe the work to them. The activation passes the request's response to , or an exception if the activation rejects the request: + delivers an to the activation with the same scheduling as `RunRequestAsync`. Call filters, logs, and traces observe the request's own interface name, method name, and arguments, so a custom request can describe the work to them. The activation passes the request's response to , or an exception if the activation rejects the request. The request reports that outcome, including any failure, to its observers: :::code language="csharp" source="../snippets/compiled/Grains/RequestsAndVersioningSnippets.cs" id="receive_local_request"::: diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs index f3f13e178a2..51aed3433ce 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/RequestsAndVersioningSnippets.cs @@ -319,17 +319,6 @@ public sealed class IngestGrain : Grain, IIngestGrain public ValueTask GetCount() => ValueTask.FromResult(_count); - // -// Applies backpressure to the producer: the request faults with LimitExceededException -// when the activation's queue exceeds SiloMessagingOptions.MaxEnqueuedRequestsHardLimit. -internal Task Ingest(int items, CancellationToken cancellationToken) => - this.RunRequestAsync( - static (state, _) => state.Grain._count += state.Items, - (Grain: this, Items: items), - new LocalRequestOptions { EnforceQueueLimit = true }, - cancellationToken); - // - internal void Increment() => _count++; } @@ -343,8 +332,18 @@ internal sealed class IncrementRequest : Request, ILocalRequest public Task Completion => _completion.Task; // Receives the outcome once the request and its call filters complete, or when the activation rejects the request. - public bool Complete(Response response) => - response.Exception is { } exception ? _completion.TrySetException(exception) : _completion.TrySetResult(); + // The task keeps the first outcome. + public void Complete(Response response) + { + if (response.Exception is { } exception) + { + _completion.TrySetException(exception); + } + else + { + _completion.TrySetResult(); + } + } public override object GetTarget() => _grain!; diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 1ef7a98b7f1..9f02b67f79c 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -287,8 +287,6 @@ public static Task RunRequestAsync(this IGrainBase gra /// /// If the activation deactivates before the request starts, the returned task faults with /// . - /// If is set and the activation's queue exceeds the configured maximum number of enqueued requests, - /// the returned task faults with . /// To present the request to call filters and tracing with a different interface name, method name, or arguments, /// queue a custom using . /// diff --git a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs index f797e711520..ff22d1dd829 100644 --- a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs +++ b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs @@ -16,16 +16,11 @@ public interface ILocalRequest : IInvokable /// /// /// The response produced by the request and its call filters, or an exception if the grain context rejects the request. - /// The grain context rejects a request with an if it deactivates before the request starts, - /// and with a if is set and its queue is full. + /// The grain context rejects a request with an if it deactivates before the request starts. /// - /// - /// if the request reports the response's exception to an observer; otherwise, , - /// in which case the runtime logs a failure as a warning. - /// /// - /// The grain context calls this method once for each time it receives the request. - /// It can be called from any thread, so implementations should complete quickly and without blocking. + /// The request owns its outcome, including reporting or logging failures, and accepts the first outcome it receives. + /// This method can be called from any thread, so implementations should complete quickly and without blocking. /// - bool Complete(Response response); + void Complete(Response response); } diff --git a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs index 0d434a5c64c..c6101b78024 100644 --- a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs +++ b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs @@ -44,15 +44,4 @@ public bool KeepAlive /// the applied. /// public bool Interleave { get; init; } - - /// - /// Gets a value indicating whether the activation rejects the request when its queue is full. - /// Defaults to . - /// - /// - /// If this value is and the activation's queue exceeds - /// the configured maximum number of enqueued requests (SiloMessagingOptions.MaxEnqueuedRequestsHardLimit), the request completes with a . - /// If this value is , the activation queues the request regardless of the number of enqueued requests. - /// - public bool EnforceQueueLimit { get; init; } } diff --git a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs index c8cb9b1bfd2..3fdd98a0891 100644 --- a/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainRequest.cs @@ -89,27 +89,23 @@ private async ValueTask AwaitCallbackAsync(Task task) private static TResult GetResult(Task task) => task is Task typed ? typed.Result : default!; - bool ILocalRequest.Complete(Response response) + void ILocalRequest.Complete(Response response) { - var task = Task; - if (task.IsCompleted) + // The task keeps its first outcome: the callback's, a cancellation, or a call filter's which completed the invocation without running the callback. + if (Task.IsCompleted) { - // The callback completed the task, so the task reports the callback's failure but not a later filter's failure. - return response.Exception is not { } exception - || task.IsCanceled && exception is OperationCanceledException - || task.IsFaulted && ReferenceEquals(task.Exception!.InnerException, exception); + return; } - // A call filter completed the invocation without running the callback. - // Cancellation can complete the task first, in which case the task does not report the filter's failure. _registration.Dispose(); - if (response.Exception is { } filterException) + if (response.Exception is { } exception) { - return TrySetException(filterException); + TrySetException(exception); + } + else + { + TrySetResult(response.Result is TResult result ? result : default!); } - - TrySetResult(response.Result is TResult result ? result : default!); - return true; } diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index ca9eb79e9a0..2f7c7215cc9 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -480,6 +480,42 @@ private void SetState(ActivationState state) } } + /// + /// Check whether this activation is overloaded. + /// Returns LimitExceededException if overloaded, otherwise nullc> + /// + /// Returns LimitExceededException if overloaded, otherwise nullc> + public LimitExceededException? CheckOverloaded() + { + string limitName = nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit); + int maxRequestsHardLimit = _shared.MessagingOptions.MaxEnqueuedRequestsHardLimit; + int maxRequestsSoftLimit = _shared.MessagingOptions.MaxEnqueuedRequestsSoftLimit; + if (IsStatelessWorker) + { + limitName = nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker); + maxRequestsHardLimit = _shared.MessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker; + maxRequestsSoftLimit = _shared.MessagingOptions.MaxEnqueuedRequestsSoftLimit_StatelessWorker; + } + + if (maxRequestsHardLimit <= 0 && maxRequestsSoftLimit <= 0) return null; // No limits are set + + int count = GetRequestCount(); + + if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit + { + LogRejectActivationTooManyRequests(_shared.Logger, count, this, maxRequestsHardLimit); + return new LimitExceededException(limitName, count, maxRequestsHardLimit, ToString()); + } + + if (maxRequestsSoftLimit > 0 && count > maxRequestsSoftLimit) // Soft limit + { + LogWarnActivationTooManyRequests(_shared.Logger, count, this, maxRequestsSoftLimit); + return null; + } + + return null; + } + internal int GetRequestCount() { lock (_lock) @@ -1680,7 +1716,7 @@ public void ReceiveMessage(Message message) } else // Request or OneWay { - ReceiveRequest(message, enforceQueueLimit: !message.IsLocalOnly); + ReceiveRequest(message); } } @@ -1688,7 +1724,7 @@ public void ReceiveLocalRequest(ILocalRequest request, LocalRequestOptions optio { var message = _shared.InternalRuntime.RuntimeClient.CreateLocalRequest(this, request, options); _shared.InternalRuntime.MessagingTrace.OnDispatcherReceiveMessage(message); - ReceiveRequest(message, options.EnforceQueueLimit); + ReceiveRequest(message); } private void ReceiveResponse(Message message) @@ -1707,48 +1743,19 @@ private void ReceiveResponse(Message message) _shared.InternalRuntime.RuntimeClient.ReceiveResponse(message); } - private void ReceiveRequest(Message message, bool enforceQueueLimit) + private void ReceiveRequest(Message message) { - var options = _shared.MessagingOptions; - var (hardLimit, softLimit) = IsStatelessWorker - ? (options.MaxEnqueuedRequestsHardLimit_StatelessWorker, options.MaxEnqueuedRequestsSoftLimit_StatelessWorker) - : (options.MaxEnqueuedRequestsHardLimit, options.MaxEnqueuedRequestsSoftLimit); - - var enforceHardLimit = hardLimit > 0 && enforceQueueLimit; - int count; - bool admitted; - lock (_lock) - { - // Counting and enqueuing under one lock keeps concurrent producers within the hard limit. - count = _runningRequests.Count + _waitingRequests.Count; - admitted = !enforceHardLimit || count <= hardLimit; - if (admitted) - { - _waitingRequests.Add((message, CoarseStopwatch.StartNew())); - } - } - - if (!admitted) + var overloadException = CheckOverloaded(); + if (overloadException != null && !message.IsLocalOnly) { - LogRejectActivationTooManyRequests(_shared.Logger, count, this, hardLimit); - var limitName = IsStatelessWorker ? nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit_StatelessWorker) : nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit); - var overloadException = new LimitExceededException(limitName, count, hardLimit, ToString()); - if (message.IsLocalOnly) - { - ((ILocalRequest)message.BodyObject!).Complete(Response.FromException(overloadException)); - } - else - { - _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); - _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); - } - + _shared.MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); + _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); return; } - if (softLimit > 0 && count > softLimit) + lock (_lock) { - LogWarnActivationTooManyRequests(_shared.Logger, count, this, softLimit); + _waitingRequests.Add((message, CoarseStopwatch.StartNew())); } _workSignal.Signal(); diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index e18cacbc481..f256dd8ee4d 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -301,8 +301,6 @@ public void SniffIncomingMessage(Message message) public async Task Invoke(IGrainContext target, Message message) { - // A local request is completed exactly once, even if completing it fails. - var localRequestCompleted = false; try { // Don't process messages that have already timed out @@ -352,18 +350,15 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } - // A local request reports its outcome to its own observers. - var reportedByRequest = false; + // A local request owns its outcome, including reporting its failure. if (message.IsLocalOnly) { - localRequestCompleted = true; - reportedByRequest = ((ILocalRequest)message.BodyObject!).Complete(response); + ((ILocalRequest)message.BodyObject!).Complete(response); } if (response.Exception is { } invocationException) { - // Callers observe two-way failures and failures which local requests report. - var logLevel = message.Direction != Message.Directions.OneWay || reportedByRequest ? LogLevel.Debug : LogLevel.Warning; + var logLevel = message.Direction != Message.Directions.OneWay || message.IsLocalOnly ? LogLevel.Debug : LogLevel.Warning; LogGrainInvokeException(this.invokeExceptionLogger, logLevel, invocationException, message); // If a grain allowed an inconsistent state exception to escape and the exception originated from @@ -396,7 +391,7 @@ public async Task Invoke(IGrainContext target, Message message) catch (Exception exc2) { LogWarningInvokeException(this.logger, exc2, message); - if (message.IsLocalOnly && !localRequestCompleted) + if (message.IsLocalOnly) { ((ILocalRequest)message.BodyObject!).Complete(Response.FromException(exc2)); } diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index 83f49bfcb73..93c6c9275f4 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -303,8 +303,10 @@ public void SetTarget(ITargetHolder holder) public ValueTask Invoke() => timer.InvokeGrainTimerCallbackAsync(); - // No caller observes a tick, so the runtime logs a failed tick. - public bool Complete(Response response) => false; + // The timer logs callback failures as the tick runs. + public void Complete(Response response) + { + } // This method is declared for the sake of IGrainTimerCore, but it is not intended to be called directly. // It exists for grain call interceptors which inspect the implementation method. diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 102c9c71e5b..9f928f8383d 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -2716,7 +2716,7 @@ public partial interface IGrainTimer : System.IDisposable public partial interface ILocalRequest : Orleans.Serialization.Invocation.IInvokable, System.IDisposable { - bool Complete(Orleans.Serialization.Invocation.Response response); + void Complete(Orleans.Serialization.Invocation.Response response); } public partial interface IRehydrationContext @@ -2823,8 +2823,6 @@ public LimitExceededException(string message) { } public readonly partial struct LocalRequestOptions { private readonly int _dummyPrimitive; - public bool EnforceQueueLimit { get { throw null; } init { } } - public bool Interleave { get { throw null; } init { } } public bool KeepAlive { get { throw null; } init { } } diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 10de4de2bd0..897073882f3 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -11,7 +11,7 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartCustomInvokableRequest(string value); - Task QueueUnlimitedRequests(int count); + Task QueueRequests(int count); Task StartBlockingRequest(); @@ -22,9 +22,6 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartValueRequest(int value); - Task> QueueRequests(int count); - - Task QueueRequestsConcurrently(int count); Task SetRequestFilterMode(RequestFilterMode mode); diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 10f43eb4530..03d303f96a9 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -61,17 +61,17 @@ public async Task StartInterleavingRequestDuringCall() _log.Add("call-end"); } - - public Task QueueUnlimitedRequests(int count) + public async Task QueueRequests(int count) { - var admitted = new List(count); + var requests = new List(count); for (var i = 0; i < count; i++) { - admitted.Add(QueueLoggedRequest($"request-{i}", default)); + requests.Add(this.RunRequestAsync(static (state, _) => state.Grain._log.Add(state.Entry), (Grain: this, Entry: $"request-{i}"), new LocalRequestOptions { Interleave = true })); } - _pending = Task.WhenAll(admitted); - return Task.FromResult(admitted.Count(request => !request.IsFaulted)); + // The requests interleave with this call, which waits for them so that no other call arrives while they are queued. + await Task.WhenAll(requests); + return _log.Count; } public Task StartCustomInvokableRequest(string value) @@ -113,57 +113,6 @@ public Task StartValueRequest(int value) return Task.CompletedTask; } - public Task> QueueRequests(int count) - { - var outcomes = new List(count); - var admitted = new List(count); - for (var i = 0; i < count; i++) - { - var request = QueueLoggedRequest($"request-{i}", EnforceQueueLimit); - if (request.IsFaulted) - { - outcomes.Add(request.Exception!.InnerException!.GetType().Name); - } - else - { - outcomes.Add("queued"); - admitted.Add(request); - } - } - - _pending = Task.WhenAll(admitted); - return Task.FromResult(outcomes); - } - - public async Task QueueRequestsConcurrently(int count) - { - // Dedicated threads, released together, queue the requests while this call blocks the activation. - using var barrier = new Barrier(count); - var requests = await Task.WhenAll(Enumerable.Range(0, count).Select(i => Task.Factory.StartNew( - () => - { - barrier.SignalAndWait(); - return QueueLoggedRequest($"request-{i}", EnforceQueueLimit); - }, - CancellationToken.None, - TaskCreationOptions.LongRunning, - TaskScheduler.Default))); - - var admitted = requests.Where(request => !request.IsFaulted).ToList(); - if (requests.FirstOrDefault(request => request.IsFaulted && request.Exception!.InnerException is not LimitExceededException) is { } unexpected) - { - await unexpected; - } - - _pending = Task.WhenAll(admitted); - return admitted.Count; - } - - private static readonly LocalRequestOptions EnforceQueueLimit = new() { EnforceQueueLimit = true }; - - private Task QueueLoggedRequest(string entry, LocalRequestOptions options) => - this.RunRequestAsync(static (state, _) => state.Grain._log.Add(state.Entry), (Grain: this, Entry: entry), options); - public Task StartRequestContextRequest(string value) { RequestContext.Set("run-request", value); @@ -272,12 +221,11 @@ internal sealed class CustomRequest(string value, Action? onComplete = return response.Exception is { } exception ? throw exception : (string?)response.Result; } - public bool Complete(Response response) + public void Complete(Response response) { Interlocked.Increment(ref _completions); onComplete?.Invoke(response); _outcome.TrySetResult(response); - return true; } public override object GetTarget() => _grain!; diff --git a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs index 821d5dfdb35..378cc36d295 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainRequestTests.cs @@ -56,18 +56,18 @@ public async Task Complete_CallbackFailure_IsReportedByRequest() var response = await request.Invoke(); - Assert.True(((ILocalRequest)request).Complete(response)); + ((ILocalRequest)request).Complete(response); Assert.Same(exception, await Assert.ThrowsAsync(() => request.Task)); } [Fact] - public async Task Complete_FilterFailureAfterCallback_IsNotReportedByRequest() + public async Task Complete_FilterFailureAfterCallback_KeepsCallbackResult() { var request = new GrainRequest((_, _) => Task.FromResult("result"), null, default); await request.Invoke(); - Assert.False(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); + ((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed."))); Assert.Equal("result", await request.Task); } @@ -84,13 +84,13 @@ public async Task Complete_FilterFailureWithoutCallback_IsReportedByRequest() null, default); - Assert.True(((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed.")))); + ((ILocalRequest)request).Complete(Response.FromException(new InvalidOperationException("Filter failed."))); Assert.False(callbackRan); await Assert.ThrowsAsync(() => request.Task); } [Fact] - public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTheTaskCarries() + public async Task Complete_FilterFailureRacingCancellation_KeepsFirstOutcome() { var filterException = new InvalidOperationException("Filter failed."); var response = Response.FromException(filterException); @@ -106,11 +106,11 @@ public async Task Complete_FilterFailureRacingCancellation_ReportsOnlyFailuresTh }, TestContext.Current.CancellationToken); start.SignalAndWait(TestContext.Current.CancellationToken); - var reported = ((ILocalRequest)request).Complete(response); + ((ILocalRequest)request).Complete(response); await cancel; - // The request reports the filter's failure exactly when its task carries that failure. - Assert.Equal(request.Task.IsFaulted && ReferenceEquals(request.Task.Exception!.InnerException, filterException), reported); + // The task carries whichever outcome arrived first. + Assert.True(request.Task.IsCanceled || request.Task.IsFaulted && ReferenceEquals(request.Task.Exception!.InnerException, filterException)); } } @@ -193,18 +193,17 @@ public async Task Complete_FilterResultWithoutCallback_CompletesTask() { var request = new GrainRequest((_, _) => Task.FromResult("unused"), null, TestContext.Current.CancellationToken); - Assert.True(((ILocalRequest)request).Complete(Response.FromResult("filtered"))); + ((ILocalRequest)request).Complete(Response.FromResult("filtered")); Assert.Equal("filtered", await request.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken)); } [Fact] - public void LocalRequestOptions_Default_KeepsActivationAliveWithoutInterleavingOrQueueLimit() + public void LocalRequestOptions_Default_KeepsActivationAliveWithoutInterleaving() { Assert.True(default(LocalRequestOptions).KeepAlive); Assert.True(new LocalRequestOptions().KeepAlive); Assert.False(new LocalRequestOptions { KeepAlive = false }.KeepAlive); Assert.False(default(LocalRequestOptions).Interleave); - Assert.False(default(LocalRequestOptions).EnforceQueueLimit); } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 5fdd8a5397f..02b7edefa68 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -47,7 +47,6 @@ public async Task ImmediateTick_UsesOrdinaryActivationAdmission(bool interleave, Assert.Equal(1, fixture.Time.TimerCreations); Assert.Equal(interleave, request.Options.Interleave); Assert.Equal(keepAlive, request.Options.KeepAlive); - Assert.False(request.Options.EnforceQueueLimit); Assert.Same(parent, RequestContext.Get("parent")); await fixture.InvokeAsync(request); fixture.Time.Advance(TimeSpan.FromDays(1)); diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 7dc8ee5a7d4..b3679294e5d 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -287,41 +287,13 @@ protected override void ConfigureTestCluster(TestClusterBuilder builder) } [Fact] - public async Task RunRequest_EnforcingQueueLimit_BeyondHardLimit_FaultsWithLimitExceeded() + public async Task RunRequest_IgnoresHardLimit() { var grain = GrainFactory.GetGrain(Guid.NewGuid()); - // The queuing call counts toward the limit, so the activation admits requests until its count exceeds the limit. - var outcomes = await grain.QueueRequests(HardLimit + 2); - await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.Equal(["queued", "queued", "queued", nameof(LimitExceededException), nameof(LimitExceededException)], outcomes); - Assert.Equal(["request-0", "request-1", "request-2"], await grain.GetLog()); - } - - [Fact] - public async Task RunRequest_ByDefault_IgnoresHardLimit() - { - var grain = GrainFactory.GetGrain(Guid.NewGuid()); - - var admitted = await grain.QueueUnlimitedRequests(HardLimit * 4); - await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.Equal(HardLimit * 4, admitted); - Assert.Equal(HardLimit * 4, (await grain.GetLog()).Count); + Assert.Equal(HardLimit * 4, await grain.QueueRequests(HardLimit * 4).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); } - [Fact] - public async Task RunRequest_ConcurrentProducers_StayWithinHardLimit() - { - var grain = GrainFactory.GetGrain(Guid.NewGuid()); - - var admitted = await grain.QueueRequestsConcurrently(64); - await grain.AwaitPendingRequest().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.Equal(HardLimit, admitted); - Assert.Equal(HardLimit, (await grain.GetLog()).Count); - } private sealed class SiloConfigurator : ISiloConfigurator { From a6479b9e8dc98dc0b099aac65c3a6c579c9669ce Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 16:33:48 -0700 Subject: [PATCH 17/31] refactor(runtime): leave the local request interface type unset --- src/Orleans.Core/Messaging/MessageFactory.cs | 4 ++-- src/Orleans.Runtime/Core/InsideRuntimeClient.cs | 8 +------- test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs | 5 ++--- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index 426d9449337..77d28982489 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -59,8 +59,9 @@ private CorrelationId GetNextCorrelationId() /// The request starts a new call chain, never expires, and completes on or is rejected with it. /// Local-only messages always carry an body. /// - public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, GrainInterfaceType interfaceType, LocalRequestOptions options) + public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, LocalRequestOptions options) { + ArgumentNullException.ThrowIfNull(request); var siloAddress = target.Address.SiloAddress; var message = CreateMessage(request, InvokeMethodOptions.OneWay, requestContextData: null); message.SetInfiniteTimeToLive(); @@ -68,7 +69,6 @@ public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, G message.TargetGrain = target.GrainId; message.SendingSilo = siloAddress; message.TargetSilo = siloAddress; - message.InterfaceType = interfaceType; message.IsKeepAlive = options.KeepAlive; message.IsAlwaysInterleave = options.Interleave; message.IsLocalOnly = true; diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index f256dd8ee4d..df3d40c9900 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -48,7 +48,6 @@ internal sealed partial class InsideRuntimeClient : IRuntimeClient, ILifecyclePa private HostedClient HostedClient => this.hostedClient; private readonly MessageFactory messageFactory; - private readonly GrainInterfaceTypeResolver interfaceTypeResolver; private IGrainReferenceRuntime grainReferenceRuntime = null!; private Task? callbackTimerTask; private readonly MessagingTrace messagingTrace; @@ -77,7 +76,6 @@ public InsideRuntimeClient( this.MySilo = siloDetails.SiloAddress; this.callbacks = new ConcurrentDictionary<(GrainId, CorrelationId), CallbackData>(); this.messageFactory = messageFactory; - this.interfaceTypeResolver = interfaceIdResolver; this.ConcreteGrainFactory = new GrainFactory(this, referenceActivator, interfaceIdResolver, interfaceToTypeResolver); this.logger = loggerFactory.CreateLogger(); this.invokeExceptionLogger = loggerFactory.CreateLogger($"{typeof(Grain).FullName}.InvokeException"); @@ -140,11 +138,7 @@ internal void ConsumeServices() /// Creates the message which queues on as a local request. /// public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, LocalRequestOptions options) - { - ArgumentNullException.ThrowIfNull(request); - var interfaceType = interfaceTypeResolver.GetGrainInterfaceType(request.GetInterfaceType()); - return messageFactory.CreateLocalRequest(target, request, interfaceType, options); - } + => messageFactory.CreateLocalRequest(target, request, options); public void SendRequest( GrainReference target, diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 02b7edefa68..4795c9be25f 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -65,12 +65,11 @@ public async Task ImmediateTick_UsesOrdinaryActivationAdmission(bool interleave, public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool interleave, bool keepAlive) { using var fixture = new TimerFixture(); - var interfaceType = GrainInterfaceType.Create("local-request"); RequestContext.Set("parent", new UncopyableContextValue()); try { var body = Substitute.For(); - var message = fixture.MessageFactory.CreateLocalRequest(fixture.Grain, body, interfaceType, new() { Interleave = interleave, KeepAlive = keepAlive }); + var message = fixture.MessageFactory.CreateLocalRequest(fixture.Grain, body, new() { Interleave = interleave, KeepAlive = keepAlive }); Assert.Same(body, message.BodyObject); Assert.Equal(Message.Directions.OneWay, message.Direction); @@ -79,7 +78,7 @@ public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool int Assert.True(message.IsLocalOnly); Assert.Null(message.TimeToLive); Assert.Null(message.RequestContextData); - Assert.Equal(interfaceType, message.InterfaceType); + Assert.True(message.InterfaceType.IsDefault); Assert.Equal(fixture.Grain.GrainId, message.TargetGrain); Assert.Equal(message.SendingGrain, message.TargetGrain); Assert.Equal(fixture.Grain.Address.SiloAddress, message.TargetSilo); From 3083802fc175746a2eaa18130a97cc24b9088451 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 17:00:41 -0700 Subject: [PATCH 18/31] chore(runtime): assert local-only messages never reach responses or rejections --- src/Orleans.Runtime/Core/InsideRuntimeClient.cs | 4 +++- src/Orleans.Runtime/Messaging/MessageCenter.cs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index df3d40c9900..9ee99fab2fa 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -295,6 +295,8 @@ public void SniffIncomingMessage(Message message) public async Task Invoke(IGrainContext target, Message message) { + // Local requests are one-way: they complete themselves on the target instead of sending a response. + Debug.Assert(!message.IsLocalOnly || message.Direction == Message.Directions.OneWay); try { // Don't process messages that have already timed out @@ -344,7 +346,7 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } - // A local request owns its outcome, including reporting its failure. + // A local request owns its outcome, including reporting its failure, in place of a response. if (message.IsLocalOnly) { ((ILocalRequest)message.BodyObject!).Complete(response); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index a7af6263f7d..de9e2d04b7a 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -270,6 +270,7 @@ public void RejectMessage( Exception? exc, string? rejectInfo = null) { + Debug.Assert(!message.IsLocalOnly); try { if (message.Direction == Message.Directions.Request @@ -541,6 +542,7 @@ void OnAddressingFailure(Message m, Exception ex) internal void SendResponse(Message request, Response response) { + Debug.Assert(!request.IsLocalOnly); // create the response var message = this.messageFactory.CreateResponseMessage(request); message.BodyObject = response; From bfda2f048007ee3916f04dd52fc7e453352d6a05 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 18:15:47 -0700 Subject: [PATCH 19/31] fix(runtime): isolate local request completion failures and dispose after completion --- .../Core/ILocalRequest.cs | 4 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 2 +- .../Core/InsideRuntimeClient.cs | 22 +++++---- src/Orleans.Runtime/Core/LocalRequests.cs | 38 ++++++++++++++++ src/Orleans.Runtime/Timers/GrainTimer.cs | 19 ++++---- .../TestGrainInterfaces/IRunRequestGrain.cs | 8 ++++ test/Grains/TestGrains/RunRequestGrain.cs | 45 +++++++++++++++++-- .../Runtime/GrainTimerTests.cs | 21 +++++++++ .../RunRequestTests.cs | 21 +++++++++ 9 files changed, 156 insertions(+), 24 deletions(-) create mode 100644 src/Orleans.Runtime/Core/LocalRequests.cs diff --git a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs index ff22d1dd829..4585adcb867 100644 --- a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs +++ b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs @@ -6,8 +6,8 @@ namespace Orleans.Runtime; /// A request which a grain context receives through and which receives its own response. /// /// -/// The grain context invokes the request on the activation, through its incoming grain call filters, and then passes the -/// response to . Call filters, logs, and traces observe the request's interface name, method name, and arguments. +/// The grain context invokes the request on the activation, through its incoming grain call filters, passes the +/// response to , and then disposes the request. Call filters, logs, and traces observe the request's interface name, method name, and arguments. /// public interface ILocalRequest : IInvokable { diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 2f7c7215cc9..eb404630954 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1188,7 +1188,7 @@ void ProcessPendingRequests() { foreach (var (request, exception) in rejectedLocalRequests) { - request.Complete(Response.FromException(exception)); + LocalRequests.Complete(request, Response.FromException(exception), _shared.Logger); } } } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 9ee99fab2fa..5cdc7b01125 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -334,7 +334,12 @@ public async Task Invoke(IGrainContext target, Message message) response = this.responseCopier.Copy(response)!; } - invokable.Dispose(); + // A local request is disposed once it is completed. + if (!message.IsLocalOnly) + { + invokable.Dispose(); + } + break; } default: @@ -346,12 +351,6 @@ public async Task Invoke(IGrainContext target, Message message) response = Response.FromException(exc1); } - // A local request owns its outcome, including reporting its failure, in place of a response. - if (message.IsLocalOnly) - { - ((ILocalRequest)message.BodyObject!).Complete(response); - } - if (response.Exception is { } invocationException) { var logLevel = message.Direction != Message.Directions.OneWay || message.IsLocalOnly ? LogLevel.Debug : LogLevel.Warning; @@ -377,6 +376,13 @@ public async Task Invoke(IGrainContext target, Message message) } } + // A local request owns its outcome, including reporting its failure, in place of a response. + // Completing it is the last step, so a failure before it completes the request exactly once. + if (message.IsLocalOnly) + { + LocalRequests.Complete((ILocalRequest)message.BodyObject!, response, this.logger); + } + if (message.Direction != Message.Directions.OneWay) { SafeSendResponse(message, response); @@ -389,7 +395,7 @@ public async Task Invoke(IGrainContext target, Message message) LogWarningInvokeException(this.logger, exc2, message); if (message.IsLocalOnly) { - ((ILocalRequest)message.BodyObject!).Complete(Response.FromException(exc2)); + LocalRequests.Complete((ILocalRequest)message.BodyObject!, Response.FromException(exc2), this.logger); } if (message.Direction != Message.Directions.OneWay) diff --git a/src/Orleans.Runtime/Core/LocalRequests.cs b/src/Orleans.Runtime/Core/LocalRequests.cs new file mode 100644 index 00000000000..e1ab6533241 --- /dev/null +++ b/src/Orleans.Runtime/Core/LocalRequests.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.Logging; +using Orleans.Serialization.Invocation; + +namespace Orleans.Runtime; + +internal static partial class LocalRequests +{ + /// + /// Completes with and then disposes it. + /// + /// + /// A failure in the request's completion or disposal is logged rather than thrown, so callers completing a batch of requests complete every request. + /// + public static void Complete(ILocalRequest request, Response response, ILogger logger) + { + try + { + try + { + request.Complete(response); + } + finally + { + request.Dispose(); + } + } + catch (Exception exception) + { + LogErrorCompletingLocalRequest(logger, exception, request); + } + } + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error completing local request {Request}." + )] + private static partial void LogErrorCompletingLocalRequest(ILogger logger, Exception exception, ILocalRequest request); +} diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index 93c6c9275f4..e17ed0ebb1b 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -159,7 +159,7 @@ private ValueTask InvokeGrainTimerCallbackAsync() { GrainTimerEvents.EmitTickStop(GrainContext, this, exc); OnTickCompleted(); - return new(OnCallbackException(exc)); + return new(Response.FromException(exc)); } } @@ -184,12 +184,6 @@ private void OnTickCompleted() } } - private Response OnCallbackException(Exception exc) - { - LogWarningCallbackException(Logger, exc, this); - return Response.FromException(exc); - } - private async ValueTask AwaitCallbackTask(Task task) { try @@ -206,7 +200,7 @@ private async ValueTask AwaitCallbackTask(Task task) { GrainTimerEvents.EmitTickStop(GrainContext, this, exc); - return OnCallbackException(exc); + return Response.FromException(exc); } finally { @@ -303,9 +297,14 @@ public void SetTarget(ITargetHolder holder) public ValueTask Invoke() => timer.InvokeGrainTimerCallbackAsync(); - // The timer logs callback failures as the tick runs. + // No caller observes a tick, so the tick logs its failures, including call filter failures. + // Ticks dropped by deactivation are expected and are not logged. public void Complete(Response response) { + if (response.Exception is { } exception and not OrleansMessageRejectionException) + { + LogWarningCallbackException(timer.Logger, exception, timer); + } } // This method is declared for the sake of IGrainTimerCore, but it is not intended to be called directly. @@ -361,7 +360,7 @@ public void Dispose() [LoggerMessage( EventId = (int)ErrorCode.Timer_GrainTimerCallbackError, Level = LogLevel.Warning, - Message = "Caught and ignored exception thrown from timer callback for timer '{Timer}'." + Message = "Caught and ignored exception thrown from timer tick for timer '{Timer}'." )] private static partial void LogWarningCallbackException(ILogger logger, Exception exception, GrainTimer timer); diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 897073882f3..5259f044753 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -11,6 +11,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task StartCustomInvokableRequest(string value); + Task RunCustomRequestLifecycle(); + Task QueueRequests(int count); Task StartBlockingRequest(); @@ -63,6 +65,12 @@ public interface IMayInterleaveRunRequestGrain : IRunRequestGrain /// [AlwaysInterleave] Task RunCustomRequestDuringBlockingRequest(); + + /// + /// Queues two requests which the activation rejects together, the first of which throws from its completion. + /// + [AlwaysInterleave] + Task RejectRequestsWithThrowingCompletion(); } /// diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index 03d303f96a9..b283529d69a 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -82,6 +82,16 @@ public Task StartCustomInvokableRequest(string value) return Task.CompletedTask; } + public async Task RunCustomRequestLifecycle() + { + // The request interleaves so that it runs while this call waits for it. + var request = new CustomRequest("lifecycle"); + ((IGrainBase)this).GrainContext.ReceiveLocalRequest(request, new LocalRequestOptions { Interleave = true }); + var response = await request.Outcome.WaitAsync(TimeSpan.FromSeconds(10)); + await request.Disposed.WaitAsync(TimeSpan.FromSeconds(10)); + return $"{response.Result}:disposed-before-completion={request.DisposedBeforeCompletion}"; + } + public Task StartBlockingRequest() { var gate = _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -208,11 +218,18 @@ internal sealed class CustomRequest(string value, Action? onComplete = public const string InterfaceName = "ICustomRequests"; public const string MethodName = "Apply"; private readonly TaskCompletionSource _outcome = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _disposed = new(TaskCreationOptions.RunContinuationsAsynchronously); private RunRequestGrainBase? _grain; private int _completions; + public string Value => value; + public Task Outcome => _outcome.Task; + public Task Disposed => _disposed.Task; + + public bool DisposedBeforeCompletion { get; private set; } + public int Completions => Volatile.Read(ref _completions); public async Task GetResultAsync() @@ -224,8 +241,9 @@ internal sealed class CustomRequest(string value, Action? onComplete = public void Complete(Response response) { Interlocked.Increment(ref _completions); - onComplete?.Invoke(response); + DisposedBeforeCompletion = _disposed.Task.IsCompleted; _outcome.TrySetResult(response); + onComplete?.Invoke(response); } public override object GetTarget() => _grain!; @@ -234,7 +252,7 @@ public void Complete(Response response) protected override ValueTask InvokeInner() => new($"{value}:{_grain is not null}"); - public override void Dispose() { } + public override void Dispose() => _disposed.TrySetResult(); public override string GetMethodName() => MethodName; @@ -259,7 +277,28 @@ public sealed class ReentrantRunRequestGrain : RunRequestGrainBase, IReentrantRu [MayInterleave(nameof(CanInterleave))] public sealed class MayInterleaveRunRequestGrain : RunRequestGrainBase, IMayInterleaveRunRequestGrain { - public static bool CanInterleave(IInvokable request) => request is CustomRequest; + public const string RejectedValue = "rejected"; + + public static bool CanInterleave(IInvokable request) => request switch + { + CustomRequest { Value: RejectedValue } => throw new InvalidOperationException("Predicate failed."), + CustomRequest => true, + _ => false, + }; + + public async Task RejectRequestsWithThrowingCompletion() + { + // Both requests are queued in one turn, so the activation rejects them together when the predicate throws. + var context = ((IGrainBase)this).GrainContext; + var first = new CustomRequest(RejectedValue, _ => throw new InvalidOperationException("Completion failed.")); + var second = new CustomRequest(RejectedValue); + context.ReceiveLocalRequest(first, default); + context.ReceiveLocalRequest(second, default); + + var secondResponse = await second.Outcome.WaitAsync(TimeSpan.FromSeconds(10)); + var firstResponse = await first.Outcome.WaitAsync(TimeSpan.FromSeconds(10)); + return $"{firstResponse.Exception?.Message}:{first.Completions}|{secondResponse.Exception?.Message}:{second.Completions}"; + } public Task RunCustomRequestDuringBlockingRequest() { diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 4795c9be25f..99b039c5d14 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -90,6 +90,21 @@ public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool int } } + [Fact] + public void Tick_Complete_LogsFailuresExceptRejections() + { + using var fixture = new TimerFixture(); + using var timer = fixture.Register((_, _) => Task.CompletedTask, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan); + var tick = Assert.Single(fixture.Requests).Request; + var failure = new InvalidOperationException("Filter failed."); + + tick.Complete(Response.FromException(new OrleansMessageRejectionException("Rejected."))); + tick.Complete(Response.Completed); + tick.Complete(Response.FromException(failure)); + + Assert.Same(failure, Assert.Single(fixture.LoggedWarnings())); + } + [Fact] public void MessageFactory_DefaultContextIsCopiedWhileExplicitContextSkipsExport() { @@ -520,6 +535,7 @@ private sealed class TimerFixture : IObserver, IDis public TimerFixture() { _logger.IsEnabled(LogLevel.Error).Returns(true); + _logger.IsEnabled(LogLevel.Warning).Returns(true); var loggerFactory = Substitute.For(); loggerFactory.CreateLogger(Arg.Any()).Returns(_logger); var grainId = GrainId.Create("timer-test", "one"); @@ -548,6 +564,11 @@ public async Task InvokeAsync((ILocalRequest Request, LocalRequestOptions Option Assert.Same(expectedException, response.Exception); } + public List LoggedWarnings() => + [.. _logger.ReceivedCalls() + .Where(call => call.GetMethodInfo().Name == nameof(ILogger.Log) && Equals(call.GetArguments()[0], LogLevel.Warning)) + .Select(call => (Exception)call.GetArguments()[3]!)]; + public Exception SingleLoggedError() { var log = Assert.Single(_logger.ReceivedCalls(), call => diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index b3679294e5d..504397dec24 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -145,6 +145,27 @@ public async Task ReceiveLocalRequest_CustomRequest_ReceivesResponse() Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } + [Fact] + public async Task ReceiveLocalRequest_CustomRequest_IsDisposedAfterCompletion() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + Assert.Equal("lifecycle:True:disposed-before-completion=False", await grain.RunCustomRequestLifecycle().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ReceiveLocalRequest_RejectedBatchWithThrowingCompletion_CompletesEveryRequestOnce() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + await grain.StartBlockingRequest(); + await WaitForLogEntryAsync(grain, "request-start"); + + Assert.Equal("Predicate failed.:1|Predicate failed.:1", await grain.RejectRequestsWithThrowingCompletion().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + + await grain.ReleaseBlockingRequest(); + await grain.AwaitPendingRequest().WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + [Fact] public async Task ReceiveLocalRequest_MayInterleavePredicate_ObservesCustomRequest() { From eb55c18e258b0835eb9c81c53bace2661cc6635e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 04:35:05 -0700 Subject: [PATCH 20/31] fix(runtime): log timer tick failures until the timer is disposed --- src/Orleans.Runtime/Timers/GrainTimer.cs | 15 +++++++++++++-- .../Runtime/GrainTimerTests.cs | 17 +++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index e17ed0ebb1b..848f73ed1b1 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -67,6 +67,17 @@ private bool ScheduleNextTick() private ILogger Logger => _shared.TimerLogger; + private bool IsDisposed + { + get + { + lock (_cts) + { + return _state == TimerState.Disposed; + } + } + } + [DoesNotReturn] private static void ThrowIncorrectGrainContext() => throw new InvalidOperationException("Current grain context differs from specified grain context."); @@ -298,10 +309,10 @@ public void SetTarget(ITargetHolder holder) public ValueTask Invoke() => timer.InvokeGrainTimerCallbackAsync(); // No caller observes a tick, so the tick logs its failures, including call filter failures. - // Ticks dropped by deactivation are expected and are not logged. + // An activation disposes its timers before it rejects their queued ticks, so a disposed timer's failures are expected. public void Complete(Response response) { - if (response.Exception is { } exception and not OrleansMessageRejectionException) + if (response.Exception is { } exception && !timer.IsDisposed) { LogWarningCallbackException(timer.Logger, exception, timer); } diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 99b039c5d14..0e07e64248a 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -91,18 +91,23 @@ public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool int } [Fact] - public void Tick_Complete_LogsFailuresExceptRejections() + public void Tick_Complete_LogsFailuresUntilTimerIsDisposed() { using var fixture = new TimerFixture(); - using var timer = fixture.Register((_, _) => Task.CompletedTask, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan); + var timer = fixture.Register((_, _) => Task.CompletedTask, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan); var tick = Assert.Single(fixture.Requests).Request; - var failure = new InvalidOperationException("Filter failed."); + var filterFailure = new InvalidOperationException("Filter failed."); + var downstreamRejection = new OrleansMessageRejectionException("Downstream call rejected."); - tick.Complete(Response.FromException(new OrleansMessageRejectionException("Rejected."))); tick.Complete(Response.Completed); - tick.Complete(Response.FromException(failure)); + tick.Complete(Response.FromException(filterFailure)); + tick.Complete(Response.FromException(downstreamRejection)); - Assert.Same(failure, Assert.Single(fixture.LoggedWarnings())); + // An activation disposes its timers before it rejects their queued ticks. + timer.Dispose(); + tick.Complete(Response.FromException(new OrleansMessageRejectionException("Activation is no longer valid."))); + + Assert.Equal([filterFailure, downstreamRejection], fixture.LoggedWarnings()); } [Fact] From e7e8fec5d6087be3ac70c182894835a8808762e4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 05:41:10 -0700 Subject: [PATCH 21/31] docs(runtime): explain LocalRequestOptions keep-alive storage --- src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs index c6101b78024..72a267d6ff1 100644 --- a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs +++ b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs @@ -11,6 +11,7 @@ namespace Orleans.Runtime; [SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "Per-call options are passed to the runtime rather than compared.")] public readonly struct LocalRequestOptions { + // Stored inverted so that default(LocalRequestOptions), which callers pass as a default argument, keeps the activation alive. private readonly bool _suppressKeepAlive; /// @@ -22,7 +23,7 @@ public LocalRequestOptions() /// /// Gets a value indicating whether the request extends the lifetime of the grain activation. - /// Defaults to . + /// Defaults to , including for (). /// /// /// A request which keeps the activation alive restarts the activation's idle collection period when it completes, From d688e343a840cb099e8c22af5fb6ed9b9dd81dae Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 05:41:22 -0700 Subject: [PATCH 22/31] docs(runtime): keep KeepAlive summary concise --- src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs index 72a267d6ff1..59b8663f83f 100644 --- a/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs +++ b/src/Orleans.Core.Abstractions/Core/LocalRequestOptions.cs @@ -23,7 +23,7 @@ public LocalRequestOptions() /// /// Gets a value indicating whether the request extends the lifetime of the grain activation. - /// Defaults to , including for (). + /// Defaults to . /// /// /// A request which keeps the activation alive restarts the activation's idle collection period when it completes, From a0876f6a7aa29ac8e6aa46a594549f58c062b94a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 05:51:40 -0700 Subject: [PATCH 23/31] test(runtime): deliver late stateless worker requests as local requests --- ...tatelessWorkerSingleActivationRaceTests.cs | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerSingleActivationRaceTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerSingleActivationRaceTests.cs index ede5f576de4..8ba908cc6b5 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerSingleActivationRaceTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerSingleActivationRaceTests.cs @@ -77,10 +77,12 @@ public async Task TerminatedContextForwardsLateDeliveryToFreshWrapper() collector.Clear(); - c1.ReceiveMessage(CreateLateDeliveryMessage(grainId)); + var lateDelivery = new DoWorkRequest(); + c1.ReceiveMessage(CreateLateDeliveryMessage(grainId, lateDelivery)); var forwarded = await WaitForMessageForwardedAsync(collector, c1); var workerCreated = await WaitForWorkerCreatedAsync(collector, forwarded.ReplacementContext); + await AssertCompletedSuccessfullyAsync(lateDelivery); Assert.NotSame(c1, forwarded.ReplacementContext); Assert.Same(forwarded.ReplacementContext, workerCreated.Context); @@ -116,11 +118,13 @@ public async Task LateDeliveryAndFreshCallUseSingleReplacementActivation() collector.Clear(); var freshCallTask = grain.DoWork(); - c1.ReceiveMessage(CreateLateDeliveryMessage(grainId)); + var lateDelivery = new DoWorkRequest(); + c1.ReceiveMessage(CreateLateDeliveryMessage(grainId, lateDelivery)); await freshCallTask; var forwarded = await WaitForMessageForwardedAsync(collector, c1); _ = await WaitForWorkerCreatedAsync(collector, forwarded.ReplacementContext); + await AssertCompletedSuccessfullyAsync(lateDelivery); var workerCreatedEvents = GetWorkerCreatedEvents(collector); var replacementCreations = Assert.Single(workerCreatedEvents); @@ -133,13 +137,14 @@ public async Task LateDeliveryAndFreshCallUseSingleReplacementActivation() $"Observed up to {tracker.MaxObserved} concurrent activations after a late delivery and a fresh call on a [StatelessWorker(maxLocalWorkers:1)] grain; expected at most 1."); } - private Message CreateLateDeliveryMessage(GrainId grainId) + private Message CreateLateDeliveryMessage(GrainId grainId, DoWorkRequest request) { var services = PrimarySilo.SiloHost.Services; var messageFactory = services.GetRequiredService(); var localSiloDetails = services.GetRequiredService(); - var message = messageFactory.CreateMessage(new DoWorkRequest(), InvokeMethodOptions.OneWay); + // Local-only messages carry an ILocalRequest, which receives the outcome of the delivery. + var message = messageFactory.CreateMessage(request, InvokeMethodOptions.OneWay); message.SetInfiniteTimeToLive(); message.IsLocalOnly = true; message.SendingGrain = grainId; @@ -149,6 +154,12 @@ private Message CreateLateDeliveryMessage(GrainId grainId) return message; } + private static async Task AssertCompletedSuccessfullyAsync(DoWorkRequest request) + { + var response = await request.Completion.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Null(response.Exception); + } + private static IReadOnlyList GetWorkerCreatedEvents(DiagnosticEventCollector collector) { return [.. collector.Events @@ -210,11 +221,16 @@ private Message CreateLateDeliveryMessage(GrainId grainId) return Assert.IsType(evt.Payload); } - private sealed class DoWorkRequest : IInvokable + private sealed class DoWorkRequest : ILocalRequest { private static readonly MethodInfo Method = typeof(IStatelessWorkerSingleActivationGrain).GetMethod(nameof(IStatelessWorkerSingleActivationGrain.DoWork))!; + private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously); private IStatelessWorkerSingleActivationGrain? _target; + public Task Completion => _completion.Task; + + public void Complete(Response response) => _completion.TrySetResult(response); + public object? GetTarget() => _target; public void SetTarget(ITargetHolder holder) From a668fd7d1abf37c87808f9e18f4f028ecfd1bb9d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 07:55:57 -0700 Subject: [PATCH 24/31] refactor(runtime): reject and forward unprocessed requests outside the activation lock --- src/Orleans.Runtime/Catalog/ActivationData.cs | 171 +++++++++--------- 1 file changed, 82 insertions(+), 89 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index eb404630954..029ce5ca0db 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -524,20 +524,13 @@ internal int GetRequestCount() } } - private List DequeueAllWaitingRequests(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) + private List DequeueAllWaitingRequests() { lock (_lock) { var result = new List(_waitingRequests.Count); foreach (var (message, _) in _waitingRequests) { - // Local-only messages are not allowed to escape the activation. - if (message.IsLocalOnly) - { - (rejectedLocalRequests ??= []).Add(((ILocalRequest)message.BodyObject!, new OrleansMessageRejectionException($"Activation {this} is no longer valid."))); - continue; - } - result.Add(message); } @@ -1176,31 +1169,17 @@ private async Task RunMessageLoop() void ProcessPendingRequests() { - List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests = null; - try - { - ProcessPendingRequestsCore(ref rejectedLocalRequests); - } - finally - { - // Rejection can run application code which queues further requests, so it runs once the queue is consistent and unlocked. - if (rejectedLocalRequests is not null) - { - foreach (var (request, exception) in rejectedLocalRequests) - { - LocalRequests.Complete(request, Response.FromException(exception), _shared.Logger); - } - } - } - } - - void ProcessPendingRequestsCore(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) - { + // Requests which this activation will not process are removed from the queue under the lock and rejected or forwarded + // after it is released, since that can deliver messages to other activations and complete local requests, which can + // queue further requests on this activation. + List? invalidRequests = null; + var rejectInvalidRequests = false; var i = 0; do { Message? message = null; + Exception? rejection = null; lock (_lock) { if (_waitingRequests.Count <= i) @@ -1214,7 +1193,7 @@ void ProcessPendingRequestsCore(ref List<(ILocalRequest Request, Exception Excep // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - ProcessRequestsToInvalidActivation(ref rejectedLocalRequests); + invalidRequests = DequeueRequestsToInvalidActivation(out rejectInvalidRequests); break; } @@ -1270,30 +1249,33 @@ void ProcessPendingRequestsCore(ref List<(ILocalRequest Request, Exception Excep } catch (Exception exception) { - if (!message.IsLocalOnly) - { - _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); - } - else - { - (rejectedLocalRequests ??= []).Add(((ILocalRequest)message.BodyObject!, exception)); - } - - _waitingRequests.RemoveAt(i); - continue; + rejection = exception; } - // Process this message, removing it from the queue. + // Remove the message from the queue to either process or reject it. _waitingRequests.RemoveAt(i); + if (rejection is null) + { + Debug.Assert(State == ActivationState.Valid || message.IsLocalOnly); + RecordRunning(message, message.IsAlwaysInterleave); + } + } - Debug.Assert(State == ActivationState.Valid || message.IsLocalOnly); - RecordRunning(message, message.IsAlwaysInterleave); + if (rejection is not null) + { + RejectRequest(message, rejection); + continue; } // Start invoking the message outside of the lock InvokeIncomingRequest(message); } while (true); + + if (invalidRequests is not null) + { + ProcessRequestsToInvalidActivation(invalidRequests, rejectInvalidRequests); + } } void RecordRunning(Message message, bool isInterleavable) @@ -1309,12 +1291,14 @@ void RecordRunning(Message message, bool isInterleavable) _busyDuration = stopwatch; } - void ProcessRequestsToInvalidActivation(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) + // Called with the lock held. Returns the requests to reject or forward, or null while the activation may still process them. + List? DequeueRequestsToInvalidActivation(out bool reject) { + reject = false; if (State is ActivationState.Creating or ActivationState.Activating) { // Do nothing until the activation becomes either valid or invalid - return; + return null; } if (State is ActivationState.Deactivating) @@ -1337,21 +1321,13 @@ void ProcessRequestsToInvalidActivation(ref List<(ILocalRequest Request, Excepti { // Do not forward messages while the grain is still deactivating and has not been declared stuck, since they // will be forwarded to the same grain instance. - return; + return null; } } - if (DeactivationException is null || ForwardingAddress is { }) - { - // Either this was a duplicate activation or it was at some point successfully activated - // Forward all pending messages - RerouteAllQueuedMessages(ref rejectedLocalRequests); - } - else - { - // Reject all pending messages - RejectAllQueuedMessages(ref rejectedLocalRequests); - } + // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. + reject = DeactivationException is not null && ForwardingAddress is null; + return DequeueAllWaitingRequests(); } void AbandonStuckDeactivatingActivation() @@ -1762,52 +1738,69 @@ private void ReceiveRequest(Message message) } /// - /// Rejects all messages enqueued for the provided activation. + /// Rejects a request which this activation removed from its queue without processing it. /// - private void RejectAllQueuedMessages(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) + private void RejectRequest(Message message, Exception exception) { - lock (_lock) + if (message.IsLocalOnly) { - List msgs = DequeueAllWaitingRequests(ref rejectedLocalRequests); - if (msgs == null || msgs.Count <= 0) return; - - LogRejectAllQueuedMessages(_shared.Logger, msgs.Count, this); - _shared.InternalRuntime.GrainLocator.InvalidateCache(Address); - _shared.InternalRuntime.MessageCenter.ProcessRequestsToInvalidActivation( - msgs, - Address, - forwardingAddress: ForwardingAddress, - failedOperation: DeactivationReason.Description, - exc: DeactivationException, - rejectMessages: true); + LocalRequests.Complete((ILocalRequest)message.BodyObject!, Response.FromException(exception), _shared.Logger); + } + else + { + _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); } } - private void RerouteAllQueuedMessages(ref List<(ILocalRequest Request, Exception Exception)>? rejectedLocalRequests) + /// + /// Rejects or forwards requests which this activation dequeued once it could no longer process them. + /// + /// + /// Local requests never leave the activation, so they are rejected. + /// + private void ProcessRequestsToInvalidActivation(List requests, bool reject) { - lock (_lock) + foreach (var message in requests) { - List msgs = DequeueAllWaitingRequests(ref rejectedLocalRequests); - if (msgs is not { Count: > 0 }) + if (message.IsLocalOnly) { - return; + LocalRequests.Complete( + (ILocalRequest)message.BodyObject!, + Response.FromException(new OrleansMessageRejectionException($"Activation {this} is no longer valid.")), + _shared.Logger); } + } - if (_shared.Logger.IsEnabled(LogLevel.Debug)) + requests.RemoveAll(static message => message.IsLocalOnly); + if (requests.Count == 0) + { + return; + } + + if (reject) + { + LogRejectAllQueuedMessages(_shared.Logger, requests.Count, this); + } + else if (_shared.Logger.IsEnabled(LogLevel.Debug)) + { + if (ForwardingAddress is { } address) { - if (ForwardingAddress is { } address) - { - LogReroutingMessages(_shared.Logger, msgs.Count, this, address); - } - else - { - LogReroutingMessagesNoForwarding(_shared.Logger, msgs.Count, this); - } + LogReroutingMessages(_shared.Logger, requests.Count, this, address); + } + else + { + LogReroutingMessagesNoForwarding(_shared.Logger, requests.Count, this); } - - _shared.InternalRuntime.GrainLocator.InvalidateCache(Address); - _shared.InternalRuntime.MessageCenter.ProcessRequestsToInvalidActivation(msgs, Address, ForwardingAddress, DeactivationReason.Description, DeactivationException); } + + _shared.InternalRuntime.GrainLocator.InvalidateCache(Address); + _shared.InternalRuntime.MessageCenter.ProcessRequestsToInvalidActivation( + requests, + Address, + ForwardingAddress, + DeactivationReason.Description, + DeactivationException, + rejectMessages: reject); } #region Activation From a5ef07c1af6b4b056fdd8122ed0b001f804ab50b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 08:02:43 -0700 Subject: [PATCH 25/31] refactor(runtime): decide invalid request handling at the call site --- src/Orleans.Runtime/Catalog/ActivationData.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 029ce5ca0db..19a853c7057 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1193,7 +1193,10 @@ void ProcessPendingRequests() // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - invalidRequests = DequeueRequestsToInvalidActivation(out rejectInvalidRequests); + invalidRequests = DequeueRequestsToInvalidActivation(); + + // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. + rejectInvalidRequests = DeactivationException is not null && ForwardingAddress is null; break; } @@ -1292,9 +1295,8 @@ void RecordRunning(Message message, bool isInterleavable) } // Called with the lock held. Returns the requests to reject or forward, or null while the activation may still process them. - List? DequeueRequestsToInvalidActivation(out bool reject) + List? DequeueRequestsToInvalidActivation() { - reject = false; if (State is ActivationState.Creating or ActivationState.Activating) { // Do nothing until the activation becomes either valid or invalid @@ -1325,8 +1327,6 @@ void RecordRunning(Message message, bool isInterleavable) } } - // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. - reject = DeactivationException is not null && ForwardingAddress is null; return DequeueAllWaitingRequests(); } From 4f14396b6a900b24c7b935988799ae29acb761e5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 08:19:59 -0700 Subject: [PATCH 26/31] refactor(runtime): rename invalid activation dequeue to TryDequeueUnprocessableRequests --- src/Orleans.Runtime/Catalog/ActivationData.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 19a853c7057..1da3d3875d1 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1193,10 +1193,12 @@ void ProcessPendingRequests() // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - invalidRequests = DequeueRequestsToInvalidActivation(); + if (TryDequeueUnprocessableRequests(out invalidRequests)) + { + // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. + rejectInvalidRequests = DeactivationException is not null && ForwardingAddress is null; + } - // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. - rejectInvalidRequests = DeactivationException is not null && ForwardingAddress is null; break; } @@ -1294,13 +1296,14 @@ void RecordRunning(Message message, bool isInterleavable) _busyDuration = stopwatch; } - // Called with the lock held. Returns the requests to reject or forward, or null while the activation may still process them. - List? DequeueRequestsToInvalidActivation() + // Called with the lock held. Dequeues the waiting requests once this activation can no longer process them. + bool TryDequeueUnprocessableRequests([NotNullWhen(true)] out List? requests) { + requests = null; if (State is ActivationState.Creating or ActivationState.Activating) { // Do nothing until the activation becomes either valid or invalid - return null; + return false; } if (State is ActivationState.Deactivating) @@ -1323,11 +1326,12 @@ void RecordRunning(Message message, bool isInterleavable) { // Do not forward messages while the grain is still deactivating and has not been declared stuck, since they // will be forwarded to the same grain instance. - return null; + return false; } } - return DequeueAllWaitingRequests(); + requests = DequeueAllWaitingRequests(); + return true; } void AbandonStuckDeactivatingActivation() From 38d1664032e93dafb74baeca65cc7395f877617b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 08:24:54 -0700 Subject: [PATCH 27/31] refactor(runtime): simplify rejecting and forwarding unprocessable requests --- src/Orleans.Runtime/Catalog/ActivationData.cs | 46 ++++++------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 1da3d3875d1..35541b9f34f 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -524,21 +524,6 @@ internal int GetRequestCount() } } - private List DequeueAllWaitingRequests() - { - lock (_lock) - { - var result = new List(_waitingRequests.Count); - foreach (var (message, _) in _waitingRequests) - { - result.Add(message); - } - - _waitingRequests.Clear(); - return result; - } - } - /// /// Returns how long this activation has been idle. /// @@ -1172,8 +1157,7 @@ void ProcessPendingRequests() // Requests which this activation will not process are removed from the queue under the lock and rejected or forwarded // after it is released, since that can deliver messages to other activations and complete local requests, which can // queue further requests on this activation. - List? invalidRequests = null; - var rejectInvalidRequests = false; + List? unprocessableRequests = null; var i = 0; do @@ -1193,12 +1177,7 @@ void ProcessPendingRequests() // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - if (TryDequeueUnprocessableRequests(out invalidRequests)) - { - // Forward pending messages if this was a duplicate activation or it was at some point successfully activated. Otherwise, reject them. - rejectInvalidRequests = DeactivationException is not null && ForwardingAddress is null; - } - + TryDequeueUnprocessableRequests(out unprocessableRequests); break; } @@ -1277,9 +1256,9 @@ void ProcessPendingRequests() } while (true); - if (invalidRequests is not null) + if (unprocessableRequests is not null) { - ProcessRequestsToInvalidActivation(invalidRequests, rejectInvalidRequests); + RejectOrForwardRequests(unprocessableRequests); } } @@ -1330,7 +1309,13 @@ bool TryDequeueUnprocessableRequests([NotNullWhen(true)] out List? requ } } - requests = DequeueAllWaitingRequests(); + requests = new List(_waitingRequests.Count); + foreach (var (message, _) in _waitingRequests) + { + requests.Add(message); + } + + _waitingRequests.Clear(); return true; } @@ -1760,18 +1745,16 @@ private void RejectRequest(Message message, Exception exception) /// Rejects or forwards requests which this activation dequeued once it could no longer process them. /// /// + /// Requests are forwarded if this was a duplicate activation or it was at some point successfully activated, and rejected otherwise. /// Local requests never leave the activation, so they are rejected. /// - private void ProcessRequestsToInvalidActivation(List requests, bool reject) + private void RejectOrForwardRequests(List requests) { foreach (var message in requests) { if (message.IsLocalOnly) { - LocalRequests.Complete( - (ILocalRequest)message.BodyObject!, - Response.FromException(new OrleansMessageRejectionException($"Activation {this} is no longer valid.")), - _shared.Logger); + RejectRequest(message, new OrleansMessageRejectionException($"Activation {this} is no longer valid.")); } } @@ -1781,6 +1764,7 @@ private void ProcessRequestsToInvalidActivation(List requests, bool rej return; } + var reject = DeactivationException is not null && ForwardingAddress is null; if (reject) { LogRejectAllQueuedMessages(_shared.Logger, requests.Count, this); From 0818b6384b2c7341930fd3329336db467c8e8c9e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 08:44:53 -0700 Subject: [PATCH 28/31] refactor(runtime): return unprocessable requests directly --- src/Orleans.Runtime/Catalog/ActivationData.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 35541b9f34f..5785d8cdcde 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1177,7 +1177,7 @@ void ProcessPendingRequests() // Local-only messages are used for internal system operations and should not be rejected while the grain is valid or deactivating. if (State != ActivationState.Valid && !(message.IsLocalOnly && State is ActivationState.Deactivating)) { - TryDequeueUnprocessableRequests(out unprocessableRequests); + unprocessableRequests = DequeueUnprocessableRequests(); break; } @@ -1275,14 +1275,14 @@ void RecordRunning(Message message, bool isInterleavable) _busyDuration = stopwatch; } - // Called with the lock held. Dequeues the waiting requests once this activation can no longer process them. - bool TryDequeueUnprocessableRequests([NotNullWhen(true)] out List? requests) + // Called with the lock held. Dequeues the waiting requests once this activation can no longer process them, + // or returns null while it may still process them. + List? DequeueUnprocessableRequests() { - requests = null; if (State is ActivationState.Creating or ActivationState.Activating) { // Do nothing until the activation becomes either valid or invalid - return false; + return null; } if (State is ActivationState.Deactivating) @@ -1305,18 +1305,18 @@ bool TryDequeueUnprocessableRequests([NotNullWhen(true)] out List? requ { // Do not forward messages while the grain is still deactivating and has not been declared stuck, since they // will be forwarded to the same grain instance. - return false; + return null; } } - requests = new List(_waitingRequests.Count); + var requests = new List(_waitingRequests.Count); foreach (var (message, _) in _waitingRequests) { requests.Add(message); } _waitingRequests.Clear(); - return true; + return requests; } void AbandonStuckDeactivatingActivation() From c05c00666392255d1196558327c5ffa2bad1204d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 09:40:18 -0700 Subject: [PATCH 29/31] refactor(runtime): complete and reject local requests from their messages --- src/Orleans.Runtime/Catalog/ActivationData.cs | 2 +- src/Orleans.Runtime/Core/InsideRuntimeClient.cs | 4 ++-- src/Orleans.Runtime/Core/LocalRequests.cs | 13 +++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 5785d8cdcde..6bbd50ede7c 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1733,7 +1733,7 @@ private void RejectRequest(Message message, Exception exception) { if (message.IsLocalOnly) { - LocalRequests.Complete((ILocalRequest)message.BodyObject!, Response.FromException(exception), _shared.Logger); + LocalRequests.Reject(message, exception, _shared.Logger); } else { diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 5cdc7b01125..23eff4c20d1 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -380,7 +380,7 @@ public async Task Invoke(IGrainContext target, Message message) // Completing it is the last step, so a failure before it completes the request exactly once. if (message.IsLocalOnly) { - LocalRequests.Complete((ILocalRequest)message.BodyObject!, response, this.logger); + LocalRequests.Complete(message, response, this.logger); } if (message.Direction != Message.Directions.OneWay) @@ -395,7 +395,7 @@ public async Task Invoke(IGrainContext target, Message message) LogWarningInvokeException(this.logger, exc2, message); if (message.IsLocalOnly) { - LocalRequests.Complete((ILocalRequest)message.BodyObject!, Response.FromException(exc2), this.logger); + LocalRequests.Reject(message, exc2, this.logger); } if (message.Direction != Message.Directions.OneWay) diff --git a/src/Orleans.Runtime/Core/LocalRequests.cs b/src/Orleans.Runtime/Core/LocalRequests.cs index e1ab6533241..20cb9f53f30 100644 --- a/src/Orleans.Runtime/Core/LocalRequests.cs +++ b/src/Orleans.Runtime/Core/LocalRequests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using Microsoft.Extensions.Logging; using Orleans.Serialization.Invocation; @@ -6,13 +7,15 @@ namespace Orleans.Runtime; internal static partial class LocalRequests { /// - /// Completes with and then disposes it. + /// Completes the local request which carries with and then disposes it. /// /// /// A failure in the request's completion or disposal is logged rather than thrown, so callers completing a batch of requests complete every request. /// - public static void Complete(ILocalRequest request, Response response, ILogger logger) + public static void Complete(Message message, Response response, ILogger logger) { + Debug.Assert(message.IsLocalOnly); + var request = (ILocalRequest)message.BodyObject!; try { try @@ -30,6 +33,12 @@ public static void Complete(ILocalRequest request, Response response, ILogger lo } } + /// + /// Completes the local request which carries with and then disposes it. + /// + /// + public static void Reject(Message message, Exception exception, ILogger logger) => Complete(message, Response.FromException(exception), logger); + [LoggerMessage( Level = LogLevel.Error, Message = "Error completing local request {Request}." From 24c8a59dc0de78ed6422d674f474e7a262dd7697 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 10:18:25 -0700 Subject: [PATCH 30/31] fix(runtime): pass local request responses by reference --- .../Core/IGrainBase.cs | 1 + .../Core/ILocalRequest.cs | 1 + src/Orleans.Core/Core/GrainMethodInvoker.cs | 6 ++- .../Core/InsideRuntimeClient.cs | 9 +++- .../TestGrainInterfaces/IRunRequestGrain.cs | 10 ++++ test/Grains/TestGrains/RunRequestGrain.cs | 50 +++++++++++++++++++ .../RunRequestTests.cs | 16 ++++++ 7 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index 9f02b67f79c..54be926cf18 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -278,6 +278,7 @@ public static Task RunRequestAsync(this IGrainBase gra /// The request passes through grain call filters as IGrainRequestInvoker.RunRequestAsync. /// It starts a new call chain and, by default, keeps the activation alive. /// A call filter which fails or sets the result without invoking the request completes the returned task with that outcome. + /// The callback's state and result are passed by reference rather than copied. /// /// /// Use this method to run code from a background task, such as a stream or messaging handler, as a grain request. diff --git a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs index 4585adcb867..2ae9d15e634 100644 --- a/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs +++ b/src/Orleans.Core.Abstractions/Core/ILocalRequest.cs @@ -8,6 +8,7 @@ namespace Orleans.Runtime; /// /// The grain context invokes the request on the activation, through its incoming grain call filters, passes the /// response to , and then disposes the request. Call filters, logs, and traces observe the request's interface name, method name, and arguments. +/// The request and its response are passed by reference rather than copied, so a request which shares mutable state with its caller copies it itself. /// public interface ILocalRequest : IInvokable { diff --git a/src/Orleans.Core/Core/GrainMethodInvoker.cs b/src/Orleans.Core/Core/GrainMethodInvoker.cs index ec3a0a6673a..52fa57c199d 100644 --- a/src/Orleans.Core/Core/GrainMethodInvoker.cs +++ b/src/Orleans.Core/Core/GrainMethodInvoker.cs @@ -132,7 +132,11 @@ public async Task Invoke() ExceptionDispatchInfo.Capture(exception).Throw(); } - this.Response = this.responseCopier.Copy(this.Response); + // A local request is passed by reference, and so is its response. + if (!message.IsLocalOnly) + { + this.Response = this.responseCopier.Copy(this.Response); + } return; } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 23eff4c20d1..5a202c65e25 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -330,8 +330,13 @@ public async Task Invoke(IGrainContext target, Message message) else { response = await invokable.Invoke(); - // The copier preserves the null state of its input. - response = this.responseCopier.Copy(response)!; + + // A local request is passed by reference, and so is its response. + if (!message.IsLocalOnly) + { + // The copier preserves the null state of its input. + response = this.responseCopier.Copy(response)!; + } } // A local request is disposed once it is completed. diff --git a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs index 5259f044753..3e9c448f15e 100644 --- a/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs +++ b/test/Grains/TestGrainInterfaces/IRunRequestGrain.cs @@ -13,6 +13,8 @@ public interface IRunRequestGrain : IGrainWithGuidKey Task RunCustomRequestLifecycle(); + Task ReceiveLocalRequestReturnsResultInstance(); + Task QueueRequests(int count); Task StartBlockingRequest(); @@ -55,6 +57,14 @@ public interface IReentrantRunRequestGrain : IRunRequestGrain { } +/// +/// A grain without incoming grain call filters, whose local requests run without a call filter chain. +/// +public interface IUnfilteredRunRequestGrain : IGrainWithGuidKey +{ + Task ReceiveLocalRequestReturnsResultInstance(); +} + /// /// A non-reentrant grain whose [MayInterleave] predicate admits custom local requests. /// diff --git a/test/Grains/TestGrains/RunRequestGrain.cs b/test/Grains/TestGrains/RunRequestGrain.cs index b283529d69a..e7f4d64ffd5 100644 --- a/test/Grains/TestGrains/RunRequestGrain.cs +++ b/test/Grains/TestGrains/RunRequestGrain.cs @@ -82,6 +82,8 @@ public Task StartCustomInvokableRequest(string value) return Task.CompletedTask; } + public Task ReceiveLocalRequestReturnsResultInstance() => ResultInstanceRequest.ReturnsResultInstanceAsync(this); + public async Task RunCustomRequestLifecycle() { // The request interleaves so that it runs while this call waits for it. @@ -265,6 +267,54 @@ public void Complete(Response response) public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; } +/// +/// A local request which returns a mutable result instance and records the response it receives. +/// +internal sealed class ResultInstanceRequest(List result) : Request>, ILocalRequest +{ + private readonly TaskCompletionSource _outcome = new(TaskCreationOptions.RunContinuationsAsynchronously); + private object? _grain; + + /// + /// Returns whether a local request on completes with the same result instance which it returned. + /// + public static async Task ReturnsResultInstanceAsync(IGrainBase grain) + { + var result = new List { "mutable" }; + var request = new ResultInstanceRequest(result); + + // The request interleaves so that it runs while the calling request waits for it. + grain.GrainContext.ReceiveLocalRequest(request, new LocalRequestOptions { Interleave = true }); + var response = await request._outcome.Task.WaitAsync(TimeSpan.FromSeconds(10)); + return ReferenceEquals(result, response.Result); + } + + public void Complete(Response response) => _outcome.TrySetResult(response); + + public override object GetTarget() => _grain!; + + public override void SetTarget(ITargetHolder holder) => _grain = holder.GetTarget(); + + protected override ValueTask> InvokeInner() => new(result); + + public override void Dispose() { } + + public override string GetMethodName() => "ReturnResult"; + + public override string GetInterfaceName() => CustomRequest.InterfaceName; + + public override string GetActivityName() => $"{CustomRequest.InterfaceName}/ReturnResult"; + + public override Type GetInterfaceType() => typeof(IRunRequestGrain); + + public override MethodInfo GetMethod() => typeof(IRunRequestGrain).GetMethod(nameof(IRunRequestGrain.Ping))!; +} + +public sealed class UnfilteredRunRequestGrain : Grain, IUnfilteredRunRequestGrain +{ + public Task ReceiveLocalRequestReturnsResultInstance() => ResultInstanceRequest.ReturnsResultInstanceAsync(this); +} + public sealed class NonReentrantRunRequestGrain : RunRequestGrainBase, INonReentrantRunRequestGrain { } diff --git a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs index 504397dec24..141d97a55c0 100644 --- a/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs +++ b/test/Orleans.DefaultCluster.Tests/RunRequestTests.cs @@ -145,6 +145,22 @@ public async Task ReceiveLocalRequest_CustomRequest_ReceivesResponse() Assert.Contains("ICustomRequests.Apply", await grain.GetFilteredCalls()); } + [Fact] + public async Task ReceiveLocalRequest_ThroughCallFilter_CompletesWithResultInstance() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + Assert.True(await grain.ReceiveLocalRequestReturnsResultInstance().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ReceiveLocalRequest_WithoutCallFilters_CompletesWithResultInstance() + { + var grain = GrainFactory.GetGrain(Guid.NewGuid()); + + Assert.True(await grain.ReceiveLocalRequestReturnsResultInstance().WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + [Fact] public async Task ReceiveLocalRequest_CustomRequest_IsDisposedAfterCompletion() { From 01549a133ab4d07850e626972bef1f8c26586191 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 16 Oct 2026 13:07:54 -0700 Subject: [PATCH 31/31] fix(runtime): give local requests a constant interface type --- src/Orleans.Core/Messaging/MessageFactory.cs | 4 ++++ test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index 77d28982489..660e7713c79 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -52,6 +52,9 @@ private CorrelationId GetNextCorrelationId() return new CorrelationId(unchecked((long)id)); } + // Incoming grain calls carry an interface type. Local requests share this one, and call filters observe each request's own interface name. + private static readonly GrainInterfaceType LocalRequestInterfaceType = GrainInterfaceType.Create("Orleans.Runtime.ILocalRequest"); + /// /// Creates a one-way request which queues on itself. /// @@ -69,6 +72,7 @@ public Message CreateLocalRequest(IGrainContext target, ILocalRequest request, L message.TargetGrain = target.GrainId; message.SendingSilo = siloAddress; message.TargetSilo = siloAddress; + message.InterfaceType = LocalRequestInterfaceType; message.IsKeepAlive = options.KeepAlive; message.IsAlwaysInterleave = options.Interleave; message.IsLocalOnly = true; diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs index 0e07e64248a..0c46037b58d 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -78,7 +78,7 @@ public void MessageFactory_CreateLocalRequest_CreatesOneWayLocalMessage(bool int Assert.True(message.IsLocalOnly); Assert.Null(message.TimeToLive); Assert.Null(message.RequestContextData); - Assert.True(message.InterfaceType.IsDefault); + Assert.Equal(GrainInterfaceType.Create("Orleans.Runtime.ILocalRequest"), message.InterfaceType); Assert.Equal(fixture.Grain.GrainId, message.TargetGrain); Assert.Equal(message.SendingGrain, message.TargetGrain); Assert.Equal(fixture.Grain.Address.SiloAddress, message.TargetSilo);