diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs index 7c876932c6c..27cf9b74ece 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs @@ -20,6 +20,9 @@ namespace Orleans.Runtime.GrainDirectory; /// internal sealed partial class GrainDirectoryPartition : SystemTarget, IGrainDirectoryPartition, IGrainDirectoryTestHooks { + private const float MaxActivationQueryRangePercent = 5f; + private const int MaxActivationQueryRangeCount = 32; + internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartitionType, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture)); private readonly Dictionary _directory = []; private readonly int _partitionIndex; @@ -515,18 +518,29 @@ private async Task TransferSnapshotAsync( var stopwatch = ValueStopwatch.StartNew(); var partition = GetPartitionReference(previousOwner, partitionIndex); + var transferredEntries = 0; var waitedForRange = false; - foreach (var transferRange in GetSnapshotTransferRanges(previousOwnerRange, addedRange)) + var previousTransferRange = RingRange.Empty; + foreach (var (transferRange, queryRange) in GetSnapshotTransferQueryRanges(previousOwnerRange, addedRange)) { - currentTransferRange = transferRange; - LogTraceRequestingEntries(_logger, transferRange, previousOwner, previousVersion); + if (transferRange != previousTransferRange) + { + if (!previousTransferRange.IsEmpty) + { + LogDebugTransferredEntries(_logger, transferredEntries, previousTransferRange, previousOwner); + transferredEntries = 0; + } + + currentTransferRange = previousTransferRange = transferRange; + LogTraceRequestingEntries(_logger, transferRange, previousOwner, previousVersion); + } // Alternatively, the previous owner could push the snapshot. The pull-based approach is used here because it is simpler. - var snapshot = await partition.GetSnapshotAsync(current.Version, previousVersion, transferRange).AsTask().WaitAsync(ShutdownToken); + var snapshot = await partition.GetSnapshotAsync(current.Version, previousVersion, queryRange).AsTask().WaitAsync(ShutdownToken); if (snapshot is null) { - LogWarningExpectedValidSnapshot(_logger, previousOwner, transferRange); + LogWarningExpectedValidSnapshot(_logger, previousOwner, queryRange); return false; } @@ -546,7 +560,12 @@ private async Task TransferSnapshotAsync( _directory[entry.GrainId] = entry; } - LogDebugTransferredEntries(_logger, snapshot.GrainAddresses.Count, transferRange, previousOwner); + transferredEntries += snapshot.GrainAddresses.Count; + } + + if (!previousTransferRange.IsEmpty) + { + LogDebugTransferredEntries(_logger, transferredEntries, previousTransferRange, previousOwner); } // The acknowledgement step lets the previous owner know that the snapshot has been received so that it can proceed. @@ -604,26 +623,29 @@ private async IAsyncEnumerable> GetRegisteredActivations(Dire var clusterMembershipSnapshot = _owner.ClusterMembershipSnapshot; Debug.Assert(clusterMembershipSnapshot.Version >= current.Version); - var tasks = new List>>(); - foreach (var member in clusterMembershipSnapshot.Members.Values) + foreach (var queryRange in GetActivationQueryRanges(range)) { - if (member.Status is not (SiloStatus.Active or SiloStatus.Joining or SiloStatus.ShuttingDown)) + var tasks = new List>>(); + foreach (var member in clusterMembershipSnapshot.Members.Values) { - continue; - } + if (member.Status is not (SiloStatus.Active or SiloStatus.Joining or SiloStatus.ShuttingDown)) + { + continue; + } - tasks.Add(GetRegisteredActivationsFromClusterMember(current.Version, range, member.SiloAddress, isValidation)); - } + tasks.Add(GetRegisteredActivationsFromClusterMember(current.Version, queryRange, member.SiloAddress, isValidation)); + } - await Task.WhenAll(tasks).WaitAsync(ShutdownToken).SuppressThrowing(); - if (ShutdownToken.IsCancellationRequested) - { - yield break; - } + await Task.WhenAll(tasks).WaitAsync(ShutdownToken).SuppressThrowing(); + if (ShutdownToken.IsCancellationRequested) + { + yield break; + } - foreach (var task in tasks) - { - yield return await task; + foreach (var task in tasks) + { + yield return await task; + } } async Task> GetRegisteredActivationsFromClusterMember(MembershipVersion version, RingRange range, SiloAddress siloAddress, bool isValidation) @@ -661,6 +683,67 @@ internal static IEnumerable GetSnapshotTransferRanges(RingRange previ return previousOwnerRange.Intersections(addedRange); } + // Directory recovery and validation can temporarily involve very large ranges. + // Split those requests so the returned GrainAddress list stays comfortably under MaxMessageBodySize. + internal static IEnumerable<(RingRange TransferRange, RingRange QueryRange)> GetSnapshotTransferQueryRanges(RingRange previousOwnerRange, RingRange addedRange) + { + foreach (var transferRange in GetSnapshotTransferRanges(previousOwnerRange, addedRange)) + { + foreach (var queryRange in GetActivationQueryRanges(transferRange)) + { + yield return (transferRange, queryRange); + } + } + } + + internal static IEnumerable GetActivationQueryRanges(RingRange range) + { + if (range.IsEmpty) + { + yield break; + } + + var queryRangeCount = Math.Clamp( + (int)Math.Ceiling(range.SizePercent / MaxActivationQueryRangePercent), + 1, + MaxActivationQueryRangeCount); + if (queryRangeCount == 1) + { + yield return range; + yield break; + } + + var totalSize = GetRangeCoverageSize(range); + queryRangeCount = (int)Math.Min((ulong)queryRangeCount, totalSize); + if (queryRangeCount == 1) + { + yield return range; + yield break; + } + + var (baseSize, remainder) = Math.DivRem(totalSize, (ulong)queryRangeCount); + var start = range.Start; + for (var i = 0; i < queryRangeCount; i++) + { + var size = baseSize + ((ulong)i < remainder ? 1UL : 0UL); + var end = unchecked((uint)((ulong)start + size)); + yield return RingRange.Create(start, end); + start = end; + } + + static ulong GetRangeCoverageSize(RingRange range) + { + if (range.IsFull) + { + return (ulong)uint.MaxValue + 1; + } + + return range.IsWrapped + ? (ulong)uint.MaxValue - range.Start + range.End + 1 + : range.Size; + } + } + private async Task InvokeOnClusterMember(SiloAddress siloAddress, Func> func, T defaultValue, string operationName) { GrainRuntime.CheckRuntimeContext(this); diff --git a/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryPartitionBatchingTests.cs b/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryPartitionBatchingTests.cs new file mode 100644 index 00000000000..cbc9e9e1574 --- /dev/null +++ b/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryPartitionBatchingTests.cs @@ -0,0 +1,97 @@ +#nullable enable +using System.Linq; +using Orleans.Runtime.GrainDirectory; +using TestExtensions; +using Xunit; + +namespace UnitTests.GrainDirectory; + +[TestCategory("BVT"), TestCategory("Directory")] +public sealed class GrainDirectoryPartitionBatchingTests +{ + [Fact] + public void GetActivationQueryRanges_ReturnsNoRangesForEmptyRange() + { + var ranges = GrainDirectoryPartition.GetActivationQueryRanges(RingRange.Empty).ToArray(); + + Assert.Empty(ranges); + } + + [Fact] + public void GetActivationQueryRanges_ReturnsOriginalRangeForSmallRange() + { + var range = RingRange.Create(100, 200); + + var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray(); + + var result = Assert.Single(ranges); + Assert.Equal(range, result); + } + + [Fact] + public void GetActivationQueryRanges_SplitsLargeRangeIntoContiguousBalancedRanges() + { + var range = RingRange.Create(0x1000, 0x30001000); + + var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray(); + + Assert.True(ranges.Length > 1); + AssertRangePartition(range, ranges); + AssertBalancedRanges(ranges); + } + + [Fact] + public void GetActivationQueryRanges_SplitsWrappedRangeIntoContiguousBalancedRanges() + { + var range = RingRange.Create(0xD0000000, 0x20000000); + + var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray(); + + Assert.True(ranges.Length > 1); + AssertRangePartition(range, ranges); + AssertBalancedRanges(ranges); + } + + [Fact] + public void GetActivationQueryRanges_SplitsFullRangeAcrossEntireRing() + { + var ranges = GrainDirectoryPartition.GetActivationQueryRanges(RingRange.Full).ToArray(); + + Assert.True(ranges.Length > 1); + AssertRangePartition(RingRange.Full, ranges); + AssertBalancedRanges(ranges); + } + + private static void AssertRangePartition(RingRange range, RingRange[] ranges) + { + Assert.NotEmpty(ranges); + Assert.All(ranges, static candidate => Assert.False(candidate.IsEmpty)); + Assert.Equal(range.Start, ranges[0].Start); + Assert.Equal(range.End, ranges[^1].End); + Assert.Equal(GetRangeSize(range), ranges.Aggregate(0UL, static (sum, range) => sum + GetRangeSize(range))); + + for (var i = 1; i < ranges.Length; i++) + { + Assert.Equal(ranges[i - 1].End, ranges[i].Start); + } + } + + private static void AssertBalancedRanges(RingRange[] ranges) + { + var sizes = ranges.Select(GetRangeSize).ToArray(); + + Assert.True(sizes.Max() - sizes.Min() <= 1); + } + + private static ulong GetRangeSize(RingRange range) + { + if (range.IsFull) + { + return (ulong)uint.MaxValue + 1; + } + + return range.IsWrapped + ? (ulong)uint.MaxValue - range.Start + range.End + 1 + : range.Size; + } +}