Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 104 additions & 21 deletions src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ namespace Orleans.Runtime.GrainDirectory;
/// </summary>
internal sealed partial class GrainDirectoryPartition : SystemTarget, IGrainDirectoryPartition, IGrainDirectoryTestHooks
{
private const float MaxActivationQueryRangePercent = 5f;
private const int MaxActivationQueryRangeCount = 32;

internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartitionType, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture));
private readonly Dictionary<GrainId, GrainAddress> _directory = [];
private readonly int _partitionIndex;
Expand Down Expand Up @@ -515,18 +518,29 @@ private async Task<bool> TransferSnapshotAsync(
var stopwatch = ValueStopwatch.StartNew();

var partition = GetPartitionReference(previousOwner, partitionIndex);
var transferredEntries = 0;
var waitedForRange = false;
foreach (var transferRange in GetSnapshotTransferRanges(previousOwnerRange, addedRange))
var previousTransferRange = RingRange.Empty;
foreach (var (transferRange, queryRange) in GetSnapshotTransferQueryRanges(previousOwnerRange, addedRange))
{
currentTransferRange = transferRange;
LogTraceRequestingEntries(_logger, transferRange, previousOwner, previousVersion);
if (transferRange != previousTransferRange)
{
if (!previousTransferRange.IsEmpty)
{
LogDebugTransferredEntries(_logger, transferredEntries, previousTransferRange, previousOwner);
transferredEntries = 0;
}

currentTransferRange = previousTransferRange = transferRange;
LogTraceRequestingEntries(_logger, transferRange, previousOwner, previousVersion);
}

// Alternatively, the previous owner could push the snapshot. The pull-based approach is used here because it is simpler.
var snapshot = await partition.GetSnapshotAsync(current.Version, previousVersion, transferRange).AsTask().WaitAsync(ShutdownToken);
var snapshot = await partition.GetSnapshotAsync(current.Version, previousVersion, queryRange).AsTask().WaitAsync(ShutdownToken);

if (snapshot is null)
{
LogWarningExpectedValidSnapshot(_logger, previousOwner, transferRange);
LogWarningExpectedValidSnapshot(_logger, previousOwner, queryRange);
return false;
}

Expand All @@ -546,7 +560,12 @@ private async Task<bool> TransferSnapshotAsync(
_directory[entry.GrainId] = entry;
}

LogDebugTransferredEntries(_logger, snapshot.GrainAddresses.Count, transferRange, previousOwner);
transferredEntries += snapshot.GrainAddresses.Count;
}

if (!previousTransferRange.IsEmpty)
{
LogDebugTransferredEntries(_logger, transferredEntries, previousTransferRange, previousOwner);
}

// The acknowledgement step lets the previous owner know that the snapshot has been received so that it can proceed.
Expand Down Expand Up @@ -604,26 +623,29 @@ private async IAsyncEnumerable<List<GrainAddress>> GetRegisteredActivations(Dire
var clusterMembershipSnapshot = _owner.ClusterMembershipSnapshot;
Debug.Assert(clusterMembershipSnapshot.Version >= current.Version);

var tasks = new List<Task<List<GrainAddress>>>();
foreach (var member in clusterMembershipSnapshot.Members.Values)
foreach (var queryRange in GetActivationQueryRanges(range))
{
if (member.Status is not (SiloStatus.Active or SiloStatus.Joining or SiloStatus.ShuttingDown))
var tasks = new List<Task<List<GrainAddress>>>();
foreach (var member in clusterMembershipSnapshot.Members.Values)
{
continue;
}
if (member.Status is not (SiloStatus.Active or SiloStatus.Joining or SiloStatus.ShuttingDown))
{
continue;
}

tasks.Add(GetRegisteredActivationsFromClusterMember(current.Version, range, member.SiloAddress, isValidation));
}
tasks.Add(GetRegisteredActivationsFromClusterMember(current.Version, queryRange, member.SiloAddress, isValidation));
}

await Task.WhenAll(tasks).WaitAsync(ShutdownToken).SuppressThrowing();
if (ShutdownToken.IsCancellationRequested)
{
yield break;
}
await Task.WhenAll(tasks).WaitAsync(ShutdownToken).SuppressThrowing();
if (ShutdownToken.IsCancellationRequested)
{
yield break;
}

foreach (var task in tasks)
{
yield return await task;
foreach (var task in tasks)
{
yield return await task;
}
}

async Task<List<GrainAddress>> GetRegisteredActivationsFromClusterMember(MembershipVersion version, RingRange range, SiloAddress siloAddress, bool isValidation)
Expand Down Expand Up @@ -661,6 +683,67 @@ internal static IEnumerable<RingRange> GetSnapshotTransferRanges(RingRange previ
return previousOwnerRange.Intersections(addedRange);
}

// Directory recovery and validation can temporarily involve very large ranges.
// Split those requests so the returned GrainAddress list stays comfortably under MaxMessageBodySize.
internal static IEnumerable<(RingRange TransferRange, RingRange QueryRange)> GetSnapshotTransferQueryRanges(RingRange previousOwnerRange, RingRange addedRange)
{
foreach (var transferRange in GetSnapshotTransferRanges(previousOwnerRange, addedRange))
{
foreach (var queryRange in GetActivationQueryRanges(transferRange))
{
yield return (transferRange, queryRange);
}
}
}

internal static IEnumerable<RingRange> GetActivationQueryRanges(RingRange range)
{
if (range.IsEmpty)
{
yield break;
}

var queryRangeCount = Math.Clamp(
(int)Math.Ceiling(range.SizePercent / MaxActivationQueryRangePercent),
1,
MaxActivationQueryRangeCount);
if (queryRangeCount == 1)
{
yield return range;
yield break;
}

var totalSize = GetRangeCoverageSize(range);
queryRangeCount = (int)Math.Min((ulong)queryRangeCount, totalSize);
if (queryRangeCount == 1)
{
yield return range;
yield break;
}

var (baseSize, remainder) = Math.DivRem(totalSize, (ulong)queryRangeCount);
var start = range.Start;
for (var i = 0; i < queryRangeCount; i++)
{
var size = baseSize + ((ulong)i < remainder ? 1UL : 0UL);
var end = unchecked((uint)((ulong)start + size));
yield return RingRange.Create(start, end);
start = end;
}

static ulong GetRangeCoverageSize(RingRange range)
{
if (range.IsFull)
{
return (ulong)uint.MaxValue + 1;
}

return range.IsWrapped
? (ulong)uint.MaxValue - range.Start + range.End + 1
: range.Size;
}
}

private async Task<T> InvokeOnClusterMember<T>(SiloAddress siloAddress, Func<Task<T>> func, T defaultValue, string operationName)
{
GrainRuntime.CheckRuntimeContext(this);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#nullable enable
using System.Linq;
using Orleans.Runtime.GrainDirectory;
using TestExtensions;
using Xunit;

namespace UnitTests.GrainDirectory;

[TestCategory("BVT"), TestCategory("Directory")]
public sealed class GrainDirectoryPartitionBatchingTests
{
[Fact]
public void GetActivationQueryRanges_ReturnsNoRangesForEmptyRange()
{
var ranges = GrainDirectoryPartition.GetActivationQueryRanges(RingRange.Empty).ToArray();

Assert.Empty(ranges);
}

[Fact]
public void GetActivationQueryRanges_ReturnsOriginalRangeForSmallRange()
{
var range = RingRange.Create(100, 200);

var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray();

var result = Assert.Single(ranges);
Assert.Equal(range, result);
}

[Fact]
public void GetActivationQueryRanges_SplitsLargeRangeIntoContiguousBalancedRanges()
{
var range = RingRange.Create(0x1000, 0x30001000);

var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray();

Assert.True(ranges.Length > 1);
AssertRangePartition(range, ranges);
AssertBalancedRanges(ranges);
}

[Fact]
public void GetActivationQueryRanges_SplitsWrappedRangeIntoContiguousBalancedRanges()
{
var range = RingRange.Create(0xD0000000, 0x20000000);

var ranges = GrainDirectoryPartition.GetActivationQueryRanges(range).ToArray();

Assert.True(ranges.Length > 1);
AssertRangePartition(range, ranges);
AssertBalancedRanges(ranges);
}

[Fact]
public void GetActivationQueryRanges_SplitsFullRangeAcrossEntireRing()
{
var ranges = GrainDirectoryPartition.GetActivationQueryRanges(RingRange.Full).ToArray();

Assert.True(ranges.Length > 1);
AssertRangePartition(RingRange.Full, ranges);
AssertBalancedRanges(ranges);
}

private static void AssertRangePartition(RingRange range, RingRange[] ranges)
{
Assert.NotEmpty(ranges);
Assert.All(ranges, static candidate => Assert.False(candidate.IsEmpty));
Assert.Equal(range.Start, ranges[0].Start);
Assert.Equal(range.End, ranges[^1].End);
Assert.Equal(GetRangeSize(range), ranges.Aggregate(0UL, static (sum, range) => sum + GetRangeSize(range)));

for (var i = 1; i < ranges.Length; i++)
{
Assert.Equal(ranges[i - 1].End, ranges[i].Start);
}
}

private static void AssertBalancedRanges(RingRange[] ranges)
{
var sizes = ranges.Select(GetRangeSize).ToArray();

Assert.True(sizes.Max() - sizes.Min() <= 1);
}

private static ulong GetRangeSize(RingRange range)
{
if (range.IsFull)
{
return (ulong)uint.MaxValue + 1;
}

return range.IsWrapped
? (ulong)uint.MaxValue - range.Start + range.End + 1
: range.Size;
}
}
Loading