Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,30 @@ public async Task DeleteMembershipTableEntriesAsync(string clusterId, Cancellati
public async Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var entities = await this._storage.ReadAllEntities<SiloInstanceEntity>(cancellationToken);
var defunctEntries = entities
.Where(entity => entity.Id != this._partitionId)
.Where(entity => entity.Status != (int)SiloStatus.Active)
.Where(entity => GetEffectiveUpdateTime(entity) < beforeDate)
.ToArray();

await Task.WhenAll(defunctEntries.Chunk(FirestoreDataManager.MaxBatchSize)
.Select(chunk => this._storage.DeleteEntities(chunk, cancellationToken)));
var query = this._storage.GetCollection()
.WhereEqualTo(nameof(SiloInstanceEntity.Status), (int)SiloStatus.Dead);
while (await this._storage.ExecuteTransaction(async transaction =>
{
var snapshot = await transaction.GetSnapshotAsync(query, transaction.CancellationToken);
var defunct = snapshot.Documents
.Where(document => GetEffectiveUpdateTime(document.ConvertTo<SiloInstanceEntity>()) < beforeDate)
.Take(FirestoreDataManager.MaxBatchSize)
.ToArray();
if (defunct.Length == 0)
{
return false;
}

foreach (var document in defunct)
{
transaction.Delete(document.Reference, Precondition.LastUpdated(document.UpdateTime!.Value));
}

return true;
}, cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
}
}

[Obsolete("Use ReadRowAsync instead.")]
Expand Down Expand Up @@ -211,26 +226,28 @@ public async Task<bool> UpdateRowAsync(MembershipEntry entry, string etag, Table
LogUpdateRow(entry, etag, tableVersion);

var silo = SiloInstanceEntity.FromMembershipEntry(entry, tableVersion.Version);
silo.ETag = Utils.ParseTimestamp(etag);
var version = CreateClusterVersionEntity(tableVersion.Version);
version.ETag = Utils.ParseTimestamp(tableVersion.VersionEtag);

var collection = this._storage.GetCollection();
var siloReference = collection.Document(silo.Id);
var versionReference = collection.Document(this._partitionId);
bool result;
try
var result = false;
if (string.Equals(etag, tableVersion.VersionEtag, StringComparison.Ordinal))
{
result = await this._storage.ExecuteTransaction(transaction =>
try
{
transaction.Update(siloReference, silo.GetFields(), Precondition.LastUpdated(silo.ETag.Value));
transaction.Update(versionReference, version.GetFields(), Precondition.LastUpdated(version.ETag.Value));
return Task.FromResult(true);
}, cancellationToken);
}
catch (RpcException exception) when (IsContention(exception))
{
result = false;
result = await this._storage.ExecuteTransaction(transaction =>
{
transaction.Update(siloReference, silo.GetFields(), Precondition.MustExist);
transaction.Update(versionReference, version.GetFields(), Precondition.LastUpdated(version.ETag.Value));
return Task.FromResult(true);
}, cancellationToken);
}
catch (RpcException exception) when (IsContention(exception))
{
result = false;
}
}

if (result == false)
Expand All @@ -257,23 +274,13 @@ public async Task UpdateIAmAliveAsync(MembershipEntry entry, CancellationToken c
var id = entry.SiloAddress.ToParsableString();
var iAmAliveTime = new DateTimeOffset(DateTime.SpecifyKind(entry.IAmAliveTime, DateTimeKind.Utc));
var document = this._storage.GetCollection().Document(id);
await this._storage.ExecuteTransaction(async transaction =>
{
var snapshot = await transaction.GetSnapshotAsync(document, transaction.CancellationToken);
if (!snapshot.Exists)
throw new KeyNotFoundException($"Could not find silo entry for {id}");

if (snapshot.ConvertTo<SiloInstanceEntity>().IAmAliveTime >= iAmAliveTime)
{
return false;
}

transaction.Update(document, new Dictionary<string, object?>
{
[nameof(SiloInstanceEntity.IAmAliveTime)] = iAmAliveTime,
});
return true;
}, cancellationToken);
await document.UpdateAsync(
nameof(SiloInstanceEntity.IAmAliveTime), iAmAliveTime, cancellationToken: cancellationToken);
Comment thread
ReubenBond marked this conversation as resolved.
Resolved
Comment thread
ReubenBond marked this conversation as resolved.
Resolved
}
catch (RpcException exception) when (
cancellationToken.IsCancellationRequested && exception.StatusCode == StatusCode.Cancelled)
{
throw new OperationCanceledException("The Firestore operation was canceled.", exception, cancellationToken);
}
catch (Exception exc) when (exc is not OperationCanceledException)
{
Expand Down Expand Up @@ -328,10 +335,12 @@ private static DateTimeOffset GetEffectiveUpdateTime(SiloInstanceEntity entity)

private static MembershipTableData Convert((SiloInstanceEntity[] Silos, ClusterVersionEntity Version) data)
{
// Row tokens identify the canonical table version, which advances only with membership mutations.
var version = data.Version.ToTableVersion();
return new MembershipTableData
(
data.Silos.Select(s => Tuple.Create(s.ToMembershipEntry(), Utils.FormatTimestamp(s.ETag!.Value))).ToList(),
data.Version.ToTableVersion()
data.Silos.Select(s => Tuple.Create(s.ToMembershipEntry(), version.VersionEtag)).ToList(),
version
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,23 +95,47 @@ public async Task FindAllGatewayProxyEndpoints()
}

[Fact]
public async Task UpdateIAmAliveDoesNotOverwriteNewerHeartbeat()
public async Task UpdateIAmAlivePreservesMembershipFieldsAndVersion()
{
var current = TestStartTime.AddMinutes(2);
await WriteSiloInstance(SiloStatus.Active, TestContext.Current.CancellationToken, current);

var before = await this._membershipTable.ReadRowAsync(this._siloAddress, TestContext.Current.CancellationToken);
var expected = Assert.Single(before.Members).Item1;
expected.IAmAliveTime = current.AddMinutes(1).UtcDateTime;
var entry = this._entity.ToMembershipEntry();
entry.IAmAliveTime = current.AddMinutes(-1).UtcDateTime;
entry.IAmAliveTime = expected.IAmAliveTime;
entry.HostName = "stale-host";
entry.Status = SiloStatus.Joining;
await this._membershipTable.UpdateIAmAliveAsync(entry, TestContext.Current.CancellationToken);

var row = await this._membershipTable.ReadRowAsync(entry.SiloAddress, TestContext.Current.CancellationToken);
Assert.Equal(current.UtcDateTime, Assert.Single(row.Members).Item1.IAmAliveTime);

entry.IAmAliveTime = current.AddMinutes(1).UtcDateTime;
await this._membershipTable.UpdateIAmAliveAsync(entry, TestContext.Current.CancellationToken);
var after = await this._membershipTable.ReadRowAsync(entry.SiloAddress, TestContext.Current.CancellationToken);
Assert.Equal(expected.ToFullString(), Assert.Single(after.Members).Item1.ToFullString());
Assert.Equal(Assert.Single(before.Members).Item2, Assert.Single(after.Members).Item2);
Assert.Equal(before.Version.Version, after.Version.Version);
Assert.Equal(before.Version.VersionEtag, after.Version.VersionEtag);
}

row = await this._membershipTable.ReadRowAsync(entry.SiloAddress, TestContext.Current.CancellationToken);
Assert.Equal(entry.IAmAliveTime, Assert.Single(row.Members).Item1.IAmAliveTime);
[Fact]
public async Task UpdateRowUsesOriginalCanonicalTokensAfterHeartbeat()
{
await WriteSiloInstance(SiloStatus.Active, TestContext.Current.CancellationToken);
var before = await this._membershipTable.ReadRowAsync(this._siloAddress, TestContext.Current.CancellationToken);
var original = Assert.Single(before.Members);
var heartbeat = this._entity.ToMembershipEntry();
heartbeat.IAmAliveTime = TestStartTime.AddMinutes(1).UtcDateTime;
await this._membershipTable.UpdateIAmAliveAsync(heartbeat, TestContext.Current.CancellationToken);

original.Item1.Status = SiloStatus.ShuttingDown;
Assert.True(await this._membershipTable.UpdateRowAsync(
original.Item1, original.Item2, before.Version.Next(), TestContext.Current.CancellationToken));

var after = await this._membershipTable.ReadRowAsync(this._siloAddress, TestContext.Current.CancellationToken);
var updated = Assert.Single(after.Members);
Assert.Equal(original.Item1.ToFullString(), updated.Item1.ToFullString());
Assert.Equal(before.Version.Version + 1, after.Version.Version);
Assert.NotEqual(before.Version.VersionEtag, after.Version.VersionEtag);
Assert.Equal(after.Version.VersionEtag, updated.Item2);
}

[Fact]
Expand Down
Loading
Loading