diff --git a/docs/site/src/content/docs/grains/grain-persistence/dynamodb-storage.md b/docs/site/src/content/docs/grains/grain-persistence/dynamodb-storage.md index 93efa0b46d0..61a2f0761f7 100644 --- a/docs/site/src/content/docs/grains/grain-persistence/dynamodb-storage.md +++ b/docs/site/src/content/docs/grains/grain-persistence/dynamodb-storage.md @@ -20,6 +20,8 @@ The [AWS SDK for .NET credential and profile resolution chain](https://docs.aws. must remain stable across deployments that share the same logical application state. The provider uses optimistic concurrency and rejects stale writes. +When is empty, the keys start with an underscore instead of , unlike the other grain storage providers, and the provider logs a warning. Set to use instead. State written before then stays under its old key unless is also set: a grain with no state under the new key then reads it from the old one, and its next write moves it, in one transaction; a grain that has state under the new key keeps it, and its next write or clear deletes the old item in the same transaction. Set together with : once the keys have been built from without a migration, the provider refuses to start one, since a grain whose state there has since been deleted or has expired would read back its older state. Enable the migration once every silo runs a version that supports it, with the same options on every silo: a silo still writing the old keys, from an older version or with set to `false`, is not protected from a migrating silo moving or deleting state it recreates, since an item recreated after starts again at ETag 0. While the migration is on, every write or clear is a transaction over two items, which uses more write capacity; set to `false` when every grain has been written, to spare that and the second read for grains that have no state. The provider records its choice in the table, once for the whole table, so every provider that uses the table with an empty must have the same options. It keeps that choice while these options are left unset, and refuses to start with set to `false` once the table records , since the state written since would not be found. + ## Serialization Set to customize the stored representation. Changing serializers doesn't rewrite existing items, so the replacement must read the previous representation or be accompanied by a migration. diff --git a/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageProviderBuilder.cs b/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageProviderBuilder.cs index 59eb923a631..0c915535338 100644 --- a/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageProviderBuilder.cs +++ b/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageProviderBuilder.cs @@ -56,6 +56,16 @@ public void Configure(ISiloBuilder builder, string? name, IConfigurationSection options.ServiceId = serviceId; } + if (bool.TryParse(configurationSection[nameof(options.UseClusterServiceId)], out var useClusterServiceId)) + { + options.UseClusterServiceId = useClusterServiceId; + } + + if (bool.TryParse(configurationSection[nameof(options.MigrateLegacyKeys)], out var migrateLegacyKeys)) + { + options.MigrateLegacyKeys = migrateLegacyKeys; + } + var tableName = configurationSection[nameof(options.TableName)]; if (!string.IsNullOrEmpty(tableName)) { diff --git a/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageServiceCollectionExtensions.cs b/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageServiceCollectionExtensions.cs index 70ffaa3e60e..1b000753e12 100644 --- a/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageServiceCollectionExtensions.cs +++ b/src/AWS/Orleans.Persistence.DynamoDB/Hosting/DynamoDBGrainStorageServiceCollectionExtensions.cs @@ -49,6 +49,7 @@ public static IServiceCollection AddDynamoDBGrainStorage(this IServiceCollection services.AddTransient(sp => new DynamoDBGrainStorageOptionsValidator(sp.GetRequiredService>().Get(name), name)); services.ConfigureNamedOptionForLogging(name); services.AddTransient, DefaultStorageProviderSerializerOptionsConfigurator>(); + services.TryAddEnumerable(ServiceDescriptor.Transient, DynamoDBStorageClusterServiceIdConfigurator>()); return services.AddGrainStorage(name, DynamoDBGrainStorageFactory.Create); } } diff --git a/src/AWS/Orleans.Persistence.DynamoDB/Options/DynamoDBStorageOptions.cs b/src/AWS/Orleans.Persistence.DynamoDB/Options/DynamoDBStorageOptions.cs index b85ae7ce454..8e0ebbc49d6 100644 --- a/src/AWS/Orleans.Persistence.DynamoDB/Options/DynamoDBStorageOptions.cs +++ b/src/AWS/Orleans.Persistence.DynamoDB/Options/DynamoDBStorageOptions.cs @@ -1,4 +1,7 @@ using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration.Overrides; using Orleans.Persistence.DynamoDB; using Orleans.Runtime; using Orleans.Storage; @@ -12,9 +15,37 @@ public class DynamoDBStorageOptions : DynamoDBClientOptions, IStorageProviderSer { /// /// Gets or sets a unique identifier for this service, which should survive deployment and redeployment. + /// When left empty, see . /// public string ServiceId { get; set; } = string.Empty; + /// + /// Gets or sets whether an empty falls back to , + /// as the other grain storage providers do. When , an empty is used + /// as it is, and every key starts with an underscore; initialization fails if the table already records + /// . When not set, the key format already recorded in the table is kept, and a + /// new table uses the empty ; this default changes in a future major version. The key format + /// is recorded once for the whole table. Has no effect when is set. + /// + public bool? UseClusterServiceId { get; set; } + + /// + /// Gets or sets whether state written with an empty is read when a grain has no state under + /// , and moved there on the grain's next write; a grain that has state there + /// keeps it, and its next write or clear deletes the legacy item as well, in a transaction over both items. When not + /// set, a migration recorded in the table goes on. Has an effect only when is + /// used for an empty , and set it together with : + /// initialization fails if the table records without a migration. Every silo + /// must run with the same options while it is set: a silo still writing the keys with an empty + /// is not protected from a migrating one. + /// + public bool? MigrateLegacyKeys { get; set; } + + /// + /// The of the silo, used when applies. + /// + internal string ClusterServiceId { get; set; } = ClusterOptions.DefaultServiceId; + /// /// Use Provisioned Throughput for tables /// @@ -71,6 +102,23 @@ public class DynamoDBStorageOptions : DynamoDBClientOptions, IStorageProviderSer public IGrainStorageSerializer GrainStorageSerializer { get; set; } = null!; } + /// + /// Supplies with the of the silo. + /// + internal sealed class DynamoDBStorageClusterServiceIdConfigurator(IServiceProvider serviceProvider) : IPostConfigureOptions + { + public void PostConfigure(string? name, DynamoDBStorageOptions options) + { + // a provider-specific override first, then the silo's + var clusterOptions = serviceProvider.GetProviderClusterOptions(name ?? Microsoft.Extensions.Options.Options.DefaultName).Value; + + if (clusterOptions.ServiceId is { Length: > 0 } serviceId) + { + options.ClusterServiceId = serviceId; + } + } + } + /// /// Configuration validator for DynamoDBStorageOptions /// diff --git a/src/AWS/Orleans.Persistence.DynamoDB/Provider/DynamoDBGrainStorage.cs b/src/AWS/Orleans.Persistence.DynamoDB/Provider/DynamoDBGrainStorage.cs index aef0ab23eb2..c217c5e6f43 100755 --- a/src/AWS/Orleans.Persistence.DynamoDB/Provider/DynamoDBGrainStorage.cs +++ b/src/AWS/Orleans.Persistence.DynamoDB/Provider/DynamoDBGrainStorage.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Globalization; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -31,6 +32,14 @@ public partial class DynamoDBGrainStorage : IGrainStorage, ILifecycleParticipant private const string ETAG_PROPERTY_NAME = "ETag"; private const string GRAIN_TTL_PROPERTY_NAME = "GrainTtl"; private const string CURRENT_ETAG_ALIAS = ":currentETag"; + private const string KEY_FORMAT_MARKER = "__OrleansKeyFormat"; + private const string KEY_FORMAT_PROPERTY_NAME = "KeyFormat"; + private const string LEGACY_KEY_FORMAT = "EmptyServiceId"; + private const string CLUSTER_KEY_FORMAT = "ClusterServiceId"; + private const string MIGRATING_KEY_FORMAT = "MigratingToClusterServiceId"; + + // prefixes the ETag of a state read from its legacy key, which its next write or clear moves + private const string LEGACY_ETAG_PREFIX = "legacy:"; private readonly DynamoDBStorageOptions options; private readonly IActivatorProvider _activatorProvider; @@ -38,6 +47,11 @@ public partial class DynamoDBGrainStorage : IGrainStorage, ILifecycleParticipant private readonly string name; private DynamoDBStorage storage = null!; + private string _keyServiceId = string.Empty; + private bool _migrateLegacyKeys; + + /// Runs between reading and writing the key format record, so that tests can interleave another silo. + internal Func? BeforeKeyFormatWriteForTesting { get; set; } /// /// Default Constructor @@ -99,6 +113,8 @@ await storage.InitializeTable(this.options.TableName, secondaryIndexes: null, ttlAttributeName: this.options.TimeToLive.HasValue ? GRAIN_TTL_PROPERTY_NAME : null, cancellationToken: ct); + var keyFormat = await ResolveKeyFormatAsync(ct); + LogInformationKeyFormat(logger, this.name, this.options.TableName, _keyServiceId, keyFormat); stopWatch.Stop(); LogInformationProviderInitialized(logger, this.name, this.GetType().Name, this.options.InitStage, stopWatch.ElapsedMilliseconds); } @@ -127,30 +143,25 @@ public async Task ReadStateAsync(string grainType, GrainId grainId, IGrainSta LogTraceReadingGrainState(logger, grainType, partitionKey, grainId, this.options.TableName); string rowKey = AWSUtils.ValidateDynamoDBRowKey(grainType); + var record = await ReadRecordAsync(partitionKey, rowKey); - var record = await this.storage.ReadSingleEntryAsync(this.options.TableName, - new Dictionary - { - { GRAIN_REFERENCE_PROPERTY_NAME, new AttributeValue(partitionKey) }, - { GRAIN_TYPE_PROPERTY_NAME, new AttributeValue(rowKey) } - }, - (fields) => + if (record == null && _migrateLegacyKeys) + { + record = await ReadRecordAsync(GetLegacyKeyString(grainId), rowKey); + if (record != null) { - return new GrainStateRecord - { - GrainType = fields[GRAIN_TYPE_PROPERTY_NAME].S, - GrainReference = fields[GRAIN_REFERENCE_PROPERTY_NAME].S, - ETag = int.Parse(fields[ETAG_PROPERTY_NAME].N, NumberStyles.Integer, CultureInfo.InvariantCulture), - State = fields.TryGetValue(BINARY_STATE_PROPERTY_NAME, out var propertyName) ? propertyName.B?.ToArray() : null, - }; - }).ConfigureAwait(false); + LogWarningReadingLegacyKey(logger, grainType, grainId, this.options.TableName); + } + } if (record != null) { var loadedState = ConvertFromStorageFormat(record); grainState.RecordExists = loadedState != null; grainState.State = loadedState ?? CreateInstance(); - grainState.ETag = record.ETag.ToString(CultureInfo.InvariantCulture); + grainState.ETag = record.GrainReference == partitionKey + ? record.ETag.ToString(CultureInfo.InvariantCulture) + : LEGACY_ETAG_PREFIX + record.ETag.ToString(CultureInfo.InvariantCulture); } else { @@ -172,12 +183,27 @@ public async Task WriteStateAsync(string grainType, GrainId grainId, IGrainSt try { ConvertToStorageFormat(grainState.State, record); - await WriteStateInternal(grainState, record); + if (TryParseLegacyETag(grainState.ETag, out var legacyETag)) + { + await MigrateStateAsync(grainState, record, GetLegacyKeyString(grainId), legacyETag, clear: false); + } + else if (_migrateLegacyKeys && !string.IsNullOrWhiteSpace(grainState.ETag)) + { + await WriteAndRetireLegacyAsync(grainState, record, GetLegacyKeyString(grainId), clear: false); + } + else + { + await WriteStateInternal(grainState, record); + } } catch (ConditionalCheckFailedException exc) { throw new InconsistentStateException($"Inconsistent grain state: {exc}"); } + catch (TransactionCanceledException exc) when (IsConditionalCheckFailure(exc)) + { + throw new InconsistentStateException($"Inconsistent grain state: {exc}"); + } catch (Exception exc) { LogErrorWritingGrainState(logger, exc, grainType, grainId, grainState.ETag, this.options.TableName); @@ -188,19 +214,12 @@ public async Task WriteStateAsync(string grainType, GrainId grainId, IGrainSt private async Task WriteStateInternal(IGrainState grainState, GrainStateRecord record, bool clear = false) { var fields = new Dictionary(); - if (this.options.TimeToLive.HasValue) + if (TtlAttribute() is { } ttl) { - fields.Add(GRAIN_TTL_PROPERTY_NAME, new AttributeValue { N = ((DateTimeOffset)DateTime.UtcNow.Add(this.options.TimeToLive.Value)).ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture) }); + fields.Add(GRAIN_TTL_PROPERTY_NAME, ttl); } - if (record.State != null && record.State.Length > 0) - { - fields.Add(BINARY_STATE_PROPERTY_NAME, new AttributeValue { B = new MemoryStream(record.State) }); - } - else - { - fields.Add(BINARY_STATE_PROPERTY_NAME, new AttributeValue { NULL = true }); - } + fields.Add(BINARY_STATE_PROPERTY_NAME, StateAttribute(record.State)); int newEtag = 0; if (clear) @@ -275,11 +294,12 @@ public async Task ClearStateAsync(string grainType, GrainId grainId, IGrainSt LogTraceClearingGrainState(logger, grainType, partitionKey, grainId, grainState.ETag, this.options.DeleteStateOnClear, this.options.TableName); string rowKey = AWSUtils.ValidateDynamoDBRowKey(grainType); + var fromLegacyKey = TryParseLegacyETag(grainState.ETag, out var legacyETag); var record = new GrainStateRecord { GrainReference = partitionKey, - ETag = string.IsNullOrWhiteSpace(grainState.ETag) - ? 0 + ETag = fromLegacyKey ? legacyETag + : string.IsNullOrWhiteSpace(grainState.ETag) ? 0 : int.Parse(grainState.ETag, NumberStyles.Integer, CultureInfo.InvariantCulture), GrainType = rowKey }; @@ -287,7 +307,22 @@ public async Task ClearStateAsync(string grainType, GrainId grainId, IGrainSt var operation = "Clearing"; try { - if (this.options.DeleteStateOnClear) + if (fromLegacyKey) + { + operation = "Migrating"; + if (this.options.DeleteStateOnClear) + { + await DeleteLegacyRecordAsync(partitionKey, GetLegacyKeyString(grainId), rowKey, legacyETag); + ResetGrainState(grainState); + } + else + { + await MigrateStateAsync(grainState, record, GetLegacyKeyString(grainId), legacyETag, clear: true); + grainState.State = CreateInstance(); + grainState.RecordExists = false; + } + } + else if (this.options.DeleteStateOnClear) { operation = "Deleting"; var keys = new Dictionary @@ -303,16 +338,43 @@ public async Task ClearStateAsync(string grainType, GrainId grainId, IGrainSt expression = $"{ETAG_PROPERTY_NAME} = {CURRENT_ETAG_ALIAS}"; } - await this.storage.DeleteEntryAsync(this.options.TableName, keys, expression, conditionalValues).ConfigureAwait(false); + if (_migrateLegacyKeys && !string.IsNullOrWhiteSpace(grainState.ETag)) + { + // a legacy item left beside the current one, by a write made before the migration was enabled, + // would come back through the read fallback once the current item is gone: it goes with it. + // A state never read has no ETag, and its clear leaves whatever is there, as before. + await this.storage.WriteTxAsync(deletes: + [ + new Delete { TableName = this.options.TableName, Key = keys, ConditionExpression = expression, ExpressionAttributeValues = conditionalValues }, + new Delete { TableName = this.options.TableName, Key = Keys(GetLegacyKeyString(grainId), record.GrainType) }, + ]); + } + else + { + await this.storage.DeleteEntryAsync(this.options.TableName, keys, expression, conditionalValues).ConfigureAwait(false); + } + ResetGrainState(grainState); } else { - await WriteStateInternal(grainState, record, true); + if (_migrateLegacyKeys && !string.IsNullOrWhiteSpace(grainState.ETag)) + { + await WriteAndRetireLegacyAsync(grainState, record, GetLegacyKeyString(grainId), clear: true); + } + else + { + await WriteStateInternal(grainState, record, true); + } + grainState.State = CreateInstance(); grainState.RecordExists = false; } } + catch (TransactionCanceledException exc) when (IsConditionalCheckFailure(exc)) + { + throw new InconsistentStateException($"Inconsistent grain state: {exc}"); + } catch (ConditionalCheckFailedException exc) { throw new InconsistentStateException($"Inconsistent grain state: {exc}"); @@ -334,10 +396,268 @@ internal class GrainStateRecord private string GetKeyString(GrainId grainId) { - var key = $"{options.ServiceId}_{grainId}"; + var key = $"{_keyServiceId}_{grainId}"; return AWSUtils.ValidateDynamoDBPartitionKey(key); } + private static string GetLegacyKeyString(GrainId grainId) => AWSUtils.ValidateDynamoDBPartitionKey($"_{grainId}"); + + private Task ReadRecordAsync(string partitionKey, string rowKey) => + this.storage.ReadSingleEntryAsync(this.options.TableName, + Keys(partitionKey, rowKey), + (fields) => + { + return new GrainStateRecord + { + GrainType = fields[GRAIN_TYPE_PROPERTY_NAME].S, + GrainReference = fields[GRAIN_REFERENCE_PROPERTY_NAME].S, + ETag = int.Parse(fields[ETAG_PROPERTY_NAME].N, NumberStyles.Integer, CultureInfo.InvariantCulture), + State = fields.TryGetValue(BINARY_STATE_PROPERTY_NAME, out var propertyName) ? propertyName.B?.ToArray() : null, + }; + }); + + /// + /// Decides the the keys are built from. An explicit one is used as + /// it is. An empty one is kept for compatibility, with a warning, unless the options or the key format recorded in + /// the table ask for ; the choice is recorded in the table. + /// + private async Task ResolveKeyFormatAsync(CancellationToken ct) + { + if (!string.IsNullOrEmpty(this.options.ServiceId)) + { + _keyServiceId = this.options.ServiceId; + if (this.options.UseClusterServiceId is not null || this.options.MigrateLegacyKeys is not null) + { + LogWarningKeyFormatOptionsIgnored(logger, this.name); + } + + return nameof(DynamoDBStorageOptions.ServiceId); + } + + // Silos starting together may choose differently: the record is written only if it still holds what was read, + // and a silo that loses decides again from what the other one wrote. + string format; + while (true) + { + var recordedFormat = await ReadKeyFormatAsync(ct); + if (recordedFormat is not (null or LEGACY_KEY_FORMAT or CLUSTER_KEY_FORMAT or MIGRATING_KEY_FORMAT)) + { + // a format this version does not know, from a newer one: following it wrongly would split the state + throw new OrleansConfigurationException( + $"DynamoDB Grain Storage {this.name} cannot start: table {this.options.TableName} records " + + (recordedFormat.Length == 0 ? "an empty key format" : $"the key format '{recordedFormat}'") + + ", which this version does not know. Run a version that supports it."); + } + + // Going back to the empty ServiceId would hide the state written since under ClusterOptions.ServiceId, and + // every silo that follows the record would go back with this one. + if (this.options.UseClusterServiceId == false && recordedFormat is CLUSTER_KEY_FORMAT or MIGRATING_KEY_FORMAT) + { + throw new OrleansConfigurationException( + $"DynamoDB Grain Storage {this.name} cannot start: {nameof(DynamoDBStorageOptions.UseClusterServiceId)} is false, " + + $"but table {this.options.TableName} records that its keys are built from ClusterOptions.ServiceId ('{recordedFormat}'), " + + $"so the state written since would not be found. Set {nameof(DynamoDBStorageOptions.UseClusterServiceId)} to true or " + + $"leave it unset. To go back to an empty ServiceId on purpose, delete the {KEY_FORMAT_MARKER} item first."); + } + + // Grains written under ClusterOptions.ServiceId without a migration leave their legacy item behind, and the + // read fallback would bring it back for those whose current item has been deleted or has expired. + if (this.options.MigrateLegacyKeys == true && recordedFormat == CLUSTER_KEY_FORMAT) + { + throw new OrleansConfigurationException( + $"DynamoDB Grain Storage {this.name} cannot start: {nameof(DynamoDBStorageOptions.MigrateLegacyKeys)} is true, " + + $"but table {this.options.TableName} records that its keys have been built from ClusterOptions.ServiceId without " + + "a migration, so a grain whose state there has since been deleted or has expired would read back its older state " + + $"from the key without ServiceId. Set {nameof(DynamoDBStorageOptions.MigrateLegacyKeys)} to false or leave it unset. " + + $"To migrate anyway, when no such state has been deleted or has expired, delete the {KEY_FORMAT_MARKER} item first."); + } + + var useClusterServiceId = this.options.UseClusterServiceId ?? recordedFormat is CLUSTER_KEY_FORMAT or MIGRATING_KEY_FORMAT; + + _keyServiceId = useClusterServiceId ? this.options.ClusterServiceId : string.Empty; + _migrateLegacyKeys = useClusterServiceId && (this.options.MigrateLegacyKeys ?? recordedFormat == MIGRATING_KEY_FORMAT); + + format = !useClusterServiceId ? LEGACY_KEY_FORMAT + : _migrateLegacyKeys ? MIGRATING_KEY_FORMAT + : CLUSTER_KEY_FORMAT; + if (recordedFormat == format || await TryWriteKeyFormatAsync(recordedFormat, format, ct)) + { + break; + } + } + + if (string.IsNullOrEmpty(_keyServiceId)) + { + LogWarningEmptyServiceId(logger, this.name, this.options.TableName); + if (this.options.MigrateLegacyKeys == true) + { + LogWarningMigrationIgnored(logger, this.name, this.options.TableName); + } + } + + return format; + } + + private async Task TryWriteKeyFormatAsync(string? recordedFormat, string format, CancellationToken ct) + { + if (BeforeKeyFormatWriteForTesting is { } beforeWrite) + { + await beforeWrite(); + } + + try + { + var fields = Keys(KEY_FORMAT_MARKER, KEY_FORMAT_MARKER); + fields.Add(KEY_FORMAT_PROPERTY_NAME, new AttributeValue(format)); + await this.storage.PutEntryAsync(this.options.TableName, fields, ct, + recordedFormat is null ? $"attribute_not_exists({GRAIN_REFERENCE_PROPERTY_NAME})" : $"{KEY_FORMAT_PROPERTY_NAME} = :recordedFormat", + recordedFormat is null ? null : new Dictionary { { ":recordedFormat", new AttributeValue(recordedFormat) } }); + return true; + } + catch (ConditionalCheckFailedException) + { + return false; + } + } + + private async Task ReadKeyFormatAsync(CancellationToken ct) + { + var marker = await this.storage.ReadSingleEntryAsync(this.options.TableName, + Keys(KEY_FORMAT_MARKER, KEY_FORMAT_MARKER), + fields => fields.TryGetValue(KEY_FORMAT_PROPERTY_NAME, out var value) ? value.S ?? string.Empty : string.Empty, + ct); + + // null is no record; a record without a value is one this version cannot follow, and is refused as an unknown one + return marker; + } + + /// + /// Writes the state of a grain read from its legacy key under its current key, and deletes the legacy record in the + /// same transaction, so that the grain never has both or neither. + /// + private async Task MigrateStateAsync(IGrainState grainState, GrainStateRecord record, string legacyPartitionKey, int legacyETag, bool clear) + { + var newETag = legacyETag + 1; + var fields = Keys(record.GrainReference, record.GrainType); + fields.Add(ETAG_PROPERTY_NAME, new AttributeValue { N = newETag.ToString(CultureInfo.InvariantCulture) }); + fields.Add(BINARY_STATE_PROPERTY_NAME, StateAttribute(record.State)); + if (TtlAttribute() is { } ttl) + { + fields.Add(GRAIN_TTL_PROPERTY_NAME, ttl); + } + else if (await ReadLegacyTtlAsync(legacyPartitionKey, record.GrainType) is { } legacyTtl) + { + // a write without TimeToLive leaves the expiry of an item as it is, and so does the migration; the + // transaction fails if the legacy item was written since, so the expiry read here is the current one + fields.Add(GRAIN_TTL_PROPERTY_NAME, legacyTtl); + } + + await this.storage.WriteTxAsync( + puts: [new Put + { + TableName = this.options.TableName, + Item = fields, + ConditionExpression = $"attribute_not_exists({GRAIN_REFERENCE_PROPERTY_NAME}) AND attribute_not_exists({GRAIN_TYPE_PROPERTY_NAME})", + }], + deletes: [LegacyDelete(legacyPartitionKey, record.GrainType, legacyETag)]); + + grainState.ETag = newETag.ToString(CultureInfo.InvariantCulture); + grainState.RecordExists = !clear; + } + + /// + /// Writes or clears a state read from the current key, and deletes the grain's legacy item in the same transaction. + /// An item written while the migration was off leaves its legacy item behind, which the read fallback would bring + /// back once the current item is gone, through or a delete. + /// + private async Task WriteAndRetireLegacyAsync(IGrainState grainState, GrainStateRecord record, string legacyPartitionKey, bool clear) + { + // no item has an ETag that is not a number, so it cannot be the current state; taken as 0, it would pass the + // condition on an item written once + if (!int.TryParse(grainState.ETag, NumberStyles.Integer, CultureInfo.InvariantCulture, out var currentETag)) + { + throw new InconsistentStateException($"Inconsistent grain state: the ETag '{grainState.ETag}' is not one this provider writes"); + } + + var newETag = currentETag + 1; + var values = new Dictionary + { + { CURRENT_ETAG_ALIAS, new AttributeValue { N = currentETag.ToString(CultureInfo.InvariantCulture) } }, + { ":newETag", new AttributeValue { N = newETag.ToString(CultureInfo.InvariantCulture) } }, + { ":state", StateAttribute(record.State) }, + }; + var update = $"SET {ETAG_PROPERTY_NAME} = :newETag, {BINARY_STATE_PROPERTY_NAME} = :state"; + if (TtlAttribute() is { } ttl) + { + values.Add(":ttl", ttl); + update += $", {GRAIN_TTL_PROPERTY_NAME} = :ttl"; + } + + await this.storage.WriteTxAsync( + updates: [new Update + { + TableName = this.options.TableName, + Key = Keys(record.GrainReference, record.GrainType), + UpdateExpression = update, + ConditionExpression = $"{ETAG_PROPERTY_NAME} = {CURRENT_ETAG_ALIAS}", + ExpressionAttributeValues = values, + }], + deletes: [new Delete { TableName = this.options.TableName, Key = Keys(legacyPartitionKey, record.GrainType) }]); + + grainState.ETag = newETag.ToString(CultureInfo.InvariantCulture); + grainState.RecordExists = !clear; + } + + /// The ETag of a state read from its legacy key, which its next write or clear moves. + private static bool TryParseLegacyETag(string? eTag, out int legacyETag) + { + legacyETag = 0; + return eTag is not null + && eTag.StartsWith(LEGACY_ETAG_PREFIX, StringComparison.Ordinal) + && int.TryParse(eTag.AsSpan(LEGACY_ETAG_PREFIX.Length), NumberStyles.Integer, CultureInfo.InvariantCulture, out legacyETag); + } + + private async Task ReadLegacyTtlAsync(string legacyPartitionKey, string rowKey) + { + var fields = await this.storage.ReadSingleEntryAsync(this.options.TableName, Keys(legacyPartitionKey, rowKey), fields => fields); + + return fields is not null && fields.TryGetValue(GRAIN_TTL_PROPERTY_NAME, out var ttl) ? ttl : null; + } + + private static bool IsConditionalCheckFailure(TransactionCanceledException exception) => + exception.CancellationReasons?.Any(reason => string.Equals(reason.Code, "ConditionalCheckFailed", StringComparison.Ordinal)) is true; + + private Task DeleteLegacyRecordAsync(string partitionKey, string legacyPartitionKey, string rowKey, int legacyETag) => + this.storage.WriteTxAsync( + deletes: [LegacyDelete(legacyPartitionKey, rowKey, legacyETag)], + conditionChecks: [new ConditionCheck + { + TableName = this.options.TableName, + Key = Keys(partitionKey, rowKey), + ConditionExpression = $"attribute_not_exists({GRAIN_REFERENCE_PROPERTY_NAME})", + }]); + + private Delete LegacyDelete(string legacyPartitionKey, string rowKey, int legacyETag) => new() + { + TableName = this.options.TableName, + Key = Keys(legacyPartitionKey, rowKey), + ConditionExpression = $"{ETAG_PROPERTY_NAME} = {CURRENT_ETAG_ALIAS}", + ExpressionAttributeValues = new Dictionary { { CURRENT_ETAG_ALIAS, new AttributeValue { N = legacyETag.ToString(CultureInfo.InvariantCulture) } } }, + }; + + private static Dictionary Keys(string partitionKey, string rowKey) => new() + { + { GRAIN_REFERENCE_PROPERTY_NAME, new AttributeValue(partitionKey) }, + { GRAIN_TYPE_PROPERTY_NAME, new AttributeValue(rowKey) } + }; + + private static AttributeValue StateAttribute(byte[]? state) => + state is { Length: > 0 } ? new AttributeValue { B = new MemoryStream(state) } : new AttributeValue { NULL = true }; + + private AttributeValue? TtlAttribute() => this.options.TimeToLive is { } timeToLive + ? new AttributeValue { N = ((DateTimeOffset)DateTime.UtcNow.Add(timeToLive)).ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture) } + : null; + internal T? ConvertFromStorageFormat(GrainStateRecord entity) { T? dataValue = default; @@ -447,6 +767,36 @@ private void ResetGrainState(IGrainState grainState) Message = "Writing binary data size = {DataSize} for grain id = Partition={Partition} / Row={Row}" )] private static partial void LogTraceWritingBinaryData(ILogger logger, int dataSize, string partition, string row); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "DynamoDB Grain Storage {Name} has no ServiceId, so the keys in {TableName} start with an underscore rather than ClusterOptions.ServiceId as in the other providers. Set ServiceId, or set UseClusterServiceId to use ClusterOptions.ServiceId, with MigrateLegacyKeys to move the existing state. A future major version uses ClusterOptions.ServiceId by default." + )] + private static partial void LogWarningEmptyServiceId(ILogger logger, string name, string tableName); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Read the state of GrainType={GrainType} GrainId={GrainId} from its key without ServiceId in {TableName}; it is moved on the next write." + )] + private static partial void LogWarningReadingLegacyKey(ILogger logger, string grainType, GrainId grainId, string tableName); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "DynamoDB Grain Storage {Name} builds the keys in {TableName} from the ServiceId '{KeyServiceId}' (key format {KeyFormat})." + )] + private static partial void LogInformationKeyFormat(ILogger logger, string name, string tableName, string keyServiceId, string keyFormat); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "DynamoDB Grain Storage {Name} has a ServiceId, so UseClusterServiceId and MigrateLegacyKeys have no effect: state written with an empty ServiceId is neither read nor moved." + )] + private static partial void LogWarningKeyFormatOptionsIgnored(ILogger logger, string name); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "DynamoDB Grain Storage {Name} keeps an empty ServiceId in {TableName}, so MigrateLegacyKeys has no effect. Set UseClusterServiceId to true to move the state to ClusterOptions.ServiceId." + )] + private static partial void LogWarningMigrationIgnored(ILogger logger, string name, string tableName); } /// diff --git a/src/api/AWS/Orleans.Persistence.DynamoDB/Orleans.Persistence.DynamoDB.cs b/src/api/AWS/Orleans.Persistence.DynamoDB/Orleans.Persistence.DynamoDB.cs index e57cef7497c..430569fa217 100644 --- a/src/api/AWS/Orleans.Persistence.DynamoDB/Orleans.Persistence.DynamoDB.cs +++ b/src/api/AWS/Orleans.Persistence.DynamoDB/Orleans.Persistence.DynamoDB.cs @@ -26,6 +26,8 @@ public partial class DynamoDBStorageOptions : Persistence.DynamoDB.DynamoDBClien public int InitStage { get { throw null; } set { } } + public bool? MigrateLegacyKeys { get { throw null; } set { } } + public int ReadCapacityUnits { get { throw null; } set { } } public string ServiceId { get { throw null; } set { } } @@ -36,6 +38,8 @@ public partial class DynamoDBStorageOptions : Persistence.DynamoDB.DynamoDBClien public bool UpdateIfExists { get { throw null; } set { } } + public bool? UseClusterServiceId { get { throw null; } set { } } + public bool UseProvisionedThroughput { get { throw null; } set { } } public int WriteCapacityUnits { get { throw null; } set { } } diff --git a/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageKeyFormatTests.cs b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageKeyFormatTests.cs new file mode 100644 index 00000000000..33f6e480446 --- /dev/null +++ b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageKeyFormatTests.cs @@ -0,0 +1,552 @@ +using Amazon.DynamoDBv2.Model; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.AWSUtils.Tests; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Storage; +using TestExtensions; +using UnitTests.Persistence; +using Xunit; + +namespace AWSUtils.Tests.StorageTests; + +/// +/// Tests the keys DynamoDB grain storage builds when is empty. +/// +[TestCategory("Persistence"), TestCategory("AWS"), TestCategory("DynamoDb")] +[Collection(TestEnvironmentFixture.DefaultCollection)] +[TestSuite("Functional")] +[TestProvider("DynamoDB")] +[TestArea("Persistence")] +public class DynamoDBGrainStorageKeyFormatTests : IAsyncLifetime +{ + private const string GrainType = "KeyFormatGrain"; + private const string ClusterServiceId = "cluster-service"; + + private readonly TestEnvironmentFixture _fixture; + private readonly string _tableName = $"KeyFormat{Guid.NewGuid():N}"; + + public DynamoDBGrainStorageKeyFormatTests(TestEnvironmentFixture fixture) + { + _fixture = fixture; + } + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public async ValueTask DisposeAsync() + { + // every test has its own table, since the key format is recorded per table + if (!AWSTestConstants.IsDynamoDbAvailable) + { + return; + } + + try + { + await new DynamoDBStorage(NullLogger.Instance, AWSTestConstants.DynamoDbService).DeleTableAsync(_tableName); + } + catch (ResourceNotFoundException) + { + } + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_EmptyServiceId_KeepsKeysWithoutServiceId() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "a"); + + Assert.True(await RowExists($"_{grainId}")); + Assert.False(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_UseClusterServiceId_BuildsKeysFromClusterServiceId() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(o => o.UseClusterServiceId = true), grainId, "a"); + + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + Assert.False(await RowExists($"_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_ExplicitServiceId_IgnoresClusterServiceId() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(o => { o.ServiceId = "explicit"; o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }), grainId, "a"); + + Assert.True(await RowExists($"explicit_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_UseClusterServiceIdWithoutMigration_DoesNotReadLegacyState() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var state = await ReadAsync(await CreateStorage(o => o.UseClusterServiceId = true), grainId); + + Assert.False(state.RecordExists); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_ReadsLegacyStateAndMovesItOnWrite() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + var state = await ReadAsync(storage, grainId); + Assert.Equal("legacy", state.State!.A); + Assert.True(await RowExists($"_{grainId}")); + + state.State!.A = "migrated"; + await storage.WriteStateAsync(GrainType, grainId, state); + + Assert.False(await RowExists($"_{grainId}")); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + Assert.Equal("migrated", (await ReadAsync(storage, grainId)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_MovesStateReadByAnotherInstance() + { + // nothing is remembered between the read and the write, so a retry or another silo moves the state as well + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var state = await ReadAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }), grainId); + state.State!.A = "migrated"; + await (await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; })).WriteStateAsync(GrainType, grainId, state); + + Assert.False(await RowExists($"_{grainId}")); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_StaleLegacyStateIsInconsistent() + { + var grainId = NewGrainId(); + var legacy = await CreateStorage(); + await WriteAsync(legacy, grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + var state = await ReadAsync(storage, grainId); + await WriteAsync(legacy, grainId, "changed meanwhile"); + + await Assert.ThrowsAsync(() => storage.WriteStateAsync(GrainType, grainId, state)); + Assert.True(await RowExists($"_{grainId}")); + Assert.False(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithoutTimeToLive_KeepsTheLegacyExpiry() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(o => o.TimeToLive = TimeSpan.FromDays(3)), grainId, "legacy"); + var legacyTtl = (await ReadRow($"_{grainId}"))!["GrainTtl"].N; + + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }), grainId, "migrated"); + + Assert.Equal(legacyTtl, (await ReadRow($"{ClusterServiceId}_{grainId}"))!["GrainTtl"].N); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithDeleteStateOnClear_CurrentStateWrittenMeanwhileIsInconsistent() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; o.DeleteStateOnClear = true; }); + var state = await ReadAsync(storage, grainId); + + // another silo migrates the grain and writes it before this clear + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }), grainId, "current"); + + await Assert.ThrowsAsync(() => storage.ClearStateAsync(GrainType, grainId, state)); + Assert.Equal("current", (await ReadAsync(storage, grainId)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithDeleteStateOnClear_CurrentStateBesideLegacyStateIsInconsistent() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; o.DeleteStateOnClear = true; }); + var state = await ReadAsync(storage, grainId); + + // another silo writes the current key without migrating, which leaves the legacy item in place + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; }), grainId, "current"); + + await Assert.ThrowsAsync(() => storage.ClearStateAsync(GrainType, grainId, state)); + Assert.True(await RowExists($"_{grainId}")); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithDeleteStateOnClear_ClearingCurrentStateRetiresTheLegacyItem() + { + // written on the current key by another silo that does not migrate, so the legacy item is still there + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; o.DeleteStateOnClear = true; }); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; }), grainId, "current"); + + await storage.ClearStateAsync(GrainType, grainId, await ReadAsync(storage, grainId)); + + Assert.False((await ReadAsync(storage, grainId)).RecordExists); + Assert.False(await RowExists($"_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_ClearingCurrentStateKeepingTheItemRetiresTheLegacyItem() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; }), grainId, "current"); + + await storage.ClearStateAsync(GrainType, grainId, await ReadAsync(storage, grainId)); + + Assert.False(await RowExists($"_{grainId}")); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + Assert.False((await ReadAsync(storage, grainId)).RecordExists); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_WritingCurrentStateRetiresTheLegacyItem() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; }), grainId, "current"); + + await WriteAsync(storage, grainId, "current again"); + + Assert.False(await RowExists($"_{grainId}")); + Assert.Equal("current again", (await ReadAsync(storage, grainId)).State!.A); + Assert.Equal("1", (await ReadRow($"{ClusterServiceId}_{grainId}"))!["ETag"].N); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_StateReadFromTheCurrentKeyIsNotMigrated() + { + // an orphaned legacy item with the same ETag must not stand in for a current item deleted meanwhile + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "orphan"); + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; }), grainId, "current"); + + var state = await ReadAsync(storage, grainId); + Assert.Equal("current", state.State!.A); + var other = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = false; o.DeleteStateOnClear = true; }); + await other.ClearStateAsync(GrainType, grainId, await ReadAsync(other, grainId)); + + await Assert.ThrowsAsync(() => storage.WriteStateAsync(GrainType, grainId, state)); + Assert.True(await RowExists($"_{grainId}")); + Assert.False(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_ClearMovesClearedState() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + var state = await ReadAsync(storage, grainId); + await storage.ClearStateAsync(GrainType, grainId, state); + + Assert.False(await RowExists($"_{grainId}")); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + Assert.False((await ReadAsync(storage, grainId)).RecordExists); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithDeleteStateOnClear_DeletesLegacyState() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; o.DeleteStateOnClear = true; }); + var state = await ReadAsync(storage, grainId); + await storage.ClearStateAsync(GrainType, grainId, state); + + Assert.False(await RowExists($"_{grainId}")); + Assert.False(await RowExists($"{ClusterServiceId}_{grainId}")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_RecordedMigration_GoesOnWithoutOptions() + { + var migrated = NewGrainId(); + var pending = NewGrainId(); + var legacy = await CreateStorage(); + await WriteAsync(legacy, migrated, "a"); + await WriteAsync(legacy, pending, "b"); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }), migrated, "a2"); + + var storage = await CreateStorage(); + + Assert.Equal("a2", (await ReadAsync(storage, migrated)).State!.A); + Assert.Equal("b", (await ReadAsync(storage, pending)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_RecordedClusterServiceId_IsKeptWithoutOptions() + { + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(o => o.UseClusterServiceId = true), grainId, "a"); + + Assert.Equal("a", (await ReadAsync(await CreateStorage(), grainId)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_KeyFormatRecordedByAnotherSiloMeanwhile_IsFollowed() + { + // this silo reads no record and would keep the empty ServiceId, but another one records a migration first + var interleaved = false; + var storage = await CreateStorage(beforeInit: s => s.BeforeKeyFormatWriteForTesting = async () => + { + if (!interleaved) + { + interleaved = true; + await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + } + }); + + var grainId = NewGrainId(); + await WriteAsync(storage, grainId, "a"); + + Assert.True(interleaved); + Assert.True(await RowExists($"{ClusterServiceId}_{grainId}")); + Assert.Equal("MigratingToClusterServiceId", (await ReadRow("__OrleansKeyFormat", "__OrleansKeyFormat"))!["KeyFormat"].S); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithDeleteStateOnClear_ClearingUnreadStateLeavesTheLegacyItem() + { + // a clear of a state that was never read has no ETag, and leaves whatever is there, as before + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; o.DeleteStateOnClear = true; }); + await storage.ClearStateAsync(GrainType, grainId, new GrainState(new TestStoreGrainState())); + + Assert.True(await RowExists($"_{grainId}")); + Assert.Equal("legacy", (await ReadAsync(storage, grainId)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_UnknownKeyFormatRecord_FailsToStart() + { + await CreateStorage(); + await WriteKeyFormatRecord("SomethingNewer"); + + var exception = await Assert.ThrowsAsync(() => CreateStorage()); + + Assert.Contains("SomethingNewer", exception.Message); + } + + [Fact(Timeout = 15_000), TestCategory("Functional")] + public async Task DynamoDBGrainStorage_KeyFormatRecordWithoutValue_FailsToStart() + { + await CreateStorage(); + await WriteKeyFormatRecord(null); + + // a record that is there but says nothing must not be taken for a missing one, whose write would then be retried for ever + await Assert.ThrowsAsync(() => CreateStorageUntil(TestContext.Current.CancellationToken)); + } + + [Theory, TestCategory("Functional")] + [InlineData(false)] + [InlineData(true)] + public async Task DynamoDBGrainStorage_UseClusterServiceIdFalseOverRecordedClusterServiceId_FailsToStart(bool migrating) + { + // going back to the empty ServiceId would hide this state, for this silo and every one that follows the record + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = migrating; }), grainId, "current"); + + var exception = await Assert.ThrowsAsync(() => CreateStorage(o => o.UseClusterServiceId = false)); + + Assert.Contains("UseClusterServiceId is false", exception.Message); + Assert.Equal(migrating ? "MigratingToClusterServiceId" : "ClusterServiceId", (await ReadRow("__OrleansKeyFormat", "__OrleansKeyFormat"))!["KeyFormat"].S); + Assert.Equal("current", (await ReadAsync(await CreateStorage(), grainId)).State!.A); + } + + [Theory, TestCategory("Functional")] + [InlineData(true)] + [InlineData(null)] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysOverRecordedClusterServiceId_FailsToStart(bool? useClusterServiceId) + { + // the grain's state under ClusterOptions.ServiceId was deleted while the migration was off, and a migration would + // read back the legacy item it left behind + var grainId = NewGrainId(); + await WriteAsync(await CreateStorage(), grainId, "legacy"); + var withoutMigration = await CreateStorage(o => { o.UseClusterServiceId = true; o.DeleteStateOnClear = true; }); + await WriteAsync(withoutMigration, grainId, "current"); + await withoutMigration.ClearStateAsync(GrainType, grainId, await ReadAsync(withoutMigration, grainId)); + + var exception = await Assert.ThrowsAsync( + () => CreateStorage(o => { o.UseClusterServiceId = useClusterServiceId; o.MigrateLegacyKeys = true; })); + + Assert.Contains("MigrateLegacyKeys is true", exception.Message); + Assert.Equal("ClusterServiceId", (await ReadRow("__OrleansKeyFormat", "__OrleansKeyFormat"))!["KeyFormat"].S); + Assert.False((await ReadAsync(await CreateStorage(), grainId)).RecordExists); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysOnceTheRecordIsDeleted_Starts() + { + // what the refusal says to do, to migrate anyway + await CreateStorage(o => o.UseClusterServiceId = true); + await DeleteKeyFormatRecord(); + + await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + + Assert.Equal("MigratingToClusterServiceId", (await ReadRow("__OrleansKeyFormat", "__OrleansKeyFormat"))!["KeyFormat"].S); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeys_WriteWithAnETagThatIsNotANumberIsInconsistent() + { + var grainId = NewGrainId(); + var storage = await CreateStorage(o => { o.UseClusterServiceId = true; o.MigrateLegacyKeys = true; }); + await WriteAsync(storage, grainId, "a"); + + // written once, the current item is at ETag 0, which an ETag that is not a number must not stand for + var state = await ReadAsync(storage, grainId); + state.ETag = "not a number"; + state.State!.A = "overwritten"; + + await Assert.ThrowsAsync(() => storage.WriteStateAsync(GrainType, grainId, state)); + Assert.Equal("a", (await ReadAsync(storage, grainId)).State!.A); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_KeyFormatOptionsWithExplicitServiceId_LogsThatTheyHaveNoEffect() + { + var logger = new CapturingLogger(); + await CreateStorage(o => { o.ServiceId = "explicit"; o.MigrateLegacyKeys = true; }, logger: logger); + + Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning && entry.Message.Contains("UseClusterServiceId and MigrateLegacyKeys have no effect")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_MigrateLegacyKeysWithEmptyServiceId_LogsThatItHasNoEffect() + { + var logger = new CapturingLogger(); + await CreateStorage(o => { o.UseClusterServiceId = false; o.MigrateLegacyKeys = true; }, logger: logger); + + Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning && entry.Message.Contains("MigrateLegacyKeys has no effect")); + } + + [Fact, TestCategory("Functional")] + public async Task DynamoDBGrainStorage_Init_LogsTheKeyFormat() + { + var logger = new CapturingLogger(); + await CreateStorage(o => o.UseClusterServiceId = true, logger: logger); + + Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Information + && entry.Message.Contains($"from the ServiceId '{ClusterServiceId}' (key format ClusterServiceId)")); + } + + private async Task DeleteKeyFormatRecord() + { + var storage = new DynamoDBStorage(NullLogger.Instance, AWSTestConstants.DynamoDbService); + await storage.DeleteEntryAsync(_tableName, new Dictionary + { + { "GrainReference", new AttributeValue("__OrleansKeyFormat") }, + { "GrainType", new AttributeValue("__OrleansKeyFormat") } + }); + } + + private async Task WriteKeyFormatRecord(string? format) + { + var storage = new DynamoDBStorage(NullLogger.Instance, AWSTestConstants.DynamoDbService); + var fields = new Dictionary + { + { "GrainReference", new AttributeValue("__OrleansKeyFormat") }, + { "GrainType", new AttributeValue("__OrleansKeyFormat") } + }; + if (format is not null) + { + fields.Add("KeyFormat", new AttributeValue(format)); + } + + await storage.PutEntryAsync(_tableName, fields); + } + + private static GrainId NewGrainId() => GrainId.Create("keyformat", Guid.NewGuid().ToString("N")); + + private Task CreateStorage(Action? configure = null, Action? beforeInit = null, ILogger? logger = null) => + CreateStorageUntil(CancellationToken.None, configure, beforeInit, logger); + + private async Task CreateStorageUntil(CancellationToken cancellationToken, Action? configure = null, Action? beforeInit = null, ILogger? logger = null) + { + if (!AWSTestConstants.IsDynamoDbAvailable) + { + throw Xunit.Sdk.SkipException.ForSkip("Unable to connect to AWS DynamoDB simulator"); + } + + var options = new DynamoDBStorageOptions + { + Service = AWSTestConstants.DynamoDbService, + TableName = _tableName, + ClusterServiceId = ClusterServiceId, + GrainStorageSerializer = new OrleansGrainStorageSerializer(_fixture.Services.GetRequiredService()), + }; + configure?.Invoke(options); + + var storage = ActivatorUtilities.CreateInstance(_fixture.Services, "KeyFormatTests", options, logger ?? NullLogger.Instance); + beforeInit?.Invoke(storage); + await storage.Init(cancellationToken); + return storage; + } + + private static async Task WriteAsync(DynamoDBGrainStorage storage, GrainId grainId, string value) + { + var state = await ReadAsync(storage, grainId); + state.State!.A = value; + await storage.WriteStateAsync(GrainType, grainId, state); + } + + private static async Task> ReadAsync(DynamoDBGrainStorage storage, GrainId grainId) + { + var state = new GrainState(new TestStoreGrainState()); + await storage.ReadStateAsync(GrainType, grainId, state); + return state; + } + + private async Task RowExists(string partitionKey) => await ReadRow(partitionKey) is not null; + + private async Task?> ReadRow(string partitionKey, string rowKey = GrainType) + { + var storage = new DynamoDBStorage(NullLogger.Instance, AWSTestConstants.DynamoDbService); + var row = await storage.ReadSingleEntryAsync(_tableName, + new Dictionary + { + { "GrainReference", new AttributeValue(partitionKey) }, + { "GrainType", new AttributeValue(rowKey) } + }, + fields => fields); + return row; + } + + private sealed class CapturingLogger : ILogger + { + public List<(LogLevel Level, string Message)> Entries { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => Entries.Add((logLevel, formatter(state, exception))); + } +} diff --git a/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageProviderBuilderTests.cs b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageProviderBuilderTests.cs index 04c2d5284ab..2b5dbe68485 100644 --- a/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageProviderBuilderTests.cs +++ b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBGrainStorageProviderBuilderTests.cs @@ -42,6 +42,8 @@ public void Configure_MinimalConfiguration_RegistersCredentialFreeDefaults() Assert.True(options.UpdateIfExists); Assert.False(options.DeleteStateOnClear); Assert.Null(options.TimeToLive); + Assert.Null(options.UseClusterServiceId); + Assert.Null(options.MigrateLegacyKeys); Assert.Same(defaultSerializer, options.GrainStorageSerializer); } @@ -66,6 +68,8 @@ public void Configure_FullConfiguration_BindsEverySupportedSetting() (nameof(DynamoDBStorageOptions.UpdateIfExists), "false"), (nameof(DynamoDBStorageOptions.DeleteStateOnClear), "true"), (nameof(DynamoDBStorageOptions.TimeToLive), "01:02:03"), + (nameof(DynamoDBStorageOptions.UseClusterServiceId), "true"), + (nameof(DynamoDBStorageOptions.MigrateLegacyKeys), "false"), ("SerializerKey", serializerKey)); builder.Services.AddKeyedSingleton(serializerKey, keyedSerializer); @@ -87,10 +91,37 @@ public void Configure_FullConfiguration_BindsEverySupportedSetting() Assert.False(options.UpdateIfExists); Assert.True(options.DeleteStateOnClear); Assert.Equal(TimeSpan.FromHours(1) + TimeSpan.FromMinutes(2) + TimeSpan.FromSeconds(3), options.TimeToLive); + Assert.True(options.UseClusterServiceId); + Assert.False(options.MigrateLegacyKeys); Assert.Same(keyedSerializer, options.GrainStorageSerializer); Assert.NotSame(defaultSerializer, options.GrainStorageSerializer); } + [Fact] + public void Configure_ClusterServiceId_ComesFromClusterOptionsAndItsProviderOverride() + { + var (withoutClusterOptions, _) = ConfigureBuilder(); + using (var services = withoutClusterOptions.Services.BuildServiceProvider()) + { + Assert.Equal(ClusterOptions.DefaultServiceId, GetOptions(services).ClusterServiceId); + } + + var (withClusterOptions, _) = ConfigureBuilder(); + withClusterOptions.Services.Configure(o => o.ServiceId = "cluster-service"); + using (var services = withClusterOptions.Services.BuildServiceProvider()) + { + Assert.Equal("cluster-service", GetOptions(services).ClusterServiceId); + } + + var (withOverride, _) = ConfigureBuilder(); + withOverride.Services.Configure(o => o.ServiceId = "cluster-service"); + withOverride.Services.AddKeyedSingleton(ProviderName, new ClusterOptions { ServiceId = "provider-service" }); + using (var services = withOverride.Services.BuildServiceProvider()) + { + Assert.Equal("provider-service", GetOptions(services).ClusterServiceId); + } + } + [Fact] public void Configure_RegionWithoutService_UsesRegion() { @@ -111,6 +142,8 @@ public void Configure_RegionWithoutService_UsesRegion() [InlineData(nameof(DynamoDBStorageOptions.UpdateIfExists), "not-a-boolean")] [InlineData(nameof(DynamoDBStorageOptions.DeleteStateOnClear), "not-a-boolean")] [InlineData(nameof(DynamoDBStorageOptions.TimeToLive), "not-a-timespan")] + [InlineData(nameof(DynamoDBStorageOptions.UseClusterServiceId), "not-a-boolean")] + [InlineData(nameof(DynamoDBStorageOptions.MigrateLegacyKeys), "not-a-boolean")] public void Configure_InvalidTypedValue_PreservesDefault(string key, string invalidValue) { var (builder, _) = ConfigureBuilder((key, invalidValue)); @@ -128,6 +161,8 @@ public void Configure_InvalidTypedValue_PreservesDefault(string key, string inva Assert.True(options.UpdateIfExists); Assert.False(options.DeleteStateOnClear); Assert.Null(options.TimeToLive); + Assert.Null(options.UseClusterServiceId); + Assert.Null(options.MigrateLegacyKeys); } [Fact]