[Internal] Msdata/Direct: Refactors msdata/direct branch with latest v3 master and Cosmos.Direct v3.30.8

.github/fabricbot.json

@@ -102,7 +102,82 @@
           }
         ]
       }
+    },
+    {
+      "taskType": "trigger",
+      "capabilityId": "AutoMerge",
+      "subCapability": "AutoMerge",
+      "version": "1.0",
+      "config": {
+        "taskName": "PR Automerge",
+        "allowAutoMergeInstructionsWithoutLabel": false,
+        "mergeType": "squash",
+        "deleteBranches": true,
+        "removeLabelOnPush": true,
+        "label": "auto-merge",
+        "requireAllStatuses": false,
+        "requireSpecificCheckRuns": false,
+        "usePrDescriptionAsCommitMessage": false,
+        "minMinutesOpen": "60",
+        "enforceDMPAsStatus": true
+      }
+    },
+    {
+      "taskType": "scheduled",
+      "capabilityId": "ScheduledSearch",
+      "subCapability": "ScheduledSearch",
+      "version": "1.1",
+      "config": {
+        "frequency": [
+          {
+            "weekDay": 1,
+            "hours": [
+              9
+            ],
+            "timezoneOffset": -7
+          }
+        ],
+        "searchTerms": [
+          {
+            "name": "isOpen",
+            "parameters": {}
+          },
+          {
+            "name": "isIssue",
+            "parameters": {}
+          },
+          {
+            "name": "hasLabel",
+            "parameters": {
+              "label": "needs-more-information"
+            }
+          },
+          {
+            "name": "noActivitySince",
+            "parameters": {
+              "days": 14
+            }
+          },
+          {
+            "name": "noAssignees",
+            "parameters": {}
+          }
+        ],
+        "taskName": "Close inactive needs-information",
+        "actions": [
+          {
+            "name": "addReply",
+            "parameters": {
+              "comment": "@${issueAuthor} this issue requires more information for the team to be able to help. In case this information is available, please add it and re-open the Issue."
+            }
+          },
+          {
+            "name": "closeIssue",
+            "parameters": {}
+          }
+        ]
+      }
     }
   ],
   "userGroups": []
-}
+}

Directory.Build.props

@@ -1,13 +1,13 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 	<PropertyGroup>
-		<ClientOfficialVersion>3.32.2</ClientOfficialVersion>
-		<ClientPreviewVersion>3.32.2</ClientPreviewVersion>
+		<ClientOfficialVersion>3.33.0</ClientOfficialVersion>
+		<ClientPreviewVersion>3.33.0</ClientPreviewVersion>
 		<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
-		<DirectVersion>3.30.4</DirectVersion>
+		<DirectVersion>3.30.8</DirectVersion>
 		<EncryptionOfficialVersion>2.0.1</EncryptionOfficialVersion>
 		<EncryptionPreviewVersion>2.0.1</EncryptionPreviewVersion>
 		<EncryptionPreviewSuffixVersion>preview</EncryptionPreviewSuffixVersion>
-		<CustomEncryptionVersion>1.0.0-preview04</CustomEncryptionVersion>
+		<CustomEncryptionVersion>1.0.0-preview05</CustomEncryptionVersion>
 		<HybridRowVersion>1.1.0-preview3</HybridRowVersion>
 		<LangVersion>10.0</LangVersion>
 		<AboveDirBuildProps>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))</AboveDirBuildProps>

Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md

@@ -3,6 +3,11 @@ Preview features are treated as a separate branch and will not be included in th
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### <a name="1.0.0-preview05"/> [1.0.0-preview05](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview05) - 2023-04-27
+
+#### Fixes 
+- [#3809](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3809) Adds api FetchDataEncryptionKeyWithoutRawKeyAsync and FetchDataEncryptionKey to get DEK without and with raw key respectively.
+
 ### <a name="1.0.0-preview04"/> [1.0.0-preview04](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview04) - 2022-08-16
 
 #### Fixes 

Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs

@@ -166,11 +166,25 @@ public async Task InitializeAsync(
             this.container = containerResponse.Container;
         }
 
+        /// <inheritdoc/>
+        public override async Task<DataEncryptionKey> FetchDataEncryptionKeyWithoutRawKeyAsync(
+            string id,
+            string encryptionAlgorithm,
+            CancellationToken cancellationToken)
+        {
+            return await this.FetchDekAsync(id, encryptionAlgorithm, cancellationToken);
+        }
+
         /// <inheritdoc/>
         public override async Task<DataEncryptionKey> FetchDataEncryptionKeyAsync(
             string id,
             string encryptionAlgorithm,
             CancellationToken cancellationToken)
+        {
+            return await this.FetchDekAsync(id, encryptionAlgorithm, cancellationToken, true);
+        }
+
+        private async Task<DataEncryptionKey> FetchDekAsync(string id, string encryptionAlgorithm, CancellationToken cancellationToken, bool withRawKey = false)
         {
             DataEncryptionKeyProperties dataEncryptionKeyProperties = await this.dataEncryptionKeyContainerCore.FetchDataEncryptionKeyPropertiesAsync(
                 id,
@@ -200,7 +214,8 @@ public override async Task<DataEncryptionKey> FetchDataEncryptionKeyAsync(
             InMemoryRawDek inMemoryRawDek = await this.dataEncryptionKeyContainerCore.FetchUnwrappedAsync(
                 dataEncryptionKeyProperties,
                 diagnosticsContext: CosmosDiagnosticsContext.Create(null),
-                cancellationToken: cancellationToken);
+                cancellationToken: cancellationToken,
+                withRawKey);
 
             return inMemoryRawDek.DataEncryptionKey;
         }

Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs

@@ -35,14 +35,14 @@ public override async Task<byte[]> DecryptAsync(
             string encryptionAlgorithm,
             CancellationToken cancellationToken = default)
         {
-            DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyAsync(
+            DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync(
                 dataEncryptionKeyId,
                 encryptionAlgorithm,
                 cancellationToken);
 
             if (dek == null)
             {
-                throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyAsync)}.");
+                throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}.");
             }
 
             return dek.DecryptData(cipherText);
@@ -55,14 +55,14 @@ public override async Task<byte[]> EncryptAsync(
             string encryptionAlgorithm,
             CancellationToken cancellationToken = default)
         {
-            DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyAsync(
+            DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync(
                 dataEncryptionKeyId,
                 encryptionAlgorithm,
                 cancellationToken);
 
             if (dek == null)
             {
-                throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyAsync)}.");
+                throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}.");
             }
 
             return dek.EncryptData(plainText);

Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs

@@ -335,6 +335,7 @@ internal async Task<DataEncryptionKey> FetchUnWrappedMdeSupportedLegacyDekAsync(
                 unwrapResult.DataEncryptionKey);
 
             return new MdeEncryptionAlgorithm(
+                unwrapResult.DataEncryptionKey,
                 plaintextDataEncryptionKey,
                 Data.Encryption.Cryptography.EncryptionType.Randomized);
         }
@@ -378,13 +379,14 @@ internal async Task<DataEncryptionKey> FetchUnWrappedLegacySupportedMdeDekAsync(
         internal async Task<InMemoryRawDek> FetchUnwrappedAsync(
             DataEncryptionKeyProperties dekProperties,
             CosmosDiagnosticsContext diagnosticsContext,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken,
+            bool withRawKey = false)
         {
             try
             {
                 if (string.Equals(dekProperties.EncryptionAlgorithm, CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized))
                 {
-                    DataEncryptionKey dek = this.InitMdeEncryptionAlgorithm(dekProperties);
+                    DataEncryptionKey dek = this.InitMdeEncryptionAlgorithm(dekProperties, withRawKey);
 
                     // TTL is not used since DEK is not cached.
                     return new InMemoryRawDek(dek, TimeSpan.FromMilliseconds(0));
@@ -564,7 +566,7 @@ private async Task<EncryptionKeyUnwrapResult> UnWrapDekMdeEncAlgoAsync(
             return unwrapResult;
         }
 
-        internal DataEncryptionKey InitMdeEncryptionAlgorithm(DataEncryptionKeyProperties dekProperties)
+        internal DataEncryptionKey InitMdeEncryptionAlgorithm(DataEncryptionKeyProperties dekProperties, bool withRawKey = false)
         {
             if (this.DekProvider.MdeKeyWrapProvider == null)
             {
@@ -576,7 +578,8 @@ internal DataEncryptionKey InitMdeEncryptionAlgorithm(DataEncryptionKeyPropertie
                 dekProperties,
                 Data.Encryption.Cryptography.EncryptionType.Randomized,
                 this.DekProvider.MdeKeyWrapProvider.EncryptionKeyStoreProvider,
-                this.DekProvider.PdekCacheTimeToLive);
+                this.DekProvider.PdekCacheTimeToLive,
+                withRawKey);
         }
 
         private async Task<DataEncryptionKeyProperties> ReadResourceAsync(

Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyProvider.cs

@@ -14,7 +14,19 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
     public abstract class DataEncryptionKeyProvider
     {
         /// <summary>
-        /// Retrieves the data encryption key for the given id.
+        /// Retrieves the data encryption key for the given id without rawkey. RawKey will be set to null.
+        /// </summary>
+        /// <param name="id">Identifier of the data encryption key.</param>
+        /// <param name="encryptionAlgorithm">Encryption algorithm that the retrieved key will be used with.</param>
+        /// <param name="cancellationToken">Token for request cancellation.</param>
+        /// <returns>Data encryption key bytes.</returns>
+        public abstract Task<DataEncryptionKey> FetchDataEncryptionKeyWithoutRawKeyAsync(
+            string id,
+            string encryptionAlgorithm,
+            CancellationToken cancellationToken);
+
+        /// <summary>
+        /// Retrieves the data encryption key for the given id with RawKey value.
         /// </summary>
         /// <param name="id">Identifier of the data encryption key.</param>
         /// <param name="encryptionAlgorithm">Encryption algorithm that the retrieved key will be used with.</param>

Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs

@@ -791,13 +791,6 @@ public override Task<IReadOnlyList<FeedRange>> GetFeedRangesAsync(
             return this.container.GetFeedRangesAsync(cancellationToken);
         }
 
-        public override Task<IEnumerable<string>> GetPartitionKeyRangesAsync(
-            FeedRange feedRange,
-            CancellationToken cancellationToken = default)
-        {
-            return this.container.GetPartitionKeyRangesAsync(feedRange, cancellationToken);
-        }
-
         public override FeedIterator GetItemQueryStreamIterator(
             FeedRange feedRange,
             QueryDefinition queryDefinition,
@@ -1010,6 +1003,14 @@ public override async Task<FeedResponse<T>> ReadManyItemsAsync<T>(
             return this.ResponseFactory.CreateItemFeedResponse<T>(responseMessage);
         }
 
+#if ENCRYPTIONPREVIEW
+        public override Task<IEnumerable<string>> GetPartitionKeyRangesAsync(
+            FeedRange feedRange,
+            CancellationToken cancellationToken = default)
+        {
+            return this.container.GetPartitionKeyRangesAsync(feedRange, cancellationToken);
+        }
+
         public override Task<ResponseMessage> DeleteAllItemsByPartitionKeyStreamAsync(
                Cosmos.PartitionKey partitionKey,
                RequestOptions requestOptions = null,
@@ -1020,6 +1021,7 @@ public override Task<ResponseMessage> DeleteAllItemsByPartitionKeyStreamAsync(
                 requestOptions,
                 cancellationToken);
         }
+#endif
 
         private async Task<ResponseMessage> ReadManyItemsHelperAsync(
            IReadOnlyList<(string id, PartitionKey partitionKey)> items,

Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs

@@ -14,8 +14,10 @@ internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey
     {
         private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm;
 
+        private readonly byte[] unwrapKey;
+
         // unused for MDE Algorithm.
-        public override byte[] RawKey => null;
+        public override byte[] RawKey { get; }
 
         public override string EncryptionAlgorithm => CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized;
 
@@ -32,7 +34,8 @@ public MdeEncryptionAlgorithm(
             DataEncryptionKeyProperties dekProperties,
             Data.Encryption.Cryptography.EncryptionType encryptionType,
             EncryptionKeyStoreProvider encryptionKeyStoreProvider,
-            TimeSpan? cacheTimeToLive)
+            TimeSpan? cacheTimeToLive,
+            bool withRawKey=false)
         {
             if (dekProperties == null)
             {
@@ -49,36 +52,39 @@ public MdeEncryptionAlgorithm(
                 dekProperties.EncryptionKeyWrapMetadata.Value,
                 encryptionKeyStoreProvider);
 
-            ProtectedDataEncryptionKey protectedDataEncryptionKey;
-            if (cacheTimeToLive.HasValue)
+            if (!withRawKey)
             {
-                // no caching
-                if (cacheTimeToLive.Value == TimeSpan.Zero)
-                {
-                    protectedDataEncryptionKey = new ProtectedDataEncryptionKey(
+                ProtectedDataEncryptionKey protectedDataEncryptionKey = cacheTimeToLive.HasValue && cacheTimeToLive.Value == TimeSpan.Zero
+                    ? new ProtectedDataEncryptionKey(
+                        dekProperties.Id,
+                        keyEncryptionKey,
+                        dekProperties.WrappedDataEncryptionKey)
+                    : ProtectedDataEncryptionKey.GetOrCreate(
                         dekProperties.Id,
                         keyEncryptionKey,
                         dekProperties.WrappedDataEncryptionKey);
-                }
-                else
-                {
-                    protectedDataEncryptionKey = ProtectedDataEncryptionKey.GetOrCreate(
-                       dekProperties.Id,
-                       keyEncryptionKey,
-                       dekProperties.WrappedDataEncryptionKey);
-                }
+                this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate(
+                    protectedDataEncryptionKey,
+                    encryptionType);
             }
             else
             {
-                protectedDataEncryptionKey = ProtectedDataEncryptionKey.GetOrCreate(
-                       dekProperties.Id,
-                       keyEncryptionKey,
-                       dekProperties.WrappedDataEncryptionKey);
+                byte[] rawKey = keyEncryptionKey.DecryptEncryptionKey(dekProperties.WrappedDataEncryptionKey);
+                PlaintextDataEncryptionKey plaintextDataEncryptionKey = cacheTimeToLive.HasValue && (cacheTimeToLive.Value == TimeSpan.Zero)
+                    ? new PlaintextDataEncryptionKey(
+                            dekProperties.Id,
+                            rawKey)
+                    : PlaintextDataEncryptionKey.GetOrCreate(
+                           dekProperties.Id,
+                           rawKey);
+                this.RawKey = rawKey;
+                this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate(
+                    plaintextDataEncryptionKey,
+                    encryptionType);
+
             }
 
-            this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate(
-                protectedDataEncryptionKey,
-                encryptionType);
+
         }
 
         /// <summary>
@@ -90,9 +96,11 @@ public MdeEncryptionAlgorithm(
         /// <param name="dataEncryptionKey"> Data Encryption Key </param>
         /// <param name="encryptionType"> Encryption type </param>
         public MdeEncryptionAlgorithm(
+            byte[] rawkey,
             Data.Encryption.Cryptography.DataEncryptionKey dataEncryptionKey,
             Data.Encryption.Cryptography.EncryptionType encryptionType)
         {
+            this.RawKey = rawkey;
             this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate(
                 dataEncryptionKey,
                 encryptionType);