Add support for preserve references on JSON

src/libraries/System.Text.Json/ref/System.Text.Json.cs

@@ -463,6 +463,7 @@ public JsonSerializerOptions() { }
         public bool PropertyNameCaseInsensitive { get { throw null; } set { } }
         public System.Text.Json.JsonNamingPolicy? PropertyNamingPolicy { get { throw null; } set { } }
         public System.Text.Json.JsonCommentHandling ReadCommentHandling { get { throw null; } set { } }
+        public System.Text.Json.Serialization.ReferenceHandling ReferenceHandling { get { throw null; } set { } }
         public bool WriteIndented { get { throw null; } set { } }
         public System.Text.Json.Serialization.JsonConverter? GetConverter(System.Type typeToConvert) { throw null; }
     }
@@ -775,4 +776,10 @@ public JsonStringEnumConverter(System.Text.Json.JsonNamingPolicy? namingPolicy =
         public override bool CanConvert(System.Type typeToConvert) { throw null; }
         public override System.Text.Json.Serialization.JsonConverter CreateConverter(System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; }
     }
+    public sealed partial class ReferenceHandling
+    {
+        internal ReferenceHandling() { }
+        public static System.Text.Json.Serialization.ReferenceHandling Default { get { throw null; } }
+        public static System.Text.Json.Serialization.ReferenceHandling Preserve { get { throw null; } }
+    }
 }

src/libraries/System.Text.Json/src/Resources/Strings.resx

@@ -447,4 +447,45 @@
   <data name="NotNodeJsonElementParent" xml:space="preserve">
     <value>This JsonElement instance was not built from a JsonNode and is immutable.</value>
   </data>
+  <data name="MetadataCannotParsePreservedObjectToImmutable" xml:space="preserve">
+    <value>Cannot parse preserved object to Array or Immutable. Type '{0}'.</value>
+  </data>
+  <data name="MetadataDuplicateIdFound" xml:space="preserve">
+    <value>Duplicated identifier '{0}' found while reading preserved object.</value>
+  </data>
+  <data name="MetadataIdIsNotFirstProperty" xml:space="preserve">
+    <value>The metadata property $id must be the first property in the JSON object.</value>
+  </data>
+  <data name="MetadataInvalidReferenceToValueType" xml:space="preserve">
+    <value>Invalid reference to value type '{0}'.</value>
+  </data>
+  <data name="MetadataInvalidTokenAfterValues" xml:space="preserve">
+    <value>Invalid token after $values metadata property.</value>
+  </data>
+  <data name="MetadataMissingIdBeforeValues" xml:space="preserve">
+    <value>Missing $id before $values on preserved array.</value>
+  </data>
+  <data name="MetadataPreservedArrayFailed" xml:space="preserve">
+    <value>Deserialization failed for one of these reasons:
+1. {0}
+2. {1}</value>
+  </data>
+  <data name="MetadataPreservedArrayInvalidProperty" xml:space="preserve">
+    <value>Invalid property in preserved array.</value>
+  </data>
+  <data name="MetadataPreservedArrayValuesNotFound" xml:space="preserve">
+    <value>Metadata $values property was not found in preserved array.</value>
+  </data>
+  <data name="MetadataReferenceCannotContainOtherProperties" xml:space="preserve">
+    <value>Reference objects must not contain other properties except for $ref.</value>
+  </data>
+  <data name="MetadataReferenceNotFound" xml:space="preserve">
+    <value>Reference '{0}' not found.</value>
+  </data>
+  <data name="MetadataValueWasNotString" xml:space="preserve">
+    <value>Value for metadata properties $id and $ref must be of type string.</value>
+  </data>
+  <data name="MetadataInvalidPropertyWithLeadingSign" xml:space="preserve">
+    <value>Properties that start with '$' are not allowed on preserve mode, either escape the character or turn off preserve references.</value>
+  </data>
 </root>

src/libraries/System.Text.Json/src/System.Text.Json.csproj

@@ -22,6 +22,12 @@
     <Compile Include="System\Text\Json\JsonHelpers.cs" />
     <Compile Include="System\Text\Json\JsonHelpers.Date.cs" />
     <Compile Include="System\Text\Json\JsonTokenType.cs" />
+    <Compile Include="System\Text\Json\Serialization\DefaultReferenceResolver.cs" />
+    <Compile Include="System\Text\Json\Serialization\JsonPreservedReference.cs" />
+    <Compile Include="System\Text\Json\Serialization\MetadataPropertyName.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceEqualsEqualityComparer.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceHandling.cs" />
+    <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleMetadata.cs" />
     <Compile Include="System\Text\Json\ThrowHelper.cs" />
     <Compile Include="System\Text\Json\ThrowHelper.Serialization.cs" />
     <Compile Include="System\Text\Json\Document\JsonDocument.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/DefaultReferenceResolver.cs

@@ -0,0 +1,61 @@
+// Licensed to the.NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace System.Text.Json
+{
+    internal struct DefaultReferenceResolver
+    {
+        private uint _referenceCount;
+        private Dictionary<string, object>? _keyObjectMap;
+        private Dictionary<object, string>? _objectKeyMap;
+
+        public DefaultReferenceResolver(bool isWrite)
+        {
+            _referenceCount = default;
+
+            if (isWrite)
+            {
+                _objectKeyMap = new Dictionary<object, string>(ReferenceEqualsEqualityComparer<object>.Comparer);
+                _keyObjectMap = null;
+            }
+            else
+            {
+                _keyObjectMap = new Dictionary<string, object>();
+                _objectKeyMap = null;
+            }
+        }
+
+        public void AddReference(string key, object value)
+        {
+            if (!JsonHelpers.TryAdd(_keyObjectMap!, key, value))
+            {
+                ThrowHelper.ThrowJsonException_MetadataDuplicateIdFound(key);
+            }
+        }
+
+        public string GetOrAddReference(object value, out bool alreadyExists)
+        {
+            alreadyExists = _objectKeyMap!.TryGetValue(value, out string? key);
+            if (!alreadyExists)
+            {
+                key = (++_referenceCount).ToString();
+                _objectKeyMap.Add(value, key);
+            }
+
+            return key!;
+        }
+
+        public object ResolveReference(string key)
+        {
+            if (!_keyObjectMap!.TryGetValue(key, out object? value))
+            {
+                ThrowHelper.ThrowJsonException_MetadataReferenceNotFound(key);
+            }
+
+            return value!;
+        }
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPreservedReference.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Text.Json
+{
+    internal class JsonPreservedReference<T>
+    {
+#nullable disable
+        public T Values { get; set; }
+#nullable enable
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPropertyInfo.cs

@@ -20,6 +20,7 @@ internal abstract class JsonPropertyInfo
         private static readonly JsonDictionaryConverter s_jsonImmutableDictionaryConverter = new DefaultImmutableDictionaryConverter();
 
         public static readonly JsonPropertyInfo s_missingProperty = GetMissingProperty();
+        public static readonly JsonPropertyInfo s_metadataProperty = GetMetadataProperty();
 
         private JsonClassInfo? _elementClassInfo;
         private JsonClassInfo? _runtimeClassInfo;
@@ -43,6 +44,14 @@ private static JsonPropertyInfo GetMissingProperty()
             return info;
         }
 
+        public static JsonPropertyInfo GetMetadataProperty()
+        {
+            JsonPropertyInfo info = new JsonPropertyInfoNotNullable<object, object, object, object>();
+            info.ShouldDeserialize = true;
+
+            return info;
+        }
+
         // Copy any settings defined at run-time to the new property.
         public void CopyRuntimeSettingsTo(JsonPropertyInfo other)
         {
@@ -603,5 +612,7 @@ private void VerifyWrite(int originalDepth, Utf8JsonWriter writer)
                 ThrowHelper.ThrowJsonException_SerializationConverterWrite(ConverterBase);
             }
         }
+
+        public abstract Type GetJsonPreservedReferenceType();
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPropertyInfoCommon.cs

@@ -235,5 +235,10 @@ public override IDictionary CreateImmutableDictionaryInstance(ref ReadStack stat
 
             return collection;
         }
+
+        public override Type GetJsonPreservedReferenceType()
+        {
+            return typeof(JsonPreservedReference<TDeclaredProperty>);
+        }
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandleDictionary.cs

@@ -53,9 +53,15 @@ private static void HandleStartDictionary(JsonSerializerOptions options, ref Rea
 
                     state.Current.ReturnValue = classInfo.CreateObject();
                 }
-                else
+                else if (state.Current.IsProcessingObject(ClassType.Enumerable))
                 {
-                    ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(classInfo.Type);
+                    // Array with metadata within the dictionary.
+                    HandleStartObjectInEnumerable(ref state, options, classInfo.Type);
+
+                    Debug.Assert(options.ReferenceHandling.ShouldReadPreservedReferences());
+                    Debug.Assert(state.Current.JsonClassInfo!.Type.GetGenericTypeDefinition() == typeof(JsonPreservedReference<>));
+
+                    state.Current.ReturnValue = state.Current.JsonClassInfo.CreateObject!();
                 }
 
                 return;

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandleMetadata.cs

@@ -0,0 +1,112 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Text.Json
+{
+    public static partial class JsonSerializer
+    {
+        private static void HandleMetadataPropertyValue(ref Utf8JsonReader reader, ref ReadStack state)
+        {
+            if (reader.TokenType != JsonTokenType.String)
+            {
+                ThrowHelper.ThrowJsonException_MetadataValueWasNotString();
+            }
+
+            MetadataPropertyName metadata = state.Current.MetadataProperty;
+            string key = reader.GetString()!;
+
+            if (metadata == MetadataPropertyName.Id)
+            {
+                state.ReferenceResolver.AddReference(key, GetValueToPreserve(ref state));
+            }
+            else if (metadata == MetadataPropertyName.Ref)
+            {
+                state.Current.ReferenceId = key;
+            }
+
+            state.Current.ReadMetadataValue = false;
+        }
+
+        private static object GetValueToPreserve(ref ReadStack state)
+        {
+            return state.Current.IsProcessingProperty(ClassType.Dictionary) ?
+                state.Current.JsonPropertyInfo!.GetValueAsObject(state.Current.ReturnValue)! : state.Current.ReturnValue!;
+        }
+
+        internal static MetadataPropertyName GetMetadataPropertyName(ReadOnlySpan<byte> propertyName, ref ReadStack state, ref Utf8JsonReader reader)
+        {
+            if (propertyName[0] == '$')
+            {
+                switch (propertyName.Length)
+                {
+                    case 3:
+                        if (propertyName[1] == 'i' &&
+                            propertyName[2] == 'd')
+                        {
+                            return MetadataPropertyName.Id;
+                        }
+                        break;
+
+                    case 4:
+                        if (propertyName[1] == 'r' &&
+                            propertyName[2] == 'e' &&
+                            propertyName[3] == 'f')
+                        {
+                            return MetadataPropertyName.Ref;
+                        }
+                        break;
+
+                    case 7:
+                        // Only Preserved Arrays are allowed to read $values as metadata.
+                        if (state.Current.IsPreservedArray &&
+                            propertyName[1] == 'v' &&
+                            propertyName[2] == 'a' &&
+                            propertyName[3] == 'l' &&
+                            propertyName[4] == 'u' &&
+                            propertyName[5] == 'e' &&
+                            propertyName[6] == 's')
+                        {
+                            return MetadataPropertyName.Values;
+                        }
+                        break;
+                }
+
+                // Fail state.
+                // Set PropertyInfo or KeyName to write down the conflicting property name in JsonException.Path
+                if (state.Current.IsProcessingDictionary())
+                {
+                    state.Current.KeyName = reader.GetString();
+                }
+                else
+                {
+                    // TODO: Hook up JsonPropertyInfoAsString here instead.
+                    JsonPropertyInfo info = JsonPropertyInfo.s_metadataProperty;
+                    info.JsonPropertyName = propertyName.ToArray();
+                    state.Current.JsonPropertyInfo = info;
+                }
+
+                ThrowHelper.ThrowJsonException_MetadataInvalidPropertyWithLeadingSign();
+            }
+
+            return MetadataPropertyName.NoMetadata;
+        }
+
+        private static void HandleReference(ref ReadStack state)
+        {
+            object referenceValue = state.ReferenceResolver.ResolveReference(state.Current.ReferenceId);
+            if (state.Current.IsProcessingProperty(ClassType.Dictionary))
+            {
+                ApplyObjectToEnumerable(referenceValue, ref state, setPropertyDirectly: true);
+                state.Current.EndProperty();
+            }
+            else
+            {
+                state.Current.ReturnValue = referenceValue;
+                HandleEndObject(ref state);
+            }
+
+            state.Current.ShouldHandleReference = false;
+        }
+    }
+}