Skip to content

Commit d9aa8ff

Browse files
chsienkieerhardt
chsienki
and
eerhardt
committed
Make JsonGenerator be an incremental generator (dotnet#57088)
* Make JsonGenerator be an incremental generator * Improve incrementalism by doing less work when not applicable * Change SourceGeneration.UnitTests to SourceGeneration.Unit.Tests so it is built and executed in CI * Get unit tests running after IIncrementalGenerator migration * Fix duplicate file name tests by working around dotnet/roslyn#54185. * Fix unit tests now that they are running in CI against non-English languages. * Fix System.Text.Json.SourceGeneration.Unit.Tests on WASM * Disable STJ.SourceGeneration.Unit.Tests on Browser Co-authored-by: Eric Erhardt <[email protected]>
1 parent 4ebbb8e commit d9aa8ff

11 files changed

Lines changed: 101 additions & 60 deletions

‎eng/Versions.props‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@
4040
<ProjectServicingConfiguration Include="Microsoft.NETCore.App.Ref" PatchVersion="0" />
4141
</ItemGroup>
4242
<PropertyGroup>
43-
<!-- For source generator support we need to target a pinned version in order to be able to run on older versions of Roslyn -->
44-
<MicrosoftCodeAnalysisCSharpWorkspacesVersion>3.9.0</MicrosoftCodeAnalysisCSharpWorkspacesVersion>
45-
<MicrosoftCodeAnalysisVersion>3.9.0</MicrosoftCodeAnalysisVersion>
43+
<!-- For source generator support we are targeting the latest version of Roslyn for now, until we can support multi-targeting -->
44+
<MicrosoftCodeAnalysisCSharpWorkspacesVersion>4.0.0-3.final</MicrosoftCodeAnalysisCSharpWorkspacesVersion>
45+
<MicrosoftCodeAnalysisVersion>4.0.0-3.final</MicrosoftCodeAnalysisVersion>
4646
</PropertyGroup>
4747
<PropertyGroup>
4848
<!-- Code analysis dependencies -->

‎src/libraries/System.Text.Json/System.Text.Json.sln‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Encodings.Web",
4343
EndProject
4444
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Collections.Immutable", "..\System.Collections.Immutable\ref\System.Collections.Immutable.csproj", "{BE27618A-2916-4269-9AD5-6BC5EDC32B30}"
4545
EndProject
46-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.Text.Json.SourceGeneration.UnitTests", "tests\System.Text.Json.SourceGeneration.UnitTests\System.Text.Json.SourceGeneration.UnitTests.csproj", "{F6A18EB5-A8CC-4A39-9E85-5FA226019C3D}"
46+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.Text.Json.SourceGeneration.Unit.Tests", "tests\System.Text.Json.SourceGeneration.Unit.Tests\System.Text.Json.SourceGeneration.Unit.Tests.csproj", "{F6A18EB5-A8CC-4A39-9E85-5FA226019C3D}"
4747
EndProject
4848
Global
4949
GlobalSection(SolutionConfigurationPlatforms) = preSolution

‎src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,15 +79,17 @@ private sealed partial class Emitter
7979
defaultSeverity: DiagnosticSeverity.Warning,
8080
isEnabledByDefault: true);
8181

82-
private readonly GeneratorExecutionContext _executionContext;
82+
private readonly SourceProductionContext _sourceProductionContext;
8383

8484
private ContextGenerationSpec _currentContext = null!;
8585

8686
private readonly SourceGenerationSpec _generationSpec = null!;
8787

88-
public Emitter(in GeneratorExecutionContext executionContext, SourceGenerationSpec generationSpec)
88+
private readonly HashSet<string> _emittedPropertyFileNames = new();
89+
90+
public Emitter(in SourceProductionContext sourceProductionContext, SourceGenerationSpec generationSpec)
8991
{
90-
_executionContext = executionContext;
92+
_sourceProductionContext = sourceProductionContext;
9193
_generationSpec = generationSpec;
9294
}
9395

@@ -166,7 +168,7 @@ namespace {@namespace}
166168
sb.AppendLine("}");
167169
}
168170

169-
_executionContext.AddSource(fileName, SourceText.From(sb.ToString(), Encoding.UTF8));
171+
_sourceProductionContext.AddSource(fileName, SourceText.From(sb.ToString(), Encoding.UTF8));
170172
}
171173

172174
private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec)
@@ -243,7 +245,7 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec)
243245
break;
244246
case ClassType.TypeUnsupportedBySourceGen:
245247
{
246-
_executionContext.ReportDiagnostic(
248+
_sourceProductionContext.ReportDiagnostic(
247249
Diagnostic.Create(TypeNotSupported, Location.None, new string[] { typeGenerationSpec.TypeRef }));
248250
return;
249251
}
@@ -253,13 +255,16 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec)
253255
}
254256
}
255257

256-
try
258+
// Don't add a duplicate file, but instead raise a diagnostic to say the duplicate has been skipped.
259+
// Workaround https://github.com/dotnet/roslyn/issues/54185 by keeping track of the file names we've used.
260+
string propertyFileName = $"{_currentContext.ContextType.Name}.{typeGenerationSpec.TypeInfoPropertyName}.g.cs";
261+
if (_emittedPropertyFileNames.Add(propertyFileName))
257262
{
258-
AddSource($"{_currentContext.ContextType.Name}.{typeGenerationSpec.TypeInfoPropertyName}.g.cs", source);
263+
AddSource(propertyFileName, source);
259264
}
260-
catch (ArgumentException)
265+
else
261266
{
262-
_executionContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, Location.None, new string[] { typeGenerationSpec.TypeInfoPropertyName }));
267+
_sourceProductionContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, Location.None, new string[] { typeGenerationSpec.TypeInfoPropertyName }));
263268
}
264269
}
265270

‎src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Collections;
55
using System.Collections.Concurrent;
66
using System.Collections.Generic;
7+
using System.Collections.Immutable;
78
using System.Diagnostics;
89
using System.Diagnostics.CodeAnalysis;
910
using System.Linq;
@@ -31,7 +32,8 @@ private sealed class Parser
3132
private const string JsonPropertyNameAttributeFullName = "System.Text.Json.Serialization.JsonPropertyNameAttribute";
3233
private const string JsonPropertyOrderAttributeFullName = "System.Text.Json.Serialization.JsonPropertyOrderAttribute";
3334

34-
private readonly GeneratorExecutionContext _executionContext;
35+
private readonly Compilation _compilation;
36+
private readonly SourceProductionContext _sourceProductionContext;
3537
private readonly MetadataLoadContextInternal _metadataLoadContext;
3638

3739
private readonly Type _ilistOfTType;
@@ -43,7 +45,7 @@ private sealed class Parser
4345
private readonly Type? _dictionaryType;
4446
private readonly Type? _idictionaryOfTKeyTValueType;
4547
private readonly Type? _ireadonlyDictionaryType;
46-
private readonly Type? _isetType;
48+
private readonly Type? _isetType;
4749
private readonly Type? _stackOfTType;
4850
private readonly Type? _queueOfTType;
4951
private readonly Type? _concurrentStackType;
@@ -96,10 +98,11 @@ private sealed class Parser
9698
defaultSeverity: DiagnosticSeverity.Error,
9799
isEnabledByDefault: true);
98100

99-
public Parser(in GeneratorExecutionContext executionContext)
101+
public Parser(Compilation compilation, in SourceProductionContext sourceProductionContext)
100102
{
101-
_executionContext = executionContext;
102-
_metadataLoadContext = new MetadataLoadContextInternal(executionContext.Compilation);
103+
_compilation = compilation;
104+
_sourceProductionContext = sourceProductionContext;
105+
_metadataLoadContext = new MetadataLoadContextInternal(_compilation);
103106

104107
_ilistOfTType = _metadataLoadContext.Resolve(SpecialType.System_Collections_Generic_IList_T);
105108
_icollectionOfTType = _metadataLoadContext.Resolve(SpecialType.System_Collections_Generic_ICollection_T);
@@ -138,9 +141,9 @@ public Parser(in GeneratorExecutionContext executionContext)
138141
PopulateKnownTypes();
139142
}
140143

141-
public SourceGenerationSpec? GetGenerationSpec(List<ClassDeclarationSyntax> classDeclarationSyntaxList)
144+
public SourceGenerationSpec? GetGenerationSpec(ImmutableArray<ClassDeclarationSyntax> classDeclarationSyntaxList)
142145
{
143-
Compilation compilation = _executionContext.Compilation;
146+
Compilation compilation = _compilation;
144147
INamedTypeSymbol jsonSerializerContextSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSerializerContext");
145148
INamedTypeSymbol jsonSerializableAttributeSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSerializableAttribute");
146149
INamedTypeSymbol jsonSourceGenerationOptionsAttributeSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSourceGenerationOptionsAttribute");
@@ -198,7 +201,7 @@ public Parser(in GeneratorExecutionContext executionContext)
198201
if (!TryGetClassDeclarationList(contextTypeSymbol, out List<string> classDeclarationList))
199202
{
200203
// Class or one of its containing types is not partial so we can't add to it.
201-
_executionContext.ReportDiagnostic(Diagnostic.Create(ContextClassesMustBePartial, Location.None, new string[] { contextTypeSymbol.Name }));
204+
_sourceProductionContext.ReportDiagnostic(Diagnostic.Create(ContextClassesMustBePartial, Location.None, new string[] { contextTypeSymbol.Name }));
202205
continue;
203206
}
204207

@@ -400,6 +403,36 @@ private static bool TryGetClassDeclarationList(INamedTypeSymbol typeSymbol, [Not
400403
return typeGenerationSpec;
401404
}
402405

406+
internal static bool IsSyntaxTargetForGeneration(SyntaxNode node) => node is ClassDeclarationSyntax { AttributeLists: { Count: > 0 }, BaseList: { Types : {Count : > 0 } } };
407+
408+
internal static ClassDeclarationSyntax? GetSemanticTargetForGeneration(GeneratorSyntaxContext context)
409+
{
410+
var classDeclarationSyntax = (ClassDeclarationSyntax)context.Node;
411+
412+
foreach (AttributeListSyntax attributeListSyntax in classDeclarationSyntax.AttributeLists)
413+
{
414+
foreach (AttributeSyntax attributeSyntax in attributeListSyntax.Attributes)
415+
{
416+
IMethodSymbol attributeSymbol = context.SemanticModel.GetSymbolInfo(attributeSyntax).Symbol as IMethodSymbol;
417+
if (attributeSymbol == null)
418+
{
419+
continue;
420+
}
421+
422+
INamedTypeSymbol attributeContainingTypeSymbol = attributeSymbol.ContainingType;
423+
string fullName = attributeContainingTypeSymbol.ToDisplayString();
424+
425+
if (fullName == "System.Text.Json.Serialization.JsonSerializableAttribute")
426+
{
427+
return classDeclarationSyntax;
428+
}
429+
}
430+
431+
}
432+
433+
return null;
434+
}
435+
403436
private static JsonSourceGenerationMode? GetJsonSourceGenerationModeEnumVal(SyntaxNode propertyValueMode)
404437
{
405438
IEnumerable<string> enumTokens = propertyValueMode
@@ -729,7 +762,7 @@ private TypeGenerationSpec GetOrAddTypeGenerationSpec(Type type, JsonSourceGener
729762
if (!type.TryGetDeserializationConstructor(useDefaultCtorInAnnotatedStructs, out ConstructorInfo? constructor))
730763
{
731764
classType = ClassType.TypeUnsupportedBySourceGen;
732-
_executionContext.ReportDiagnostic(Diagnostic.Create(MultipleJsonConstructorAttribute, Location.None, new string[] { $"{type}" }));
765+
_sourceProductionContext.ReportDiagnostic(Diagnostic.Create(MultipleJsonConstructorAttribute, Location.None, new string[] { $"{type}" }));
733766
}
734767
else
735768
{

‎src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs‎

Lines changed: 17 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
//#define LAUNCH_DEBUGGER
45
using System.Collections.Generic;
6+
using System.Collections.Immutable;
57
using System.Diagnostics;
68
using System.Linq;
79
using System.Reflection;
@@ -16,60 +18,44 @@ namespace System.Text.Json.SourceGeneration
1618
/// Generates source code to optimize serialization and deserialization with JsonSerializer.
1719
/// </summary>
1820
[Generator]
19-
public sealed partial class JsonSourceGenerator : ISourceGenerator
21+
public sealed partial class JsonSourceGenerator : IIncrementalGenerator
2022
{
21-
/// <summary>
22-
/// Registers a syntax resolver to receive compilation units.
23-
/// </summary>
24-
/// <param name="context"></param>
25-
public void Initialize(GeneratorInitializationContext context)
23+
public void Initialize(IncrementalGeneratorInitializationContext context)
2624
{
27-
context.RegisterForSyntaxNotifications(() => new SyntaxReceiver());
25+
IncrementalValuesProvider<ClassDeclarationSyntax> classDeclarations = context.SyntaxProvider
26+
.CreateSyntaxProvider(static (s, _) => Parser.IsSyntaxTargetForGeneration(s), static (s, _) => Parser.GetSemanticTargetForGeneration(s))
27+
.Where(static c => c is not null);
28+
29+
IncrementalValueProvider<(Compilation, ImmutableArray<ClassDeclarationSyntax>)> compilationAndClasses =
30+
context.CompilationProvider.Combine(classDeclarations.Collect());
31+
32+
context.RegisterSourceOutput(compilationAndClasses, (spc, source) => Execute(source.Item1, source.Item2, spc));
2833
}
2934

30-
/// <summary>
31-
/// Generates source code to optimize serialization and deserialization with JsonSerializer.
32-
/// </summary>
33-
/// <param name="executionContext"></param>
34-
public void Execute(GeneratorExecutionContext executionContext)
35+
private void Execute(Compilation compilation, ImmutableArray<ClassDeclarationSyntax> contextClasses, SourceProductionContext context)
3536
{
3637
#if LAUNCH_DEBUGGER
3738
if (!Diagnostics.Debugger.IsAttached)
3839
{
3940
Diagnostics.Debugger.Launch();
4041
}
4142
#endif
42-
SyntaxReceiver receiver = (SyntaxReceiver)executionContext.SyntaxReceiver;
43-
List<ClassDeclarationSyntax>? contextClasses = receiver.ClassDeclarationSyntaxList;
44-
if (contextClasses == null)
43+
if (contextClasses.IsDefaultOrEmpty)
4544
{
4645
return;
4746
}
4847

49-
Parser parser = new(executionContext);
50-
SourceGenerationSpec? spec = parser.GetGenerationSpec(receiver.ClassDeclarationSyntaxList);
48+
Parser parser = new(compilation, context);
49+
SourceGenerationSpec? spec = parser.GetGenerationSpec(contextClasses);
5150
if (spec != null)
5251
{
5352
_rootTypes = spec.ContextGenerationSpecList[0].RootSerializableTypes;
5453

55-
Emitter emitter = new(executionContext, spec);
54+
Emitter emitter = new(context, spec);
5655
emitter.Emit();
5756
}
5857
}
5958

60-
private sealed class SyntaxReceiver : ISyntaxReceiver
61-
{
62-
public List<ClassDeclarationSyntax>? ClassDeclarationSyntaxList { get; private set; }
63-
64-
public void OnVisitSyntaxNode(SyntaxNode syntaxNode)
65-
{
66-
if (syntaxNode is ClassDeclarationSyntax cds)
67-
{
68-
(ClassDeclarationSyntaxList ??= new List<ClassDeclarationSyntax>()).Add(cds);
69-
}
70-
}
71-
}
72-
7359
/// <summary>
7460
/// Helper for unit tests.
7561
/// </summary>

src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/CompilationHelper.cs renamed to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System.Collections.Generic;
55
using System.Collections.Immutable;
6+
using System.Globalization;
67
using System.IO;
78
using System.Linq;
89
using System.Reflection;
@@ -16,6 +17,11 @@ namespace System.Text.Json.SourceGeneration.UnitTests
1617
{
1718
public class CompilationHelper
1819
{
20+
private static readonly CSharpParseOptions s_parseOptions =
21+
new CSharpParseOptions(kind: SourceCodeKind.Regular, documentationMode: DocumentationMode.Parse)
22+
// workaround https://github.com/dotnet/roslyn/pull/55866. We can remove "LangVersion=Preview" when we get a Roslyn build with that change.
23+
.WithLanguageVersion(LanguageVersion.Preview);
24+
1925
public static Compilation CreateCompilation(
2026
string source,
2127
MetadataReference[] additionalReferences = null,
@@ -55,18 +61,18 @@ public static Compilation CreateCompilation(
5561

5662
return CSharpCompilation.Create(
5763
assemblyName,
58-
syntaxTrees: new[] { CSharpSyntaxTree.ParseText(source) },
64+
syntaxTrees: new[] { CSharpSyntaxTree.ParseText(source, s_parseOptions) },
5965
references: references.ToArray(),
6066
options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)
6167
);
6268
}
6369

64-
private static GeneratorDriver CreateDriver(Compilation compilation, params ISourceGenerator[] generators)
70+
private static GeneratorDriver CreateDriver(Compilation compilation, IIncrementalGenerator[] generators)
6571
=> CSharpGeneratorDriver.Create(
66-
generators: ImmutableArray.Create(generators),
67-
parseOptions: new CSharpParseOptions(kind: SourceCodeKind.Regular, documentationMode: DocumentationMode.Parse));
72+
generators: generators.Select(g => g.AsSourceGenerator()),
73+
parseOptions: s_parseOptions);
6874

69-
public static Compilation RunGenerators(Compilation compilation, out ImmutableArray<Diagnostic> diagnostics, params ISourceGenerator[] generators)
75+
public static Compilation RunGenerators(Compilation compilation, out ImmutableArray<Diagnostic> diagnostics, params IIncrementalGenerator[] generators)
7076
{
7177
CreateDriver(compilation, generators).RunGeneratorsAndUpdateCompilation(compilation, out Compilation outCompilation, out diagnostics);
7278
return outCompilation;
@@ -267,7 +273,15 @@ internal static void CheckDiagnosticMessages(ImmutableArray<Diagnostic> diagnost
267273
Array.Sort(actualMessages);
268274
Array.Sort(expectedMessages);
269275

270-
Assert.Equal(expectedMessages, actualMessages);
276+
if (CultureInfo.CurrentUICulture.Name.StartsWith("en", StringComparison.OrdinalIgnoreCase))
277+
{
278+
Assert.Equal(expectedMessages, actualMessages);
279+
}
280+
else
281+
{
282+
// for non-English runs, just compare the number of messages are the same
283+
Assert.Equal(expectedMessages.Length, actualMessages.Length);
284+
}
271285
}
272286
}
273287
}

src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorDiagnosticsTests.cs renamed to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs

File renamed without changes.

src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorTests.cs renamed to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs

File renamed without changes.

src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/System.Text.Json.SourceGeneration.UnitTests.csproj renamed to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</PropertyGroup>
66

77
<ItemGroup>
8-
<PackageReference Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" PrivateAssets="all" />
8+
<PackageReference Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" />
99

1010
<ProjectReference Include="..\..\src\System.Text.Json.csproj" />
1111
<ProjectReference Include="..\..\gen\System.Text.Json.SourceGeneration.csproj" />

src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/TypeWrapperTests.cs renamed to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs

File renamed without changes.

0 commit comments

Comments
 (0)