Parallel type checking for impl files with backing sig files (#13737)

* revamp parallel checking

* simplify names

* fix diagnostics

* code formatting

* update surface area

* simplify diagnostic logging and format diagnostics eagerly when processing in parallel

* format code

* remove SplitRelatedDiagnostic

* fix build and cleanup

* further cleanup

* further cleanup

* format code

* fix flaterrors

* allow error recovery on collisions

* fix name generation to be deterministic

* fix build

* Update RecursiveSafetyAnalysis.fs

* Add flag for parallel type checking of files backed by signatures.

* Update src/Compiler/Driver/ParseAndCheckInputs.fs

Co-authored-by: Petr Pokorny <[email protected]>

* format code

* Update TypeTests.fs

* Prefix DiagnosticsLoggerProvider with I

* Remove duplicate hadSig binding.

* Add basic test for ParallelCheckingWithSignatureFiles flag.

* Add additional CI job.

* Produce binlog for ParallelCheckingWithSignatureFiles

* Update azure-pipelines.yml

* Update azure-pipelines.yml

* Update TypeTests.fs

* Update SyntaxTreeTests

* Correct code after rebase

* Correct SynModuleSigDecl in SignatureTypeTests.fs

* Format ParseAndCheckInputs.fs

Co-authored-by: Don Syme <[email protected]>
Co-authored-by: Don Syme <[email protected]>
Co-authored-by: Petr Pokorny <[email protected]>
Co-authored-by: Vlad Zarytovskii <[email protected]>

Author: 4 people

FSharpBuild.Directory.Build.props

@@ -26,6 +26,7 @@
     <WarningsAsErrors>1182;0025;$(WarningsAsErrors)</WarningsAsErrors>
     <OtherFlags>$(OtherFlags) --nowarn:3384</OtherFlags>
     <OtherFlags>$(OtherFlags) --times --nowarn:75</OtherFlags>
+    <OtherFlags Condition="$(ParallelCheckingWithSignatureFilesOn) == 'true'">$(OtherFlags) --test:ParallelCheckingWithSignatureFilesOn</OtherFlags>
   </PropertyGroup>
 
   <!-- nuget -->

azure-pipelines.yml

@@ -508,6 +508,36 @@ stages:
         #      filePath: eng\tests\UpToDate.ps1
         #      arguments: -configuration $(_BuildConfig) -ci -binaryLog
 
+        # Run Build with --test:ParallelCheckingWithSignatureFilesOn
+        - job: ParallelCheckingWithSignatureFiles
+          condition: eq(variables['Build.Reason'], 'PullRequest')
+          variables:
+            - name: _SignType
+              value: Test
+          pool:
+            name: NetCore-Public
+            demands: ImageOverride -equals $(WindowsMachineQueueName)
+          timeoutInMinutes: 90
+          steps:
+            - checkout: self
+              clean: true
+            - task: UseDotNet@2
+              displayName: install SDK
+              inputs:
+                packageType: sdk
+                useGlobalJson: true
+                includePreviewVersions: false
+                workingDirectory: $(Build.SourcesDirectory)
+                installationPath: $(Build.SourcesDirectory)/.dotnet
+            - script: .\build.cmd -c Release -binaryLog /p:ParallelCheckingWithSignatureFilesOn=true
+              displayName: ParallelCheckingWithSignatureFiles build with Debug configuration
+            - task: PublishPipelineArtifact@1
+              displayName: Publish ParallelCheckingWithSignatureFiles Logs
+              inputs:
+                targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release'
+                artifactName: 'ParallelCheckingWithSignatureFiles Attempt $(System.JobAttempt) Logs'
+              continueOnError: true
+
         # Plain build Windows
         - job: Plain_Build_Windows
           pool:

src/Compiler/Checking/CheckBasics.fs

@@ -328,13 +328,14 @@ type TcFileState =
 
     /// Create a new compilation environment
     static member Create
-         (g, isScript, niceNameGen, amap, thisCcu, isSig, haveSig, conditionalDefines, tcSink, tcVal, isInternalTestSpanStackReferring,
+         (g, isScript, amap, thisCcu, isSig, haveSig, conditionalDefines, tcSink, tcVal, isInternalTestSpanStackReferring,
           tcPat,
           tcSimplePats,
           tcSequenceExpressionEntry,
           tcArrayOrListSequenceExpression,
           tcComputationExpression) =
 
+        let niceNameGen = NiceNameGenerator()
         let infoReader = InfoReader(g, amap)
         let instantiationGenerator m tpsorig = FreshenTypars g m tpsorig
         let nameResolver = NameResolver(g, amap, infoReader, instantiationGenerator)

src/Compiler/Checking/CheckBasics.fsi

@@ -311,7 +311,6 @@ type TcFileState =
     static member Create:
         g: TcGlobals *
         isScript: bool *
-        niceNameGen: NiceNameGenerator *
         amap: ImportMap *
         thisCcu: CcuThunk *
         isSig: bool *

src/Compiler/Checking/CheckDeclarations.fs

@@ -688,13 +688,12 @@ let TcOpenDecl (cenv: cenv) mOpenDecl scopem env target =
     | SynOpenDeclTarget.Type (synType, m) ->
         TcOpenTypeDecl cenv mOpenDecl scopem env (synType, m)
 
-let MakeSafeInitField (g: TcGlobals) env m isStatic = 
+let MakeSafeInitField (cenv: cenv) env m isStatic = 
     let id =
         // Ensure that we have an g.CompilerGlobalState
-        assert(g.CompilerGlobalState |> Option.isSome)
-        ident(g.CompilerGlobalState.Value.NiceNameGenerator.FreshCompilerGeneratedName("init", m), m)
+        ident(cenv.niceNameGen.FreshCompilerGeneratedName("init", m), m)
     let taccess = TAccess [env.eAccessPath]
-    Construct.NewRecdField isStatic None id false g.int_ty true true [] [] XmlDoc.Empty taccess true
+    Construct.NewRecdField isStatic None id false cenv.g.int_ty true true [] [] XmlDoc.Empty taccess true
 
 // Checking of mutually recursive types, members and 'let' bindings in classes
 //
@@ -1268,7 +1267,7 @@ module MutRecBindingChecking =
                                 | _ -> false)
 
                         if needsSafeStaticInit && hasStaticBindings then
-                            let rfield = MakeSafeInitField g envForDecls tcref.Range true
+                            let rfield = MakeSafeInitField cenv envForDecls tcref.Range true
                             SafeInitField(mkRecdFieldRef tcref rfield.LogicalName, rfield)
                         else
                             NoSafeInitInfo
@@ -2426,7 +2425,7 @@ module EstablishTypeDefinitionCores =
     let ComputeInstanceSafeInitInfo (cenv: cenv) env m thisTy = 
         let g = cenv.g
         if InstanceMembersNeedSafeInitCheck cenv m thisTy then 
-            let rfield = MakeSafeInitField g env m false
+            let rfield = MakeSafeInitField cenv env m false
             let tcref = tcrefOfAppTy g thisTy
             SafeInitField (mkRecdFieldRef tcref rfield.LogicalName, rfield)
         else
@@ -4479,7 +4478,7 @@ let rec TcSignatureElementNonMutRec (cenv: cenv) parent typeNames endm (env: TcE
 
                     // Publish the combined module type
                     env.eModuleOrNamespaceTypeAccumulator.Value <- 
-                        CombineCcuContentFragments m [env.eModuleOrNamespaceTypeAccumulator.Value; modTyRoot]
+                        CombineCcuContentFragments [env.eModuleOrNamespaceTypeAccumulator.Value; modTyRoot]
                     env
 
             return env
@@ -4801,7 +4800,7 @@ let rec TcModuleOrNamespaceElementNonMutRec (cenv: cenv) parent typeNames scopem
 
                   // Publish the combined module type
                   env.eModuleOrNamespaceTypeAccumulator.Value <-
-                      CombineCcuContentFragments m [env.eModuleOrNamespaceTypeAccumulator.Value; modTyRoot]
+                      CombineCcuContentFragments [env.eModuleOrNamespaceTypeAccumulator.Value; modTyRoot]
                   env, openDecls
 
           let moduleContentsRoot = BuildRootModuleContents kind.IsModule enclosingNamespacePath envNS.eCompPath moduleContents
@@ -5157,7 +5156,7 @@ let MakeInitialEnv env =
 /// Typecheck, then close the inference scope and then check the file meets its signature (if any)
 let CheckOneImplFile 
        // checkForErrors: A function to help us stop reporting cascading errors 
-       (g, niceNameGen, amap,
+       (g, amap,
         thisCcu,
         openDecls0,
         checkForErrors,
@@ -5173,7 +5172,7 @@ let CheckOneImplFile
 
     cancellable {
         let cenv = 
-            cenv.Create (g, isScript, niceNameGen, amap, thisCcu, false, Option.isSome rootSigOpt,
+            cenv.Create (g, isScript, amap, thisCcu, false, Option.isSome rootSigOpt,
                 conditionalDefines, tcSink, (LightweightTcValForUsingInBuildMethodCall g), isInternalTestSpanStackReferring,
                 tcPat=TcPat,
                 tcSimplePats=TcSimplePats,
@@ -5291,17 +5290,17 @@ let CheckOneImplFile
 
         let implFile = CheckedImplFile (qualNameOfFile, scopedPragmas, implFileTy, implFileContents, hasExplicitEntryPoint, isScript, anonRecdTypes, namedDebugPointsForInlinedCode)
 
-        return (topAttrs, implFile, implFileTypePriorToSig, envAtEnd, cenv.createsGeneratedProvidedTypes)
+        return (topAttrs, implFile, envAtEnd, cenv.createsGeneratedProvidedTypes)
      } 
 
 
 
 /// Check an entire signature file
-let CheckOneSigFile (g, niceNameGen, amap, thisCcu, checkForErrors, conditionalDefines, tcSink, isInternalTestSpanStackReferring) tcEnv (ParsedSigFileInput (qualifiedNameOfFile = qualNameOfFile; modules = sigFileFrags)) = 
+let CheckOneSigFile (g, amap, thisCcu, checkForErrors, conditionalDefines, tcSink, isInternalTestSpanStackReferring) tcEnv (sigFile: ParsedSigFileInput) = 
  cancellable {     
     let cenv = 
         cenv.Create 
-            (g, false, niceNameGen, amap, thisCcu, true, false, conditionalDefines, tcSink,
+            (g, false, amap, thisCcu, true, false, conditionalDefines, tcSink,
              (LightweightTcValForUsingInBuildMethodCall g), isInternalTestSpanStackReferring,
              tcPat=TcPat,
              tcSimplePats=TcSimplePats,
@@ -5311,16 +5310,16 @@ let CheckOneSigFile (g, niceNameGen, amap, thisCcu, checkForErrors, conditionalD
 
     let envinner, moduleTyAcc = MakeInitialEnv tcEnv 
 
-    let specs = [ for x in sigFileFrags -> SynModuleSigDecl.NamespaceFragment x ]
-    let! tcEnv = TcSignatureElements cenv ParentNone qualNameOfFile.Range envinner PreXmlDoc.Empty None specs
+    let specs = [ for x in sigFile.Contents -> SynModuleSigDecl.NamespaceFragment x ]
+    let! tcEnv = TcSignatureElements cenv ParentNone sigFile.QualifiedName.Range envinner PreXmlDoc.Empty None specs
 
     let sigFileType = moduleTyAcc.Value
 
     if not (checkForErrors()) then  
         try
             sigFileType |> IterTyconsOfModuleOrNamespaceType (fun tycon ->
                 FinalTypeDefinitionChecksAtEndOfInferenceScope(cenv.infoReader, tcEnv.NameEnv, cenv.tcSink, false, tcEnv.DisplayEnv, tycon))
-        with exn -> errorRecovery exn qualNameOfFile.Range
+        with exn -> errorRecovery exn sigFile.QualifiedName.Range
 
     return (tcEnv, sigFileType, cenv.createsGeneratedProvidedTypes)
  }

src/Compiler/Checking/CheckDeclarations.fsi

@@ -49,7 +49,6 @@ val AddLocalSubModule:
 
 val CheckOneImplFile:
     TcGlobals *
-    NiceNameGenerator *
     ImportMap *
     CcuThunk *
     OpenDeclaration list *
@@ -60,17 +59,10 @@ val CheckOneImplFile:
     TcEnv *
     ModuleOrNamespaceType option *
     ParsedImplFileInput ->
-        Cancellable<TopAttribs * CheckedImplFile * ModuleOrNamespaceType * TcEnv * bool>
+        Cancellable<TopAttribs * CheckedImplFile * TcEnv * bool>
 
 val CheckOneSigFile:
-    TcGlobals *
-    NiceNameGenerator *
-    ImportMap *
-    CcuThunk *
-    (unit -> bool) *
-    ConditionalDefines option *
-    TcResultsSink *
-    bool ->
+    TcGlobals * ImportMap * CcuThunk * (unit -> bool) * ConditionalDefines option * TcResultsSink * bool ->
         TcEnv ->
         ParsedSigFileInput ->
             Cancellable<TcEnv * ModuleOrNamespaceType * bool>

src/Compiler/Checking/ConstraintSolver.fs

@@ -244,15 +244,15 @@ exception ConstraintSolverMissingConstraint of displayEnv: DisplayEnv * Typar *
 
 exception ConstraintSolverError of string * range * range
 
-exception ErrorFromApplyingDefault of tcGlobals: TcGlobals * displayEnv: DisplayEnv * Typar * TType * exn * range
+exception ErrorFromApplyingDefault of tcGlobals: TcGlobals * displayEnv: DisplayEnv * Typar * TType * error: exn * range: range
 
-exception ErrorFromAddingTypeEquation of tcGlobals: TcGlobals * displayEnv: DisplayEnv * actualTy: TType * expectedTy: TType * exn * range
+exception ErrorFromAddingTypeEquation of tcGlobals: TcGlobals * displayEnv: DisplayEnv * actualTy: TType * expectedTy: TType * error: exn * range: range
 
-exception ErrorsFromAddingSubsumptionConstraint of tcGlobals: TcGlobals * displayEnv: DisplayEnv * actualTy: TType * expectedTy: TType * exn * ContextInfo * parameterRange: range
+exception ErrorsFromAddingSubsumptionConstraint of tcGlobals: TcGlobals * displayEnv: DisplayEnv * actualTy: TType * expectedTy: TType * error: exn * ctxtInfo: ContextInfo * parameterRange: range
 
-exception ErrorFromAddingConstraint of displayEnv: DisplayEnv * exn * range
+exception ErrorFromAddingConstraint of displayEnv: DisplayEnv * error: exn * range: range
 
-exception UnresolvedOverloading of displayEnv: DisplayEnv * callerArgs: CallerArgs<Expr> * failure: OverloadResolutionFailure * range
+exception UnresolvedOverloading of displayEnv: DisplayEnv * callerArgs: CallerArgs<Expr> * failure: OverloadResolutionFailure * range: range
 
 exception UnresolvedConversionOperator of displayEnv: DisplayEnv * TType * TType * range
 

src/Compiler/Checking/ConstraintSolver.fsi

@@ -170,33 +170,39 @@ exception ConstraintSolverMissingConstraint of displayEnv: DisplayEnv * Typar *
 
 exception ConstraintSolverError of string * range * range
 
-exception ErrorFromApplyingDefault of tcGlobals: TcGlobals * displayEnv: DisplayEnv * Typar * TType * exn * range
+exception ErrorFromApplyingDefault of
+    tcGlobals: TcGlobals *
+    displayEnv: DisplayEnv *
+    Typar *
+    TType *
+    error: exn *
+    range: range
 
 exception ErrorFromAddingTypeEquation of
     tcGlobals: TcGlobals *
     displayEnv: DisplayEnv *
     actualTy: TType *
     expectedTy: TType *
-    exn *
-    range
+    error: exn *
+    range: range
 
 exception ErrorsFromAddingSubsumptionConstraint of
     tcGlobals: TcGlobals *
     displayEnv: DisplayEnv *
     actualTy: TType *
     expectedTy: TType *
-    exn *
-    ContextInfo *
+    error: exn *
+    ctxtInfo: ContextInfo *
     parameterRange: range
 
-exception ErrorFromAddingConstraint of displayEnv: DisplayEnv * exn * range
+exception ErrorFromAddingConstraint of displayEnv: DisplayEnv * error: exn * range: range
 exception UnresolvedConversionOperator of displayEnv: DisplayEnv * TType * TType * range
 
 exception UnresolvedOverloading of
     displayEnv: DisplayEnv *
     callerArgs: CallerArgs<Expr> *
     failure: OverloadResolutionFailure *
-    range
+    range: range
 
 exception NonRigidTypar of displayEnv: DisplayEnv * string option * range * TType * TType * range
 

src/Compiler/Checking/import.fs

@@ -584,7 +584,7 @@ let ImportILAssemblyTypeDefs (amap, m, auxModLoader, aref, mainmod: ILModuleDef)
     let scoref = ILScopeRef.Assembly aref
     let mtypsForExportedTypes = ImportILAssemblyExportedTypes amap m auxModLoader scoref mainmod.ManifestOfAssembly.ExportedTypes
     let mainmod = ImportILAssemblyMainTypeDefs amap m scoref mainmod
-    CombineCcuContentFragments m (mainmod :: mtypsForExportedTypes)
+    CombineCcuContentFragments (mainmod :: mtypsForExportedTypes)
 
 /// Import the type forwarder table for an IL assembly
 let ImportILAssemblyTypeForwarders (amap, m, exportedTypes: ILExportedTypesAndForwarders): CcuTypeForwarderTable =            

src/Compiler/CodeGen/IlxGen.fs

@@ -11552,6 +11552,11 @@ let CodegenAssembly cenv eenv mgbuf implFiles =
     match List.tryFrontAndBack implFiles with
     | None -> ()
     | Some (firstImplFiles, lastImplFile) ->
+
+        // Generate the assembly sequentially, implementation file by implementation file.
+        //
+        // NOTE: In theory this could be done in parallel, except for the presence of linear
+        // state in the AssemblyBuilder
         let eenv = List.fold (GenImplFile cenv mgbuf None) eenv firstImplFiles
         let eenv = GenImplFile cenv mgbuf cenv.options.mainMethodInfo eenv lastImplFile
 
@@ -11626,7 +11631,7 @@ type IlxGenResults =
 
 let GenerateCode (cenv, anonTypeTable, eenv, CheckedAssemblyAfterOptimization implFiles, assemAttribs, moduleAttribs) =
 
-    use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.IlxGen
+    use _ = UseBuildPhase BuildPhase.IlxGen
     let g = cenv.g
 
     // Generate the implementations into the mgbuf