diff --git a/azure-pipelines/.vsts-dotnet-mt-equivalence.yml b/azure-pipelines/.vsts-dotnet-mt-equivalence.yml new file mode 100644 index 00000000000..5e5265c7589 --- /dev/null +++ b/azure-pipelines/.vsts-dotnet-mt-equivalence.yml @@ -0,0 +1,210 @@ +# MSBuild -mt build-equivalence pipeline. +# +# Mirrors the experimental official build (azure-pipelines/.vsts-dotnet-exp-perf.yml) but, instead of +# producing a shippable drop, it runs the *same* official-style build two or three times in the same +# working directory and proves that turning on -mt does not change what the build produces: +# +# baseline build.cmd -pack -ci ... (official arguments) +# mt the same command with MSBUILD_MT_ENABLED=1, which is how arcade's +# eng/common/tools.ps1 appends -mt to the MSBuild command line +# control the same command again with no -mt anywhere +# +# The control run is what makes the result interpretable: it measures how much this repo's build output +# varies between two *identical* builds, so any difference that also appears in the control is +# pre-existing build non-determinism rather than something -mt caused. Keep it on unless you are +# deliberately trading rigour for pipeline time. +# +# The comparison itself (scripts/mt-equivalence) checks: +# * every produced file, byte for byte, with a documented rule set for the handful of outputs that +# are non-deterministic by construction; +# * the build logs, functionally, after normalizing away scheduling-dependent noise; +# * that the mt run really did run with -mt, read back out of the recorded MSBuild command line in +# the binary log, so the comparison cannot silently become vacuous. +# +# Nothing is signed, published or inserted: -sign and -publish are deliberately not passed, because a +# real signature would differ between the two runs for reasons that have nothing to do with -mt. + +# The pipeline FAILS when a difference is attributable to -mt, and when it cannot prove the -mt build +# actually ran with -mt. Pre-existing non-determinism found by the control run is surfaced as a +# warning instead, so a red run always means "-mt changed something". + +trigger: none +pr: none + +# Arcade's OfficialBuildId must be ., so pin the build number format rather than +# relying on the collection default. +name: $(Date:yyyyMMdd).$(Rev:r) + +schedules: +- cron: '0 2 * * *' + displayName: Daily -mt build equivalence (02:00 UTC) + branches: + include: + - main + # Run even when main has not changed: the point is to catch -mt regressions from the whole + # dependency surface (SDK, arcade, Roslyn, NuGet), not only from commits to this repo. + always: true + +parameters: +- name: buildConfiguration + displayName: Build configuration + type: string + default: Release + values: + - Release + - Debug +- name: msbuildEngine + displayName: MSBuild engine used for the build + type: string + default: vs + values: + - vs # what the official build uses; also the only engine that produces the VS insertion outputs + - dotnet +- name: runControlBuild + displayName: Also run the non-mt control build (recommended - it is what makes a diff interpretable) + type: boolean + default: true +- name: deepLogCompare + displayName: Also compare task and task-assembly coverage from a diagnostic-verbosity replay + type: boolean + default: true + +variables: +- name: Codeql.Enabled + value: false +- name: cfsNugetWarnLevel + value: warn +- name: nugetMultiFeedWarnLevel + value: none +- name: NugetSecurityAnalysisWarningLevel + value: none +- name: MTEquivalenceWorkDir + value: $(Agent.TempDirectory)\mt-equivalence +# Mirrors VisualStudio.DropName in .vsts-dotnet-build-jobs.yml. Required by AfterSigning.proj to +# generate the VS insertion manifests; the value is only embedded into those manifests, nothing is +# uploaded. It must be identical across the runs being compared, so it deliberately does not include +# $(Build.BuildNumber)-style values that could change between steps. +- name: VisualStudio.DropName + value: Products/$(System.TeamProject)/$(Build.Repository.Name)/mt-equivalence/$(Build.BuildNumber) +- group: AzureDevOps-Artifact-Feeds-Pats + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + featureFlags: + autoBaseline: false + pool: + name: VSEng-MicroBuildVSStable + os: windows + sdl: + # This pipeline produces no shippable output; it builds the repo twice and diffs the results. + # Keep the SDL surface minimal so a validation run cannot file bugs or block on scans that the + # real official build already covers. + policheck: + enabled: false + tsa: + enabled: false + sbom: + enabled: false + codeSignValidation: + enabled: false + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppressions.json + + stages: + - stage: equivalence + displayName: -mt build equivalence + jobs: + - job: Windows_NT + displayName: Build twice and compare + pool: + name: VSEng-MicroBuildVSStable + demands: + - agent.os -equals Windows_NT + + # Three full official-style builds of this repo plus the comparisons. Each build is roughly + # 20-35 minutes on this pool; the comparisons add about 5 minutes. + timeoutInMinutes: 240 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + # The reports are the product of this pipeline: JSON for machines, Markdown for humans. + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory)\mt-equivalence-reports + artifactName: mt-equivalence-reports + condition: always() + + variables: + - name: TeamName + value: MSBuild + + steps: + # No fetchDepth limit: SourceLink and the NuSpec repository-information targets read git + # metadata, and the official build does not shallow-clone either. + - checkout: self + clean: true + + - task: NuGetToolInstaller@1 + displayName: Install NuGet.exe + + - pwsh: | + New-Item -ItemType Directory -Force -Path '$(MTEquivalenceWorkDir)\reports' | Out-Null + New-Item -ItemType Directory -Force -Path '$(Build.ArtifactStagingDirectory)\mt-equivalence-reports' | Out-Null + $drive = Get-PSDrive ([System.IO.Path]::GetPathRoot('$(MTEquivalenceWorkDir)').TrimEnd('\', ':')) + Write-Host "Free space on $($drive.Name): $([math]::Round($drive.Free / 1GB, 1)) GB" + # Each snapshot of artifacts/ is roughly 4.5 GB, and up to three are kept at once. + if ($drive.Free -lt 25GB) { + Write-Host "##vso[task.logissue type=warning]Less than 25 GB free; the three artifact snapshots may not fit." + } + displayName: Prepare work directory + + # @self is required: inside an extends block, a relative template path resolves against the + # 1ESPipelineTemplates repository unless the repository alias is given explicitly. + - template: /eng/common/templates-official/steps/enable-internal-sources.yml@self + + - pwsh: | + $ErrorActionPreference = 'Stop' + # Boolean template parameters interpolate as True/False; PowerShell string comparison is + # case-insensitive, so compare against 'true'. + $scriptArgs = @{ + WorkDir = '$(MTEquivalenceWorkDir)' + Configuration = '${{ parameters.buildConfiguration }}' + MSBuildEngine = '${{ parameters.msbuildEngine }}' + OfficialBuildId = '$(Build.BuildNumber)' + VisualStudioDropName = '$(VisualStudio.DropName)' + } + if ('${{ parameters.runControlBuild }}' -ne 'true') { $scriptArgs['SkipControl'] = $true } + if ('${{ parameters.deepLogCompare }}' -eq 'true') { $scriptArgs['DeepLogCompare'] = $true } + + Write-Host ($scriptArgs.GetEnumerator() | Sort-Object Key | ForEach-Object { " -$($_.Key) $($_.Value)" }) + & '$(Build.SourcesDirectory)\scripts\mt-equivalence\Run-MTEquivalence.ps1' @scriptArgs + # Non-zero means a difference was attributed to -mt (or the -mt build could not be proven + # to have run with -mt); either way the job must go red. + exit $LASTEXITCODE + displayName: Build (baseline / -mt / control) and compare + failOnStderr: false + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - pwsh: | + $source = '$(MTEquivalenceWorkDir)\reports' + $target = '$(Build.ArtifactStagingDirectory)\mt-equivalence-reports' + if (Test-Path $source) { + Copy-Item -Path (Join-Path $source '*') -Destination $target -Recurse -Force -ErrorAction SilentlyContinue + } + $summary = Join-Path $target 'summary.md' + if (Test-Path $summary) { + Write-Host "##vso[task.uploadsummary]$summary" + } + else { + Write-Host "##vso[task.logissue type=warning]No comparison summary was produced; the run failed before the comparisons completed." + } + displayName: Collect and publish comparison reports + condition: always() diff --git a/documentation/specs/mt-build-equivalence-findings.md b/documentation/specs/mt-build-equivalence-findings.md new file mode 100644 index 00000000000..6d88eb5de7e --- /dev/null +++ b/documentation/specs/mt-build-equivalence-findings.md @@ -0,0 +1,335 @@ +# `-mt` build-equivalence findings + +Results of running MSBuild's official-style build with and without `-mt` and comparing the outputs +byte for byte. The harness that produces these results is +[`scripts/mt-equivalence`](../../scripts/mt-equivalence/README.md); the pipeline is +[`azure-pipelines/.vsts-dotnet-mt-equivalence.yml`](../../azure-pipelines/.vsts-dotnet-mt-equivalence.yml). + +Answers [#13603](https://github.com/dotnet/msbuild/issues/13603), which asks for a validation leg +proving that an `-mt` build produces the same artifacts as a standard one before the production +pipeline is switched to `-mt`. + +## Method + +Three official-style builds of the same commit, in the same working tree, differing only in whether +`MSBUILD_MT_ENABLED=1` was set (which is how `eng/common/tools.ps1` adds `-mt` to the MSBuild command +line): + +``` +build.cmd -ci -pack -configuration Release + /p:OfficialBuildId= /p:RepositoryName=dotnet/msbuild /p:TeamName=MSBuild + /p:DotNetPublishUsingPipelines=true /p:SuppressFinalPackageVersion=true /p:IsExperimental=true +``` + +`-sign` from the official job is omitted: it needs the MicroBuild signing plugin, and an Authenticode +signature embeds a trusted-timestamp countersignature, so signed binaries could never be byte-identical +between two runs. What `-mt` could actually affect is the content being signed, and that is compared +unsigned. `EnableNgenOptimization` and `GenerateSbom` are turned off (the former is the official +pipeline's own `$(SkipApplyOptimizationDataArg)`; the latter produces manifests that embed a timestamp +and a per-run GUID). `VisualStudioDropName` is supplied, exactly as the official job does, because +`AfterSigning.proj` requires it to generate the VS insertion manifests. + +`-publish` **is** passed. It is the official-build publish, not `dotnet publish`: with +`DotNetPublishUsingPipelines=true` it pushes nothing to any feed (it emits `##vso[artifact.upload]` +logging commands; feed publishing happens in the separate post-build stage) but it does produce the +Build Asset Registry manifest, the generated symbol packages and the PDBs staged for the symbol +server. All three are compared. + +The engine is `vs`, which is what the official job uses. This matters: only the `vs` engine builds the +**VS insertion outputs** — `artifacts/VSSetup` (`.vsix`, `.vsman`, `.vsmand`), the +`VS.ExternalAPIs.MSBuild` package and the arm64 flavours. With `-publish` the full official-style +build produces 28 061 artifact files, versus 23 329 for a `-msbuildEngine dotnet` build without +`-publish`, so validating that way would silently skip both the insertion surface and the publish +outputs. + +The third build (`control`) is a second non-`-mt` build. It establishes how much this repo's output +varies between two *identical* builds, so a difference in the `-mt` comparison can be attributed. + +That the `-mt` build really ran multithreaded is verified from the MSBuild command line recorded in +its binary log, not from the fact that the environment variable was set. + +Runs: `main` at `867c136`, `Release`. Several trios were run locally while developing the harness; the +authoritative ones are the full official configuration (`-MSBuildEngine vs`, `-publish`) executed on +the official MicroBuild pool by the pipeline itself, in +[DevDiv definition 28545](https://devdiv.visualstudio.com/DevDiv/_build?definitionId=28545): + +| run | commit | result | +|---|---|---| +| [14817800](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14817800&view=results) | `512bc3b` | **succeeded** — the numbers below, with the allowances restored | +| [14817555](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14817555&view=results) | `180809a` | failed on purpose — the `knownMtOnly` allowance was removed after #14550 merged, and the messages were still there because they come from the *driving* MSBuild | +| [14817031](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14817031&view=results) | `5aa9f18` | succeeded — same verdict before `main` was merged in | +| [14815634](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14815634&view=results) | `d87e327` | succeeded — same verdict before the structural comparison was added | +| [14816699](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14816699&view=results) | `13d285b` | failed — the structural reader could not load MSBuild under the agent's PowerShell (`CS1705`) | +| [14816238](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14816238&view=results) | `d5c62f8` | failed — same cause, reported as a misleading "assembly already loaded" | +| [14802518](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14802518&view=results) | `5b2e99f` | failed — DevDiv/1ES pool incident, unrelated to the comparison | +| [14802348](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14802348&view=results) | `756a173` | failed — same incident | +| [14802168](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14802168&view=results) | `9219205` | failed — inner builds' `##vso` commands were reaching the agent | +| [14801836](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14801836&view=results) | `c9ccccd` | failed — same cause; the comparison itself passed | +| [14789743](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14789743&view=results) | `ff1d43c` | failed — 1ES SDL injection noise, since filtered | + +Earlier `-msbuildEngine dotnet` trios run locally agreed with the pool runs. + +## Headline result + +**`-mt` changed nothing in the produced bits**, including every VS insertion output, the Build Asset +Registry manifest, all staged PDBs and all symbol-package payloads. + +| comparison | paths compared | byte-identical | unexpected differences | expected differences | +|---|---|---|---|---| +| `mt` vs `baseline` | 28 149 | 27 998 | **0** | 151 | +| `control` vs `baseline` | 28 149 | 27 999 | **0** | 150 | + +The count of expected differences moves by one or two between runs — those files are non-deterministic +by nature, so that number *is* the noise floor. What has to stay at zero is the unexpected column. + +Only 18 paths are excluded from the comparison altogether, and the report names every one of them: +14 Guardian SARIF files injected by the 1ES SDL template, 2 developer-convenience environment +scripts, 1 shortcut and 1 build log. Everything else in the tree is compared. + +The three builds are also structurally identical, read from the binlog event stream rather than from +rendered text: 384 distinct targets over 18 993 executions, 9 306 distinct `(project, target)` pairs, +122 tasks over 16 084 invocations, 53 projects over 1 842 builds, and no warnings or errors — every +count equal across `baseline`, `mt` and `control`. + +That the `-mt` build really ran multithreaded is asserted from the recorded command line, not assumed: + +``` +[mt-vs-baseline] baseline -mt = False / candidate -mt = True +[control] baseline -mt = False / candidate -mt = False +``` + +The harness was also shown to be capable of failing: removing the known-`-mt` log allowance surfaced +the real `-mt`-only lines (exit 1); flipping a single byte in `Microsoft.Build.dll` in the `mt` +snapshot was reported as `PE: 1 bytes, regions: .text@0x4000+1` (exit 1); appending one byte to +`manifest.json` inside `Microsoft.Build.vsix` — the one place the comparison is deliberately relaxed — +was still caught and named (exit 1); a target that executes but logs nothing was invisible to the text +comparison and caught by the structural one (exit 1); and presenting a non-`-mt` binlog as the `-mt` +candidate was rejected by the evidence guard (exit 1). Corrupting the *control* snapshot instead +produced a warning and exit 0, so pre-existing non-determinism cannot masquerade as an `-mt` +regression. + +Every file that differs between the `-mt` build and the baseline also differs between the two +identical non-`-mt` builds, and in the same way. + +There were no errors or warnings in any of the three builds, and the sets of executed tasks and of +task→assembly bindings were identical between `-mt` and baseline. + +## Pre-existing build non-determinism (not caused by `-mt`) + +These show up in the non-`-mt` control run as well. They are classified as expected by +[`ArtifactCompareRules.json`](../../scripts/mt-equivalence/ArtifactCompareRules.json). + +### 1. Three test projects are compiled with `/deterministic-` + +`Microsoft.Build.CommandLine.UnitTests`, `Microsoft.Build.Engine.UnitTests` and +`Microsoft.Build.Tasks.UnitTests` are compiled with `/deterministic-`, so each build stamps a fresh +random v4 GUID as the assembly MVID. Two identical builds therefore produce assemblies that differ in +exactly 17-18 bytes: the 16-byte MVID in the `#GUID` metadata heap plus the PE COFF `TimeDateStamp` +(which Roslyn derives from the MVID). + +``` +bin/Microsoft.Build.CommandLine.UnitTests/Release/net11.0/Microsoft.Build.CommandLine.UnitTests.dll + run A @0x247B4: 51 FB 0A F9 E9 25 B2 4D A5 47 84 6A F6 79 2B 69 + run B @0x247B4: 85 58 39 54 D3 8C 58 48 97 DB EE 2D 3C 3E 84 05 (both v4 = random) +``` + +Verified from the `Csc` command line in the binary log (`/deterministic-` for these projects, and the +`Deterministic` property evaluates to `true` at evaluation time but reaches the `Csc` task as +`False`). This affects 12 paths (6 assemblies × `bin` and `obj`). + +**Impact:** none on shipping bits — no product assembly is affected — but it does mean these test +assemblies are not reproducible. Worth fixing separately; it is unrelated to `-mt`. + +The harness does not hard-code these project names. It detects the shape of the difference: if the +only differing bytes are the `TimeDateStamp`, the `CheckSum` and a single 16-byte window, the +assembly was built non-deterministically, because under `/deterministic+` the MVID is a hash of the +emitted content and cannot differ on its own. + +### 2. MSBuild's `.vsix` — and therefore its VS insertion manifests — are not reproducible + +Every `.vsix` differs between two identical builds. The cause is a single attribute: + +```xml + + + + +``` + +`System.IO.Packaging` generates a fresh random OPC relationship id on every save, in +`_rels/.rels` and `_rels/manifest.json.rels`. Everything else in the package is byte-identical +(verified per entry). Because the `.vsix` bytes change, so does its size and SHA256, and that +cascades into every artifact derived from it: + +| file | what it inherits | +|---|---| +| `VSSetup/Release/Insertion/Microsoft.Build{,.Arm64}.vsman` | the vsix `sha256` and `size` | +| `…/Microsoft.Build{,.Arm64}.vsmand` | compressed form of the above | +| `…/Microsoft.Build{,.Arm64}.json` | the vsix `size` | +| `VSSetup.obj/Release/…/*.vsman.merge`, `*.vsman.overlay` | intermediates of the above | +| `obj/MSBuild.VSSetup{,.Arm64}/…/Microsoft.Build*.{vsix,json}` | intermediate copies | + +That is 16 of the 137 expected differences, all from one random 8-byte id. + +**Impact:** none on `-mt` (it reproduces identically in the control), but it does mean MSBuild's VS +insertion payload is not bit-reproducible, so two rebuilds of the same commit cannot be shown to +produce the same VSIX by hashing. Worth fixing separately if reproducible VS insertion is a goal. + +The harness handles this without going blind: the `.vsix` rule normalizes only the `Id="R…"` +attribute inside `_rels/*` and still compares every other byte of those files and every other entry. +A deliberate one-byte change to `manifest.json` inside a `.vsix` was still reported as +`payloadIdentical=False, contentDiff entries: manifest.json`. + +### 3. `.nupkg` entry timestamps + +All 27 packages differ. Every packed entry is byte-identical; only the zip's per-entry +`LastWriteTime` differs, because NuGet stamps entries with the wall-clock last-write time of the file +it packed. Compared with the `ComparePayload` disposition, which enforces payload equality and +tolerates the timestamps. + +### 4. MSBuild incremental-state caches + +70 `*.AssemblyReference.cache` / `*.GenerateResource.cache` files under `obj` differ: they record the +last-write timestamps of their inputs, which are wall-clock. Never shipped. + +### 5. `VS with MSBuild.slnx.lnk` + +A Windows shortcut generated for local development; embeds link-tracking data. Not a build output. + +### 6. Log-level noise that is not `-mt` specific + +Visible in the control run, so normalized away for both comparisons: + +* `Creating directory "..."` — parallel `Copy`/`MakeDir` invocations race; the loser finds the + directory already present and logs nothing, so both presence and count vary between identical + builds. +* Project-started / `Done Building Project` lines — a shared dependency is really built by whichever + requester reaches it first; every other requester is served from the results cache. Both the + requester and the exact target subset that executes are scheduling artifacts. +* Target header lines — MSBuild's text logger re-emits a target header whenever output from a + different node interleaves. +* Roslyn compiler-server startup (`Attempting to create process 'VBCSCompiler.exe'`, + `Successfully created process with process id N`, `Setting DOTNET_ROOT to ...`) — whether + `VBCSCompiler` has to be started depends on whether a server from an earlier build is still alive. +* NuGet restore chatter — depends on package-cache warmth. + +## `-mt`-only log differences + +One, and it is log noise. Under `-mt` a task routed to an out-of-process task host is represented +in-proc by a `TaskHostTask` proxy whose own assembly location is always `Microsoft.Build.dll`, so +MSBuild reported a `TaskAssemblyLocationMismatch` for most task invocations — 1 364 extra lines in an +official build. No produced byte changes, which the artifact comparison confirms, and the builds stay +structurally identical. Fixed by [#14550](https://github.com/dotnet/msbuild/pull/14550). + +**That fix landing in `main` does not clear this from the pipeline.** `-mt` is applied to the MSBuild +that *drives* the build — the pool's installed Visual Studio MSBuild — not to the MSBuild the build +produces, and the messages are logged by the former. Verified: after merging #14550, run +[14817555](https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14817555&view=results) still +reported all 1 364, attributed to +`C:\Program Files\Microsoft Visual Studio\18\Enterprise\MSBuild\Current\Bin\Microsoft.Build.dll` — +`MSBuildVersion = 18.8.2`, the MSBuild on the `VSEng-MicroBuildVSStable` pool. #14550 is in `main`, +which is 18.11. The `knownMtOnly` entries in +[`LogNormalizationRules.json`](../../scripts/mt-equivalence/LogNormalizationRules.json) therefore stay +until that pool moves to a VS carrying the fix; delete them then and the log comparison becomes +strict again. They excuse exactly 21 distinct lines — the 9 message variants (1 364 occurrences) and +the 12 target headers those messages cause — and nothing else. + +This generalises: a `knownMtOnly` entry is cleared by the fix *shipping to the agents*, not by it +merging. + +## Non-MSBuild noise from 1ES SDL injection + +The 1ES Official template injects Guardian's Roslyn analyzers into the build. That tooling is not part +of MSBuild and is not deterministic between two runs, so the harness excludes its own outputs: + +* `*.gdn.sarif` — Guardian analysis results written into the output tree. Ignored: they are analysis + results, not build outputs, and on the pool they were emitted in one build of an identical pair and + not the other. +* `__GdnDotnetAnalyzersMerged.ruleset` — a ruleset generated per project whose file + name embeds a fresh GUID on every build. The GUID is normalized away in logs and the file ignored. + +Both are excluded for the `-mt` comparison and the control alike, so they cannot mask a real +difference on one side only. + +## Audit of the exclusion rules + +Every exclusion was re-examined adversarially against the artifacts and binlogs of a green pool run, +on the assumption that a rule might be hiding a real `-mt` difference. Three things came out of it. + +**Every excused artifact difference is demonstrably pre-existing.** Of the 150 paths excused in the +`-mt` comparison, all 150 also differ between the two identical non-`-mt` builds. Not one difference +is excused on the strength of a written rule alone; each has an experimental control. + +**The target-header suppressions were justified by a control that cannot fail.** The `setOnly` rule +for target headers and its `knownMtOnly` counterpart stop comparing how many times a target header +appears — and a target header is exactly what would reveal a target running only under `-mt`. Their +stated justification was the `-DeepCompare` coverage comparison. That comparison does run, but target +coverage was deliberately excluded from the set of extractors allowed to fail it, precisely because +the header-based extraction it uses is itself unstable. So the suppression was justified by a check +that reports and never fails. + +The suppression turned out to be *correct* — reading the raw event stream out of all three binlogs +shows the builds are structurally identical: 384 distinct targets over 18 993 executions, 9 306 +distinct `(project, target)` pairs, 122 tasks over 16 083 invocations, 53 projects over 1 842 builds, +and zero warnings and errors — every count equal across baseline, `-mt` and control. The extra target +headers in the `-mt` text log are a side-effect of the spurious `TaskAssemblyLocationMismatch` +messages: the file logger only emits a target header when the target logs something, so targets that +are silent at normal verbosity acquire a header once something starts logging inside them. They will +disappear along with the messages once the agents pick up +[#14550](https://github.com/dotnet/msbuild/pull/14550). + +Fifteen target names had a higher header count under `-mt`. Twelve of them appeared in the `-mt` log +only, and every one of those header blocks contained nothing but those messages. The other three +(`PrepareForBuild`, `_GenerateSourceLinkFile`, `Build`) already appear in the baseline; their extra +headers are the file logger re-emitting a header when output from another node interleaves, which is +what the `setOnly` rule exists for. That re-emission is not `-mt` specific and swings in both +directions between two identical builds — `_CopyFilesMarkedCopyLocal`, for instance, produces 299 +headers in the baseline and 440 in the control. + +Being right by luck is not the same as being checked, so the structural comparison is now a +first-class tier of `Compare-Binlogs.ps1`: always on, enforced, netted against the control, and +costing about 2.5 seconds per binlog. It is strictly stronger than what it replaces, because it is +taken from the events rather than from rendered text — a target that executes but logs nothing is +invisible to a text comparison at any verbosity, and is caught here. That case is covered by a test. + +**Ignored paths left no trace in the report.** `Ignore` rules removed paths from the comparison +without recording anything, so nobody could review what was being hidden without re-running the +build. The artifact report now lists every `Ignore` rule with the number of paths it matched, how +many of those actually differed, and a sample, and flags rules that matched nothing. + +One latent defect was found and fixed along the way: `drop` patterns were matched against the raw log +line including the node-id/timestamp prefix, so every `^`-anchored rule silently stopped matching +whenever the replay engine emitted one. The pool's replay does not, which is why this never surfaced, +but `-msbuildEngine dotnet` does. The prefix is now stripped in its own stage ahead of `drop`. + +## What this does and does not prove + +Proven for the configuration tested (official-style `Release` build of this repo, Windows, +`-msbuildEngine vs` — the official engine — as well as `-msbuildEngine dotnet`): + +* the set of produced files is identical; +* every produced file is byte-identical except for outputs that are already non-deterministic without + `-mt`; +* the VS insertion outputs (`artifacts/VSSetup`: `.vsix`, `.vsman`, `.vsmand`, the component manifests + and the `VS.ExternalAPIs.MSBuild` package) are included, and every `.vsix` payload is identical + entry by entry; +* the `-publish` outputs are included: the Build Asset Registry manifest is byte-identical, all 34 + PDBs staged for the symbol server are byte-identical, and all 10 generated symbol packages are + payload-identical; +* no errors or warnings appear or disappear; +* the same tasks run, bound to the same task assemblies. + +Not covered: + +* Signing. It needs the MicroBuild plugin, and an Authenticode signature embeds a trusted-timestamp + countersignature, so two signed runs could never be byte-identical; `-sign` is therefore not passed + and the comparison is of unsigned binaries. Since signing is a post-processing step applied + identically to both runs, identical unsigned inputs imply equivalent signed outputs. +* OptProf / NGEN optimization data, for the same reason the official pipeline skips it when OptProf is + off: both runs would consume the same IBC drop, so it adds cost without signal. +* Feed publishing and the Build Asset Registry push, which happen in the post-build stage rather than + in the build. The manifest that drives them *is* compared. +* Non-Windows builds and the source-build/VMR leg (covered separately by + `azure-pipelines/vmr-sb-validation.yml`). +* Test execution — this pipeline deliberately does not run tests; `-mt` test coverage lives in + `.vsts-dotnet-ci.yml`. diff --git a/scripts/mt-equivalence/ArtifactCompareRules.json b/scripts/mt-equivalence/ArtifactCompareRules.json new file mode 100644 index 00000000000..9c30f32a5a0 --- /dev/null +++ b/scripts/mt-equivalence/ArtifactCompareRules.json @@ -0,0 +1,135 @@ +{ + "//": "Ordered rules; the first pattern that matches a path (relative to the artifacts root, forward slashes) wins. Disposition: Compare | ComparePayload | Informational | Ignore. Anything not matched defaults to Compare, i.e. it must be byte-identical. Patterns are matched with PowerShell -like, where '*' also crosses directory separators.", + "rules": [ + { + "//": "-publish writes these three under log/ and tmp/, which are otherwise ignored. They are real outputs of the official build - the Build Asset Registry manifest that drives what ships, the generated symbol packages, and the PDBs staged for the symbol server - so they are carved out ahead of the ignore rules below and must match.", + "pattern": "log/*/AssetManifest/*", + "disposition": "Compare", + "reason": "Build Asset Registry manifest produced by -publish. Lists every package, blob and PDB the build declares, so it is the manifest of what would ship; it must not change when -mt is turned on." + }, + { + "pattern": "tmp/*/SymbolPackages/*.nupkg", + "disposition": "ComparePayload", + "reason": "Symbol packages generated by -publish. Zip entry timestamps are wall-clock, but the packed payload must be byte-identical." + }, + { + "pattern": "tmp/*/PDBsToPublish/*", + "disposition": "Compare", + "reason": "PDBs staged by -publish for the symbol server; they are shipped debugging artifacts and must be byte-identical." + }, + { + "pattern": "log/**", + "disposition": "Ignore", + "reason": "Build logs are compared functionally by Compare-Binlogs.ps1. Binlogs and text logs embed wall-clock timestamps, durations, node ids and event ordering, so byte comparison is meaningless. Note the AssetManifest carve-out above." + }, + { + "pattern": "**/*.binlog", + "disposition": "Ignore", + "reason": "Binary logs embed wall-clock timestamps and event ordering by design." + }, + { + "pattern": "tmp/**", + "disposition": "Ignore", + "reason": "Scratch directory that arcade points TEMP/TMP at for the duration of the build; per-process and never shipped. Note the SymbolPackages and PDBsToPublish carve-outs above." + }, + { + "pattern": "**/_manifest/**", + "disposition": "Ignore", + "reason": "SBOM manifests embed a generation timestamp and a per-run document GUID." + }, + { + "//": "1ES SDL injection runs Guardian's Roslyn analyzers as part of the build and writes its results into the output tree. Those are analysis results about the source, not build products, and they are not even emitted consistently between two identical builds (observed as OnlyInCandidate in the control run on the MicroBuild pool).", + "pattern": "**/*.gdn.sarif", + "disposition": "Ignore", + "reason": "Guardian SARIF output injected by the 1ES SDL template; an analysis result, not a build output." + }, + { + "pattern": "**/*_GdnDotnetAnalyzersMerged.ruleset", + "disposition": "Ignore", + "reason": "Ruleset generated per project by 1ES SDL injection; its file name embeds a fresh GUID on every build." + }, + { + "pattern": "msbuild-build-env.*", + "disposition": "Ignore", + "reason": "Developer convenience script generated from the ambient PATH of the build process; environment-dependent, not a build output, never shipped." + }, + { + "pattern": "*.lnk", + "disposition": "Ignore", + "reason": "Windows shortcut generated for local development; embeds link-tracking data and timestamps. Not a build output." + }, + { + "pattern": "**/*.AssemblyReference.cache", + "disposition": "Informational", + "reason": "MSBuild RAR incremental-state cache. Keyed on the last-write timestamps of the referenced assemblies, which are wall-clock and therefore differ between any two builds." + }, + { + "pattern": "**/*.GenerateResource.cache", + "disposition": "Informational", + "reason": "GenerateResource incremental-state cache; records input file timestamps, which are wall-clock." + }, + { + "pattern": "**/*.CoreCompileInputs.cache", + "disposition": "Informational", + "reason": "CoreCompile incremental-state cache; hash of inputs including timestamped paths." + }, + { + "pattern": "**/*.FileListAbsolute.txt", + "disposition": "Informational", + "reason": "Clean-up bookkeeping list written by MSBuild; entry order follows the order in which copies completed, which is scheduling dependent." + }, + { + "pattern": "**/*.CopyComplete", + "disposition": "Informational", + "reason": "Zero-length marker file whose only content is its timestamp." + }, + { + "pattern": "**/*.nupkg", + "disposition": "ComparePayload", + "reason": "NuGet stores each entry's wall-clock last-write time in the zip. The packed payload must still be byte-identical." + }, + { + "pattern": "**/*.snupkg", + "disposition": "ComparePayload", + "reason": "NuGet stores each entry's wall-clock last-write time in the zip. The packed payload must still be byte-identical." + }, + { + "pattern": "**/*.vsix", + "disposition": "ComparePayload", + "reason": "The .vsix payload must be byte-identical. The OPC relationship id in _rels/*.rels is a fresh random value on every build (System.IO.Packaging generates it), so it is normalized away; everything else in those files, and every other entry, is still compared byte for byte.", + "normalizeEntries": [ + { + "entryPattern": "_rels/*", + "pattern": "Id=\"R[0-9a-fA-F]+\"", + "replacement": "Id=\"R\"" + } + ] + }, + { + "//": "The following VS insertion outputs all embed the size and/or SHA256 of the .vsix above. Because the .vsix is not byte-reproducible (random OPC relationship id), neither are they. The .vsix payload itself is verified by the rule above, so these are derived rather than independent differences. Remove these rules if the OPC id is ever made deterministic. Note: these patterns are matched with PowerShell -like, where '*' also crosses directory separators.", + "pattern": "*VSSetup*/*.vsman", + "disposition": "Informational", + "reason": "VS setup manifest; records the size and SHA256 of the .vsix, which is not byte-reproducible because System.IO.Packaging writes a random OPC relationship id. The .vsix payload itself is compared entry by entry." + }, + { + "pattern": "*VSSetup*/*.vsmand", + "disposition": "Informational", + "reason": "Compressed form of the .vsman above; inherits the same non-reproducible .vsix hash." + }, + { + "pattern": "*VSSetup*/*.vsman.merge", + "disposition": "Informational", + "reason": "Intermediate VS setup manifest; inherits the non-reproducible .vsix hash." + }, + { + "pattern": "*VSSetup*/*.vsman.overlay", + "disposition": "Informational", + "reason": "Intermediate VS setup manifest; inherits the non-reproducible .vsix hash." + }, + { + "pattern": "*VSSetup*/Microsoft.Build*.json", + "disposition": "Informational", + "reason": "VS setup component manifest; records the size of the .vsix, which is not byte-reproducible because System.IO.Packaging writes a random OPC relationship id." + } + ] +} diff --git a/scripts/mt-equivalence/BinlogStructure.cs b/scripts/mt-equivalence/BinlogStructure.cs new file mode 100644 index 00000000000..855d8f9ac01 --- /dev/null +++ b/scripts/mt-equivalence/BinlogStructure.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.Loader; +using System.Text.Json; +using Microsoft.Build.Framework; +using Microsoft.Build.Logging; + +/// +/// Writes the target/task/project execution counts of a binary log to a JSON file. +/// +/// +/// Comparing replayed *text* is unavoidably noisy: node ids interleave, and MSBuild's loggers only +/// emit a target header when that target happens to log something at the current verbosity, so a +/// target that is silent in one run and chatty in another looks like it only ran in one of them. +/// Diagnostic verbosity does not fix it either - engine-assigned TargetIds are not unique across +/// projects, and a measurable share of TaskStarted events never reach the text at all. The event +/// stream has none of that ambiguity: every target that executes raises exactly one TargetStarted. +/// +/// This is a standalone program rather than something loaded into the calling PowerShell because +/// Microsoft.Build targets a newer framework than the pwsh on the build agents runs on, so +/// Add-Type cannot even compile against it there (CS1705). Running on the SDK's own runtime side- +/// steps the whole question, and keeps MSBuild's assemblies out of the caller. +/// +/// +internal static class BinlogStructure +{ + private static int Main(string[] args) + { + if (args.Length != 3) + { + Console.Error.WriteLine("usage: binlogstructure "); + return 2; + } + + string binlog = args[0]; + string outputPath = args[1]; + string assemblyDirectory = args[2]; + + // Microsoft.Build drags in a good number of its neighbours; resolve them all out of the + // directory it came from rather than listing them. + AssemblyLoadContext.Default.Resolving += (context, name) => + { + string candidate = Path.Combine(assemblyDirectory, name.Name + ".dll"); + return File.Exists(candidate) ? context.LoadFromAssemblyPath(candidate) : null; + }; + + Counts counts = Collect(binlog); + + using (FileStream stream = File.Create(outputPath)) + { + JsonSerializer.Serialize(stream, counts); + } + + Console.WriteLine($"targets={Total(counts.Targets)} tasks={Total(counts.Tasks)} projects={Total(counts.Projects)}"); + return 0; + } + + private static int Total(Dictionary counts) + { + int total = 0; + foreach (int n in counts.Values) + { + total += n; + } + + return total; + } + + private sealed class Counts + { + public Dictionary Targets { get; } = new(StringComparer.Ordinal); + + public Dictionary TargetsByProject { get; } = new(StringComparer.Ordinal); + + public Dictionary Tasks { get; } = new(StringComparer.Ordinal); + + public Dictionary Projects { get; } = new(StringComparer.Ordinal); + + public Dictionary Diagnostics { get; } = new(StringComparer.Ordinal); + } + + private static void Bump(Dictionary counts, string key) + { + counts.TryGetValue(key, out int n); + counts[key] = n + 1; + } + + // The runs being compared share a working tree, so project paths already agree verbatim; only + // the directory separator is normalized, for the case where logs are compared across platforms. + private static string Project(string? path) => string.IsNullOrEmpty(path) ? "" : path.Replace('/', '\\'); + + private static Counts Collect(string binlogPath) + { + Counts counts = new(); + BinaryLogReplayEventSource reader = new(); + + reader.TargetStarted += (_, e) => + { + Bump(counts.Targets, "target " + e.TargetName); + Bump(counts.TargetsByProject, "target " + e.TargetName + " in " + Project(e.ProjectFile)); + }; + reader.TaskStarted += (_, e) => Bump(counts.Tasks, "task " + e.TaskName); + reader.ProjectStarted += (_, e) => Bump(counts.Projects, "project " + Project(e.ProjectFile)); + reader.WarningRaised += (_, e) => Bump(counts.Diagnostics, "warning " + e.Code); + reader.ErrorRaised += (_, e) => Bump(counts.Diagnostics, "error " + e.Code); + + reader.Replay(binlogPath); + return counts; + } +} diff --git a/scripts/mt-equivalence/Collect-Evidence.ps1 b/scripts/mt-equivalence/Collect-Evidence.ps1 new file mode 100644 index 00000000000..d60c300b9b9 --- /dev/null +++ b/scripts/mt-equivalence/Collect-Evidence.ps1 @@ -0,0 +1,160 @@ +<# +.SYNOPSIS + Collects the evidence needed to investigate an -mt build-equivalence failure after the fact. + +.DESCRIPTION + The artifact snapshots are far too large to publish (roughly 4.5 GB each, three of them), but + without *something* a failing run leaves only a list of paths - and a regression that reproduces + intermittently may never reproduce on a developer machine. This collects a bounded evidence bundle: + + binlogs/.binlog the binary log of each build, so the failure can be traced to the + task and target that produced the differing file + evidence// both versions of every differing file, so the bytes can be diffed + offline without rerunning anything + evidence-manifest.json what was collected, what was skipped, and why + + Differences are collected in priority order - the ones attributed to -mt first, then the rest of the + mt comparison, then the control comparison - so that when a cap is hit, the most important evidence + is what survives. + +.PARAMETER WorkDir + The directory holding the baseline/, mt/ and control/ snapshots plus reports/. + +.PARAMETER MaxFiles + Maximum number of file *pairs* to collect. A systemic regression could differ in thousands of files; + the first few are enough to diagnose it and the manifest records the true total. + +.PARAMETER MaxTotalBytes + Overall size cap for collected file pairs. Binlogs are always collected and do not count against it. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $WorkDir, + [string] $OutputDir, + [int] $MaxFiles = 200, + [long] $MaxTotalBytes = 500MB +) + +Set-StrictMode -Version 3.0 +$ErrorActionPreference = 'Stop' + +$reportsDir = Join-Path $WorkDir 'reports' +if (-not $OutputDir) { $OutputDir = $reportsDir } +$binlogDir = Join-Path $OutputDir 'binlogs' +$evidenceDir = Join-Path $OutputDir 'evidence' +New-Item -ItemType Directory -Force -Path $binlogDir | Out-Null + +function Get-Report { + param([string] $Name) + + $path = Join-Path $reportsDir $Name + if (-not (Test-Path -LiteralPath $path)) { return $null } + return Get-Content -Raw -LiteralPath $path | ConvertFrom-Json +} + +# --------------------------------------------------------------------------------------------- +# Binary logs. Always collected: they are small next to the snapshots, and they are the only way to +# trace a differing file back to the task that wrote it. +# --------------------------------------------------------------------------------------------- + +$collectedBinlogs = @() +foreach ($run in @('baseline', 'mt', 'control')) { + $snapshot = Join-Path $WorkDir $run + if (-not (Test-Path -LiteralPath $snapshot)) { continue } + $binlog = Get-ChildItem -LiteralPath (Join-Path $snapshot 'log') -Recurse -Filter '*.binlog' -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $binlog) { continue } + Copy-Item -LiteralPath $binlog.FullName -Destination (Join-Path $binlogDir "$run.binlog") -Force + $collectedBinlogs += [pscustomobject]@{ Run = $run; Bytes = $binlog.Length } +} + +# --------------------------------------------------------------------------------------------- +# Differing files, in priority order. +# --------------------------------------------------------------------------------------------- + +$verdict = Get-Report -Name 'verdict.json' +$mtReport = Get-Report -Name 'artifact-compare.mt-vs-baseline.json' +$controlReport = Get-Report -Name 'artifact-compare.control.json' + +$unexplained = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +if ($verdict -and $verdict.PSObject.Properties.Name -contains 'UnexplainedArtifacts') { + foreach ($p in @($verdict.UnexplainedArtifacts)) { if ($p) { [void]$unexplained.Add([string]$p) } } +} + +$queue = New-Object System.Collections.Generic.List[object] +$queued = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + +function Add-Candidate { + param([string] $Rel, [string] $CandidateRun, [string] $Priority) + + $key = "$CandidateRun|$Rel" + if (-not $queued.Add($key)) { return } + $queue.Add([pscustomobject]@{ Rel = $Rel; CandidateRun = $CandidateRun; Priority = $Priority }) +} + +# 1. Attributed to -mt: the reason the run is red. +foreach ($rel in $unexplained) { Add-Candidate -Rel $rel -CandidateRun 'mt' -Priority 'attributed-to-mt' } +# 2. The rest of the mt comparison's unexpected differences. +if ($mtReport) { foreach ($d in @($mtReport.Failures)) { if ($d) { Add-Candidate -Rel $d.Rel -CandidateRun 'mt' -Priority 'mt-comparison' } } } +# 3. The control comparison's unexpected differences: pre-existing non-determinism worth fixing. +if ($controlReport) { foreach ($d in @($controlReport.Failures)) { if ($d) { Add-Candidate -Rel $d.Rel -CandidateRun 'control' -Priority 'control-comparison' } } } + +$collected = New-Object System.Collections.Generic.List[object] +$skipped = New-Object System.Collections.Generic.List[object] +$totalBytes = 0L + +foreach ($item in $queue) { + if ($collected.Count -ge $MaxFiles) { + $skipped.Add([pscustomobject]@{ Rel = $item.Rel; Run = $item.CandidateRun; Reason = 'file-count cap reached' }) + continue + } + + $relNative = $item.Rel -replace '/', '\' + $pair = @( + [pscustomobject]@{ Run = 'baseline'; Path = Join-Path (Join-Path $WorkDir 'baseline') $relNative } + [pscustomobject]@{ Run = $item.CandidateRun; Path = Join-Path (Join-Path $WorkDir $item.CandidateRun) $relNative } + ) + + $pairBytes = 0L + foreach ($side in $pair) { if (Test-Path -LiteralPath $side.Path) { $pairBytes += (Get-Item -LiteralPath $side.Path).Length } } + if (($totalBytes + $pairBytes) -gt $MaxTotalBytes) { + $skipped.Add([pscustomobject]@{ Rel = $item.Rel; Run = $item.CandidateRun; Reason = "size cap reached (pair is $([math]::Round($pairBytes / 1MB, 1)) MB)" }) + continue + } + + $copiedSides = @() + foreach ($side in $pair) { + if (-not (Test-Path -LiteralPath $side.Path)) { continue } + $destination = Join-Path (Join-Path $evidenceDir $side.Run) $relNative + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + Copy-Item -LiteralPath $side.Path -Destination $destination -Force + $copiedSides += $side.Run + } + + $totalBytes += $pairBytes + $collected.Add([pscustomobject]@{ Rel = $item.Rel; Priority = $item.Priority; Sides = $copiedSides; Bytes = $pairBytes }) +} + +# --------------------------------------------------------------------------------------------- +# Manifest +# --------------------------------------------------------------------------------------------- + +$manifest = [pscustomobject]@{ + GeneratedUtc = (Get-Date).ToUniversalTime().ToString('o') + WorkDir = $WorkDir + Binlogs = $collectedBinlogs + CollectedPairCount = $collected.Count + CollectedBytes = $totalBytes + SkippedCount = $skipped.Count + MaxFiles = $MaxFiles + MaxTotalBytes = $MaxTotalBytes + Collected = $collected.ToArray() + Skipped = $skipped.ToArray() + Note = 'evidence// holds both versions of each differing file. binlogs/.binlog is the build log that produced it. The full artifact snapshots are not published: they are roughly 4.5 GB each.' +} +$manifest | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $OutputDir 'evidence-manifest.json') -Encoding utf8 + +Write-Host ("[evidence] {0} binlog(s); {1} differing file pair(s) collected ({2:n1} MB); {3} skipped by caps" -f ` + $collectedBinlogs.Count, $collected.Count, ($totalBytes / 1MB), $skipped.Count) +if ($skipped.Count -gt 0) { + Write-Host "[evidence] caps: MaxFiles=$MaxFiles, MaxTotalBytes=$([math]::Round($MaxTotalBytes / 1MB))MB. See evidence-manifest.json for the full list of differences." +} diff --git a/scripts/mt-equivalence/Compare-Artifacts.ps1 b/scripts/mt-equivalence/Compare-Artifacts.ps1 new file mode 100644 index 00000000000..a7b6f65c80e --- /dev/null +++ b/scripts/mt-equivalence/Compare-Artifacts.ps1 @@ -0,0 +1,519 @@ +<# +.SYNOPSIS + Byte-level comparison of two MSBuild artifact trees produced by otherwise-identical builds. + +.DESCRIPTION + Walks both trees, hashes every file, and classifies each relative path using an ordered rule set + (ArtifactCompareRules.json). Rules assign one of four dispositions: + + Compare - must be byte-identical; any difference fails the comparison. + ComparePayload - zip container (.nupkg/.vsix/...): the *entries* must be byte-identical, but the + zip's own per-entry timestamps may differ (NuGet stamps entries with the source + file's last-write time, which is wall-clock). + Informational - differences are reported but do not fail. Used only for outputs that are + non-deterministic by construction, each with a written reason. + Ignore - excluded from the comparison entirely. + + Two automatic reclassifications keep the rule file small and principled: + + * A PE image whose only differing bytes are the 16-byte MVID plus the PE COFF TimeDateStamp and + checksum was compiled with /deterministic-. Under /deterministic+ the MVID is a hash of the + emitted content, so an MVID-only delta can never be caused by the build engine: it is a random + GUID. Such files are downgraded to Informational with that reason recorded. + * A zip whose entries all hash equal but whose entry timestamps differ is payload-identical. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $BaselineDir, + [Parameter(Mandatory = $true)][string] $CandidateDir, + [Parameter(Mandatory = $true)][string] $OutputDir, + [string] $RulesFile, + [string] $Label = 'compare', + [int] $MaxReportedDiffs = 250, + [int] $ThrottleLimit = 0 +) + +Set-StrictMode -Version 3.0 +$ErrorActionPreference = 'Stop' + +if (-not $RulesFile) { $RulesFile = Join-Path $PSScriptRoot 'ArtifactCompareRules.json' } +foreach ($p in @($BaselineDir, $CandidateDir, $RulesFile)) { + if (-not (Test-Path -LiteralPath $p)) { throw "Not found: $p" } +} + +$BaselineDir = (Resolve-Path $BaselineDir).Path +$CandidateDir = (Resolve-Path $CandidateDir).Path +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +Add-Type -TypeDefinition (Get-Content -Raw -LiteralPath (Join-Path $PSScriptRoot 'MtCompareNative.cs')) -ErrorAction Stop +Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction SilentlyContinue + +# --------------------------------------------------------------------------------------------- +# Rules +# --------------------------------------------------------------------------------------------- + +$rules = @((Get-Content -Raw -LiteralPath $RulesFile | ConvertFrom-Json).rules) + +function Get-Disposition { + param([string] $RelativePath) + + foreach ($rule in $rules) { + if ($RelativePath -like $rule.pattern) { + $normalizations = @() + if ($rule.PSObject.Properties.Name -contains 'normalizeEntries') { $normalizations = @($rule.normalizeEntries) } + return [pscustomobject]@{ + Disposition = $rule.disposition + Reason = $rule.reason + Pattern = $rule.pattern + NormalizeEntries = $normalizations + } + } + } + return [pscustomobject]@{ Disposition = 'Compare'; Reason = ''; Pattern = '(default)'; NormalizeEntries = @() } +} + +# --------------------------------------------------------------------------------------------- +# Index both trees +# --------------------------------------------------------------------------------------------- + +function Get-FileIndex { + param([string] $Root) + + $index = @{} + foreach ($e in [MtCompareNative]::HashTree($Root, $ThrottleLimit)) { $index[$e.RelativePath] = $e } + return $index +} + +Write-Host "[$Label] indexing $BaselineDir and $CandidateDir ..." +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$baseIndex = Get-FileIndex -Root $BaselineDir +$candIndex = Get-FileIndex -Root $CandidateDir +$sw.Stop() +Write-Host ("[{0}] indexed {1} baseline / {2} candidate files in {3:n1}s" -f $Label, $baseIndex.Count, $candIndex.Count, $sw.Elapsed.TotalSeconds) + +# --------------------------------------------------------------------------------------------- +# Triage helpers +# --------------------------------------------------------------------------------------------- + +$script:PeExtensions = @('.dll', '.exe', '.winmd', '.netmodule') +$script:ZipExtensions = @('.nupkg', '.snupkg', '.vsix', '.zip', '.jar') +$script:TextExtensions = @('.txt', '.json', '.xml', '.props', '.targets', '.config', '.md', '.cs', '.rsp', + '.vsman', '.csv', '.nuspec', '.pkgdef', '.editorconfig', '.log', '.swr', '.man', '.filelist') + +function Get-PeLayout { + param([string] $Path) + + try { + $fs = [System.IO.File]::OpenRead($Path) + try { + $header = New-Object byte[] 4096 + $read = $fs.Read($header, 0, $header.Length) + if ($read -lt 0x40 -or $header[0] -ne 0x4D -or $header[1] -ne 0x5A) { return $null } + + $peOffset = [System.BitConverter]::ToInt32($header, 0x3C) + if ($peOffset -le 0 -or ($peOffset + 24) -ge $read) { return $null } + if ($header[$peOffset] -ne 0x50 -or $header[$peOffset + 1] -ne 0x45) { return $null } + + $coff = $peOffset + 4 + $numberOfSections = [System.BitConverter]::ToUInt16($header, $coff + 2) + $sizeOfOptionalHeader = [System.BitConverter]::ToUInt16($header, $coff + 16) + $optionalHeader = $coff + 20 + # CheckSum sits at offset 64 of the optional header for both PE32 and PE32+. + $checkSumOffset = $optionalHeader + 64 + $sectionTable = $optionalHeader + $sizeOfOptionalHeader + + $sections = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $numberOfSections; $i++) { + $entry = $sectionTable + ($i * 40) + if (($entry + 40) -gt $read) { break } + $sections.Add([pscustomobject]@{ + Name = [System.Text.Encoding]::ASCII.GetString($header, $entry, 8).TrimEnd([char]0) + RawPtr = [int64][System.BitConverter]::ToUInt32($header, $entry + 20) + RawSize = [int64][System.BitConverter]::ToUInt32($header, $entry + 16) + }) + } + + return [pscustomobject]@{ + TimeDateStampOffset = [int64]($coff + 4) + CheckSumOffset = [int64]$checkSumOffset + Sections = $sections + HeaderEnd = [int64]($sectionTable + ($numberOfSections * 40)) + } + } + finally { $fs.Dispose() } + } + catch { return $null } +} + +function Get-RegionName { + param($Layout, [int64] $Offset) + + if (-not $Layout) { return 'unknown' } + if ($Offset -ge $Layout.TimeDateStampOffset -and $Offset -lt ($Layout.TimeDateStampOffset + 4)) { return 'coff:TimeDateStamp' } + if ($Offset -ge $Layout.CheckSumOffset -and $Offset -lt ($Layout.CheckSumOffset + 4)) { return 'optional:CheckSum' } + foreach ($s in $Layout.Sections) { + if ($s.RawSize -gt 0 -and $Offset -ge $s.RawPtr -and $Offset -lt ($s.RawPtr + $s.RawSize)) { return $s.Name } + } + if ($Offset -lt $Layout.HeaderEnd) { return 'headers' } + return 'unmapped' +} + +function Compare-PeFile { + param([string] $LeftPath, [string] $RightPath, [long] $LeftLength, [long] $RightLength) + + $layout = Get-PeLayout -Path $LeftPath + $runs = [MtCompareNative]::DiffRuns($LeftPath, $RightPath, 256) + + $regionRuns = @() + $totalBytes = 0 + foreach ($run in $runs) { + $regionRuns += [pscustomobject]@{ + Region = (Get-RegionName -Layout $layout -Offset $run.Start) + Start = $run.Start + Length = $run.Length + } + $totalBytes += $run.Length + } + + # /deterministic- signature: only the COFF TimeDateStamp (which Roslyn derives from the MVID), + # the optional-header CheckSum, and exactly one 16-byte run (the MVID in the #GUID heap) differ. + $payloadRuns = @($regionRuns | Where-Object { $_.Region -notin @('coff:TimeDateStamp', 'optional:CheckSum') }) + # Coincidentally-equal bytes can split the 16-byte GUID into several runs, so require the payload + # differences to be confined to a single 16-byte window rather than to a single run. + $mvidOnly = $false + if ($runs.Count -gt 0 -and $payloadRuns.Count -gt 0 -and $LeftLength -eq $RightLength) { + $windowStart = ($payloadRuns | Measure-Object -Property Start -Minimum).Minimum + $windowEnd = ($payloadRuns | ForEach-Object { $_.Start + $_.Length } | Measure-Object -Maximum).Maximum + $stampDiffered = @($regionRuns | Where-Object { $_.Region -eq 'coff:TimeDateStamp' }).Count -gt 0 + $mvidOnly = $stampDiffered -and (($windowEnd - $windowStart) -le 16) + } + + [pscustomobject]@{ + Kind = 'pe' + DifferingRuns = $runs.Count + DifferingBytes = $totalBytes + Regions = (($regionRuns | Select-Object -First 12 | ForEach-Object { '{0}@0x{1:X}+{2}' -f $_.Region, $_.Start, $_.Length }) -join ', ') + MvidAndStampOnly = $mvidOnly + } +} + +function Compare-ZipFile { + param([string] $LeftPath, [string] $RightPath, $NormalizeEntries = @()) + + # Entry normalizations let a rule tolerate a documented, provably-irrelevant per-build value inside + # a specific entry (for example the random OPC relationship id in a .vsix) while still comparing the + # rest of that entry, and every other entry, byte for byte. + $normalizers = @() + foreach ($n in $NormalizeEntries) { + $normalizers += [pscustomobject]@{ + EntryPattern = $n.entryPattern + Regex = [regex]::new($n.pattern) + Replacement = $n.replacement + } + } + + function Get-ZipEntries([string] $Path) { + $result = @{} + $archive = [System.IO.Compression.ZipFile]::OpenRead($Path) + try { + foreach ($entry in $archive.Entries) { + $applicable = @($normalizers | Where-Object { $entry.FullName -like $_.EntryPattern }) + $stream = $entry.Open() + try { + if ($applicable.Count -gt 0) { + $reader = New-Object System.IO.StreamReader($stream) + try { $content = $reader.ReadToEnd() } finally { $reader.Dispose() } + foreach ($n in $applicable) { $content = $n.Regex.Replace($content, $n.Replacement) } + $bytes = [System.Text.Encoding]::UTF8.GetBytes($content) + $hash = [System.BitConverter]::ToString(([System.Security.Cryptography.SHA256]::Create()).ComputeHash($bytes)).Replace('-', '') + } + else { + $hash = [System.BitConverter]::ToString(([System.Security.Cryptography.SHA256]::Create()).ComputeHash($stream)).Replace('-', '') + } + } + finally { $stream.Dispose() } + $result[$entry.FullName] = [pscustomobject]@{ Hash = $hash; Time = $entry.LastWriteTime.UtcDateTime.ToString('o') } + } + } + finally { $archive.Dispose() } + return $result + } + + try { + $left = Get-ZipEntries $LeftPath + $right = Get-ZipEntries $RightPath + } + catch { + return [pscustomobject]@{ + Kind = 'zip'; Error = $_.Exception.Message; PayloadIdentical = $false + EntriesWithContentDiff = @(); EntriesOnlyInBaseline = @(); EntriesOnlyInCandidate = @(); EntriesWithTimestampDiff = @() + } + } + + $onlyLeft = @($left.Keys | Where-Object { -not $right.ContainsKey($_) } | Sort-Object) + $onlyRight = @($right.Keys | Where-Object { -not $left.ContainsKey($_) } | Sort-Object) + $contentDiff = New-Object System.Collections.Generic.List[string] + $timeDiff = New-Object System.Collections.Generic.List[string] + + foreach ($key in ($left.Keys | Sort-Object)) { + if (-not $right.ContainsKey($key)) { continue } + if ($left[$key].Hash -ne $right[$key].Hash) { $contentDiff.Add($key) } + elseif ($left[$key].Time -ne $right[$key].Time) { $timeDiff.Add($key) } + } + + [pscustomobject]@{ + Kind = 'zip' + EntryCount = $left.Count + EntriesOnlyInBaseline = $onlyLeft + EntriesOnlyInCandidate = $onlyRight + EntriesWithContentDiff = @($contentDiff) + EntriesWithTimestampDiff = @($timeDiff) + PayloadIdentical = ($onlyLeft.Count -eq 0 -and $onlyRight.Count -eq 0 -and $contentDiff.Count -eq 0) + } +} + +function Compare-TextFile { + param([string] $LeftPath, [string] $RightPath) + + try { + $left = [System.IO.File]::ReadAllLines($LeftPath) + $right = [System.IO.File]::ReadAllLines($RightPath) + } + catch { return [pscustomobject]@{ Kind = 'binary' } } + + $max = [Math]::Max($left.Length, $right.Length) + $diffs = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $max -and $diffs.Count -lt 5; $i++) { + $l = $(if ($i -lt $left.Length) { $left[$i] } else { '' }) + $r = $(if ($i -lt $right.Length) { $right[$i] } else { '' }) + if ($l -ne $r) { $diffs.Add([pscustomobject]@{ Line = $i + 1; Baseline = $l; Candidate = $r }) } + } + [pscustomobject]@{ Kind = 'text'; FirstDifferences = $diffs.ToArray() } +} + +function Get-DiffDetail { + param([string] $Rel, [string] $LeftPath, [string] $RightPath, [long] $LeftLength, [long] $RightLength, $NormalizeEntries = @()) + + $ext = [System.IO.Path]::GetExtension($Rel).ToLowerInvariant() + try { + if ($ext -in $script:ZipExtensions) { return Compare-ZipFile -LeftPath $LeftPath -RightPath $RightPath -NormalizeEntries $NormalizeEntries } + if ($ext -in $script:PeExtensions) { return Compare-PeFile -LeftPath $LeftPath -RightPath $RightPath -LeftLength $LeftLength -RightLength $RightLength } + if ($ext -in $script:TextExtensions) { return Compare-TextFile -LeftPath $LeftPath -RightPath $RightPath } + } + catch { return [pscustomobject]@{ Kind = 'error'; Error = $_.Exception.Message } } + return [pscustomobject]@{ Kind = 'binary' } +} + +# --------------------------------------------------------------------------------------------- +# Comparison +# --------------------------------------------------------------------------------------------- + +$allPaths = [System.Collections.Generic.HashSet[string]]::new([string[]]@($baseIndex.Keys)) +$allPaths.UnionWith([string[]]@($candIndex.Keys)) + +$results = New-Object System.Collections.Generic.List[object] +$comparedCount = 0 + +# Ignored paths are the one class of exclusion that leaves no trace in the diff lists, so they are +# tallied per rule and reported. Without this, an Ignore pattern that quietly grew to cover a real +# build output would be invisible in every run. +$ignoredByRule = [ordered]@{} +foreach ($rule in $rules) { + if ($rule.disposition -eq 'Ignore') { + $ignoredByRule[$rule.pattern] = [pscustomobject]@{ + Pattern = $rule.pattern + Reason = $rule.reason + Count = 0 + OnlyInBaseline = 0 + OnlyInCandidate = 0 + Differing = 0 + Samples = (New-Object System.Collections.Generic.List[string]) + } + } +} + +foreach ($rel in ($allPaths | Sort-Object)) { + $disp = Get-Disposition -RelativePath $rel + if ($disp.Disposition -eq 'Ignore') { + $bucket = $ignoredByRule[$disp.Pattern] + if ($bucket) { + $bucket.Count++ + $inB = $baseIndex.ContainsKey($rel) + $inC = $candIndex.ContainsKey($rel) + if ($inB -and -not $inC) { $bucket.OnlyInBaseline++ } + elseif ($inC -and -not $inB) { $bucket.OnlyInCandidate++ } + elseif ($baseIndex[$rel].Hash -ne $candIndex[$rel].Hash) { $bucket.Differing++ } + if ($bucket.Samples.Count -lt 25) { $bucket.Samples.Add($rel) } + } + continue + } + $comparedCount++ + + $inBase = $baseIndex.ContainsKey($rel) + $inCand = $candIndex.ContainsKey($rel) + + if ($inBase -and $inCand) { + if ($baseIndex[$rel].Hash -eq $candIndex[$rel].Hash) { continue } + $status = 'ContentDiffers' + } + elseif ($inBase) { $status = 'OnlyInBaseline' } + else { $status = 'OnlyInCandidate' } + + $results.Add([pscustomobject]@{ + Rel = $rel + Status = $status + Disposition = $disp.Disposition + Reason = $disp.Reason + Pattern = $disp.Pattern + NormalizeEntries = $disp.NormalizeEntries + BaseLength = $(if ($inBase) { $baseIndex[$rel].Length } else { $null }) + CandLength = $(if ($inCand) { $candIndex[$rel].Length } else { $null }) + Detail = $null + }) +} + +Write-Host ("[{0}] {1} paths excluded by Ignore rules: {2}" -f $Label, (($ignoredByRule.Values | Measure-Object -Property Count -Sum).Sum), ` + (($ignoredByRule.Values | ForEach-Object { "$($_.Pattern)=$($_.Count)" }) -join ', ')) +Write-Host "[$Label] triaging $($results.Count) differing paths ..." +$sw = [System.Diagnostics.Stopwatch]::StartNew() + +foreach ($r in $results) { + if ($r.Status -ne 'ContentDiffers') { continue } + $r.Detail = Get-DiffDetail -Rel $r.Rel -LeftPath $baseIndex[$r.Rel].FullPath -RightPath $candIndex[$r.Rel].FullPath -LeftLength $r.BaseLength -RightLength $r.CandLength -NormalizeEntries $r.NormalizeEntries + + if ($r.Disposition -eq 'ComparePayload') { + if ($r.Detail.Kind -eq 'zip' -and $r.Detail.PayloadIdentical) { + $r.Disposition = 'Informational' + $r.Reason = "Zip payload is byte-identical$(if ($r.NormalizeEntries.Count -gt 0) { ' after the documented per-entry normalization' }); only the container's per-entry metadata differs. Rule reason: $($r.Reason)" + } + else { + $r.Disposition = 'Compare' + } + } + elseif ($r.Disposition -eq 'Compare' -and $r.Detail.Kind -eq 'pe' -and $r.Detail.MvidAndStampOnly) { + $r.Disposition = 'Informational' + $r.Reason = 'Only the 16-byte assembly MVID and the PE TimeDateStamp/CheckSum differ: this assembly is compiled with /deterministic- so its MVID is a random GUID. Emitted IL, metadata and resources are byte-identical.' + } +} + +$sw.Stop() +Write-Host ("[{0}] triage completed in {1:n1}s" -f $Label, $sw.Elapsed.TotalSeconds) + +$failures = @($results | Where-Object { $_.Disposition -ne 'Informational' }) +$informational = @($results | Where-Object { $_.Disposition -eq 'Informational' }) + +$ignoredRules = @($ignoredByRule.Values) +$ignoredCount = 0 +foreach ($b in $ignoredRules) { $ignoredCount += $b.Count } + +$summary = [pscustomobject]@{ + Label = $Label + BaselineDir = $BaselineDir + CandidateDir = $CandidateDir + BaselineFileCount = $baseIndex.Count + CandidateFileCount = $candIndex.Count + ComparedPathCount = $comparedCount + IgnoredPathCount = $ignoredCount + IdenticalPathCount = $comparedCount - $results.Count + FailingDiffCount = $failures.Count + InformationalDiffCount = $informational.Count + Passed = ($failures.Count -eq 0) + GeneratedUtc = (Get-Date).ToUniversalTime().ToString('o') +} + +$report = [pscustomobject]@{ Summary = $summary; Failures = $failures; Informational = $informational; Ignored = $ignoredRules } +$jsonPath = Join-Path $OutputDir "artifact-compare.$Label.json" +$report | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $jsonPath -Encoding utf8 + +# --------------------------------------------------------------------------------------------- +# Markdown report +# --------------------------------------------------------------------------------------------- + +$md = New-Object System.Collections.Generic.List[string] +$md.Add("## Artifact comparison: ``$Label``") +$md.Add('') +$md.Add('| | |') +$md.Add('|---|---|') +$md.Add("| Baseline | ``$BaselineDir`` ($($baseIndex.Count) files) |") +$md.Add("| Candidate | ``$CandidateDir`` ($($candIndex.Count) files) |") +$md.Add("| Paths compared | $comparedCount |") +$md.Add("| Paths excluded by an Ignore rule | $ignoredCount |") +$md.Add("| Byte-identical | $($summary.IdenticalPathCount) |") +$md.Add("| **Unexpected differences** | **$($failures.Count)** |") +$md.Add("| Expected differences (documented) | $($informational.Count) |") +$md.Add("| Result | $(if ($summary.Passed) { 'PASS' } else { 'FAIL' }) |") +$md.Add('') + +function Add-DiffSection { + param($Items, [string] $Title) + + if ($Items.Count -eq 0) { return } + $md.Add("### $Title ($($Items.Count))") + $md.Add('') + foreach ($item in ($Items | Select-Object -First $MaxReportedDiffs)) { + $md.Add("- ``$($item.Rel)`` - **$($item.Status)**") + if ($item.Reason) { $md.Add(" - reason: $($item.Reason)") } + if ($item.Detail) { + switch ($item.Detail.Kind) { + 'pe' { $md.Add(" - PE: $($item.Detail.DifferingBytes) bytes in $($item.Detail.DifferingRuns) run(s): $($item.Detail.Regions)") } + 'zip' { + if ($item.Detail.PSObject.Properties.Name -contains 'Error') { $md.Add(" - zip: $($item.Detail.Error)") } + else { + $md.Add(" - zip: payloadIdentical=$($item.Detail.PayloadIdentical); contentDiff=$($item.Detail.EntriesWithContentDiff.Count); timestampOnly=$($item.Detail.EntriesWithTimestampDiff.Count); onlyBaseline=$($item.Detail.EntriesOnlyInBaseline.Count); onlyCandidate=$($item.Detail.EntriesOnlyInCandidate.Count)") + foreach ($e in ($item.Detail.EntriesWithContentDiff | Select-Object -First 10)) { $md.Add(" - entry content differs: ``$e``") } + foreach ($e in ($item.Detail.EntriesOnlyInBaseline | Select-Object -First 10)) { $md.Add(" - entry only in baseline: ``$e``") } + foreach ($e in ($item.Detail.EntriesOnlyInCandidate | Select-Object -First 10)) { $md.Add(" - entry only in candidate: ``$e``") } + } + } + 'text' { foreach ($d in $item.Detail.FirstDifferences) { $md.Add(" - line $($d.Line): baseline ``$($d.Baseline)`` / candidate ``$($d.Candidate)``") } } + default { $md.Add(" - $($item.Detail.Kind): baseline $($item.BaseLength) bytes, candidate $($item.CandLength) bytes") } + } + } + } + if ($Items.Count -gt $MaxReportedDiffs) { $md.Add("- _(... $($Items.Count - $MaxReportedDiffs) more; see the JSON report)_") } + $md.Add('') +} + +Add-DiffSection -Items $failures -Title 'Unexpected differences' +Add-DiffSection -Items $informational -Title 'Expected differences' + +# Everything the Ignore rules removed from the comparison, so the exclusions can be reviewed without +# having to re-run the build. 'differing' is how many of them would have been reported had the rule +# not existed - a rule with a large 'differing' count is doing real work and deserves scrutiny; a +# rule matching nothing is dead and should be deleted. +$md.Add('### Paths excluded from the comparison') +$md.Add('') +$md.Add('| Ignore rule | matched | differing | only in baseline | only in candidate |') +$md.Add('|---|---|---|---|---|') +foreach ($b in $ignoredRules) { + $md.Add("| ``$($b.Pattern)`` | $($b.Count) | $($b.Differing) | $($b.OnlyInBaseline) | $($b.OnlyInCandidate) |") +} +$md.Add('') +foreach ($b in $ignoredRules) { + if ($b.Count -eq 0) { + $md.Add("- ``$($b.Pattern)`` matched nothing in this build.") + continue + } + $md.Add("- ``$($b.Pattern)`` - $($b.Reason)") + foreach ($s in $b.Samples) { $md.Add(" - ``$s``") } + if ($b.Count -gt $b.Samples.Count) { $md.Add(" - _(... $($b.Count - $b.Samples.Count) more; see the JSON report)_") } +} +$md.Add('') + +$mdPath = Join-Path $OutputDir "artifact-compare.$Label.md" +($md -join [Environment]::NewLine) | Set-Content -LiteralPath $mdPath -Encoding utf8 + +Write-Host '' +Write-Host "[$Label] $comparedCount paths compared: $($summary.IdenticalPathCount) identical, $($failures.Count) unexpected, $($informational.Count) expected" +Write-Host "[$Label] reports: $jsonPath" +Write-Host "[$Label] $mdPath" + +if ($failures.Count -gt 0) { + Write-Host '' + Write-Host "[$Label] FAIL - unexpected differences (first 25):" + foreach ($f in ($failures | Select-Object -First 25)) { Write-Host " $($f.Status.PadRight(16)) $($f.Rel)" } + exit 1 +} + +Write-Host "[$Label] PASS" +exit 0 diff --git a/scripts/mt-equivalence/Compare-Binlogs.ps1 b/scripts/mt-equivalence/Compare-Binlogs.ps1 new file mode 100644 index 00000000000..c6ca3f6f2d1 --- /dev/null +++ b/scripts/mt-equivalence/Compare-Binlogs.ps1 @@ -0,0 +1,505 @@ +<# +.SYNOPSIS + Functional comparison of two MSBuild binary logs produced by otherwise-identical builds. + +.DESCRIPTION + Binary logs can never be byte-identical across two runs (they embed wall-clock timestamps, + durations, node ids and event ordering), so this script compares them *functionally*: + + 1. MT evidence - reads the "Command line arguments" recorded in each binlog and asserts that + the candidate build really did run with -mt / --multithreaded and that the + baseline did not. Without this the whole comparison would be vacuous. + 2. Structure - reads the raw event stream out of both binlogs and requires that every target, + every (project, target) pair, every task and every project executed exactly + the same number of times. This is the strongest of the log checks: it is taken + from the events themselves, so unlike the text tiers below it is unaffected by + verbosity, by node interleaving, or by the console logger only emitting a + target header when the target happens to produce output. + 3. Diagnostics - replays both logs at quiet verbosity (errors and warnings only) and requires + an identical multiset of diagnostics. + 4. Functional - replays both at normal verbosity, normalizes away scheduling-dependent noise + using LogNormalizationRules.json, and requires an identical multiset of lines. + 5. Deep (opt-in)- replays both at diagnostic verbosity and compares the multiset of executed + targets and tasks. Superseded by tier 2 for target/task coverage; retained + because it also compares which assembly each task was bound to. + +.PARAMETER MSBuildPath + Path to the executable used to replay the binlogs. Must be new enough to read the binlog format + version written by the build under test - use the bootstrapped MSBuild from the build itself. + +.PARAMETER MSBuildCommand + Sub-command to pass to MSBuildPath ('msbuild' for dotnet.exe, '' for MSBuild.exe). +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $BaselineBinlog, + [Parameter(Mandatory = $true)][string] $CandidateBinlog, + [Parameter(Mandatory = $true)][string] $OutputDir, + [Parameter(Mandatory = $true)][string] $MSBuildPath, + [string] $MSBuildCommand = 'msbuild', + [string] $Label = 'log', + [string] $RulesFile, + [switch] $DeepCompare, + [switch] $ExpectCandidateMultiThreaded, + [int] $MaxReportedLines = 60 +) + +Set-StrictMode -Version 3.0 +$ErrorActionPreference = 'Stop' + +if (-not $RulesFile) { $RulesFile = Join-Path $PSScriptRoot 'LogNormalizationRules.json' } +foreach ($p in @($BaselineBinlog, $CandidateBinlog, $MSBuildPath, $RulesFile)) { + if (-not (Test-Path -LiteralPath $p)) { throw "Not found: $p" } +} +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +# Scratch for the replayed text logs. These run to tens of megabytes each and are fully regenerable +# from the binlogs (which the pipeline publishes), so they are kept out of the report directory and +# deleted at the end rather than shipped in the artifact. +$workDir = Join-Path ([System.IO.Path]::GetTempPath()) "mt-equivalence-logwork-$Label-$PID" +if (Test-Path -LiteralPath $workDir) { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } +New-Item -ItemType Directory -Force -Path $workDir | Out-Null + +# --------------------------------------------------------------------------------------------- +# MT evidence: the MSBuild command line is recorded near the head of every binlog. +# --------------------------------------------------------------------------------------------- + +function Get-BinlogCommandLine { + param([string] $Path) + + $fs = [System.IO.File]::OpenRead($Path) + try { + $gz = [System.IO.Compression.GZipStream]::new($fs, [System.IO.Compression.CompressionMode]::Decompress) + try { + # The header, environment block and command line all live in the first few KB. + $buffer = New-Object byte[] (256 * 1024) + $read = 0 + $total = 0 + while ($total -lt $buffer.Length) { + $read = $gz.Read($buffer, $total, $buffer.Length - $total) + if ($read -le 0) { break } + $total += $read + } + $text = [System.Text.Encoding]::UTF8.GetString($buffer, 0, $total) + } + finally { $gz.Dispose() } + } + finally { $fs.Dispose() } + + # The record is terminated by a control character, and the value itself may be wrapped in quotes + # (desktop MSBuild quotes its own exe path because "C:\Program Files\..." contains spaces, which + # would defeat a "[^"]*" match), so consume printable characters and trim the wrapper afterwards. + $m = [regex]::Match($text, 'Command line arguments = (?[^\x00-\x1F]*)') + if (-not $m.Success) { return $null } + + $commandLine = $m.Groups['cmd'].Value.Trim() + if ($commandLine.Length -ge 2 -and $commandLine.StartsWith('"') -and $commandLine.EndsWith('"')) { + $commandLine = $commandLine.Substring(1, $commandLine.Length - 2) + } + return $commandLine +} + +function Test-CommandLineMultiThreaded { + param([string] $CommandLine) + + if (-not $CommandLine) { return $null } + # Allow a quote as a delimiter: desktop MSBuild command lines quote individual arguments. + return [bool][regex]::IsMatch($CommandLine, '(?<=^|[\s"])[-/](mt|multithreaded)(?=[\s"]|$)', 'IgnoreCase') +} + +$baselineCmd = Get-BinlogCommandLine -Path $BaselineBinlog +$candidateCmd = Get-BinlogCommandLine -Path $CandidateBinlog +$baselineMt = Test-CommandLineMultiThreaded -CommandLine $baselineCmd +$candidateMt = Test-CommandLineMultiThreaded -CommandLine $candidateCmd + +Write-Host "[$Label] baseline command line: $baselineCmd" +Write-Host "[$Label] candidate command line: $candidateCmd" +Write-Host "[$Label] baseline -mt = $baselineMt / candidate -mt = $candidateMt" + +$evidenceProblems = New-Object System.Collections.Generic.List[string] +if ($ExpectCandidateMultiThreaded) { + if ($null -eq $candidateCmd) { $evidenceProblems.Add('Could not read the command line from the candidate binlog.') } + elseif (-not $candidateMt) { $evidenceProblems.Add("Candidate build did not run with -mt. Command line: $candidateCmd") } + if ($null -eq $baselineCmd) { $evidenceProblems.Add('Could not read the command line from the baseline binlog.') } + elseif ($baselineMt) { $evidenceProblems.Add("Baseline build unexpectedly ran with -mt. Command line: $baselineCmd") } +} +else { + if ($candidateMt -or $baselineMt) { $evidenceProblems.Add('A control comparison must not have -mt on either side.') } +} + +# --------------------------------------------------------------------------------------------- +# Replay +# --------------------------------------------------------------------------------------------- + +function Invoke-Replay { + param([string] $Binlog, [string] $Verbosity, [string] $OutFile) + + $flp = "-flp:v=$Verbosity;logfile=$OutFile;ShowTimestamp=false;ShowEventId=false" + $args = @() + if ($MSBuildCommand) { $args += $MSBuildCommand } + $args += @($Binlog, '-noconlog', '-nologo', $flp) + + # A replay of a failed build exits non-zero; that is expected and not an error here. + & $MSBuildPath @args *> (Join-Path $workDir 'replay-stdout.txt') + if (-not (Test-Path -LiteralPath $OutFile)) { + throw "Replay of '$Binlog' at verbosity '$Verbosity' produced no log. See $(Join-Path $workDir 'replay-stdout.txt')." + } +} + +$rules = Get-Content -Raw -LiteralPath $RulesFile | ConvertFrom-Json +Add-Type -TypeDefinition (Get-Content -Raw -LiteralPath (Join-Path $PSScriptRoot 'MtCompareNative.cs')) -ErrorAction Stop + +$opts = [System.Text.RegularExpressions.RegexOptions]::Compiled +$prefixRegexes = [System.Text.RegularExpressions.Regex[]]@($rules.prefix | ForEach-Object { [regex]::new($_.pattern, $opts) }) +$dropRegexes = [System.Text.RegularExpressions.Regex[]]@($rules.drop | ForEach-Object { [regex]::new($_.pattern, $opts) }) +$replaceFrom = [System.Text.RegularExpressions.Regex[]]@($rules.replace | ForEach-Object { [regex]::new($_.pattern, $opts) }) +$replaceTo = [string[]]@($rules.replace | ForEach-Object { $_.replacement }) +$setOnlyRegexes = [System.Text.RegularExpressions.Regex[]]@($rules.setOnly | ForEach-Object { [regex]::new($_.pattern, $opts) }) +$knownMtOnly = @($rules.knownMtOnly | ForEach-Object { [pscustomobject]@{ Regex = [regex]::new($_.pattern, $opts); Reason = $_.reason } }) + +# Extractors for the deep (diagnostic verbosity) coverage comparison. The *set* of targets, tasks and +# task->assembly bindings must match; their counts and ordering are scheduling artifacts. +$deepExtractorNames = [string[]]@('targets', 'tasks', 'taskAssemblies') +# Target *headers* in a text log are emitted only when a target produces output in a contiguous block, +# so which target names appear is partly a logging/interleaving artifact - the non-mt control run shows +# the same instability. Target coverage is therefore reported but not treated as a failure; the task and +# task->assembly sets are stable in the control and are enforced. +$deepFailingExtractors = [string[]]@('tasks', 'taskAssemblies') +$deepPrefix = '^(?:\d{1,2}:\d{2}:\d{2}\.\d{3}\s+)?(?:\d+(?::\d+)?>)?\s*' +$deepExtractors = [System.Text.RegularExpressions.Regex[]]@( + [regex]::new($deepPrefix + '(?[A-Za-z_][A-Za-z0-9_.]*):\s*\(TargetId:\d+\)$', $opts) + [regex]::new($deepPrefix + 'Task "(?[^"]+)"\s*\(TaskId:\d+\)$', $opts) + [regex]::new($deepPrefix + 'Using "(?[^"]+" task from assembly "[^"]+)"\.$', $opts) +) + +function Get-NormalizedLineCounts { + param([string] $Path) + + $seen = [System.Collections.Generic.HashSet[string]]::new() + return [MtCompareNative]::NormalizedLineCounts($Path, $prefixRegexes, $dropRegexes, $replaceFrom, $replaceTo, $setOnlyRegexes, $seen) +} + +$sections = [ordered]@{} +$knownMtDifferences = New-Object System.Collections.Generic.List[object] + +# --------------------------------------------------------------------------------------------- +# Structural comparison, taken from the binlog event stream rather than from replayed text. +# +# The text tiers below have to tolerate a lot of noise, and two of the normalization rules +# (the target-header setOnly rule and its knownMtOnly counterpart) deliberately stop comparing +# how many times a target header appears. That is only safe if something else pins down what +# actually ran, which is what this does: exact execution counts for every target, every +# (project, target) pair, every task and every project, straight from the events. +# --------------------------------------------------------------------------------------------- + +# The extraction runs out of process, compiled and executed by the SDK itself: see +# Get-BinlogStructure.ps1 for why it cannot run inside this PowerShell session. +$structureScript = Join-Path $PSScriptRoot 'Get-BinlogStructure.ps1' +$structureHost = (Get-Process -Id $PID).Path +$structureErrors = New-Object System.Collections.Generic.List[string] +$structureAnnounced = $false + +function Get-BinlogStructuredCounts { + param([string] $Binlog) + + $outFile = Join-Path $workDir ("structure-" + [System.IO.Path]::GetRandomFileName() + '.json') + $stderr = & $structureHost -NoProfile -NonInteractive -File $structureScript ` + -Binlog $Binlog -OutFile $outFile -WorkDir $workDir 2>&1 + if ($LASTEXITCODE -eq 0 -and (Test-Path -LiteralPath $outFile)) { + if (-not $script:structureAnnounced) { + $script:structureAnnounced = $true + Write-Host "[$Label] structural comparison reading the binlog event stream directly" + } + $json = Get-Content -Raw -LiteralPath $outFile | ConvertFrom-Json + Remove-Item -LiteralPath $outFile -Force -ErrorAction SilentlyContinue + return $json + } + $structureErrors.Add("$([System.IO.Path]::GetFileName($Binlog)): $(($stderr | Out-String).Trim())") + return $null +} + +function ConvertTo-CountMap { + param($Object) + + $map = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::Ordinal) + if ($Object) { + foreach ($p in $Object.PSObject.Properties) { $map[$p.Name] = [int]$p.Value } + } + return $map +} + +function Compare-LineCounts { + param($Baseline, $Candidate) + + $onlyBaseline = New-Object System.Collections.Generic.List[object] + $onlyCandidate = New-Object System.Collections.Generic.List[object] + + foreach ($kv in $Baseline.GetEnumerator()) { + $candCount = 0 + [void]$Candidate.TryGetValue($kv.Key, [ref] $candCount) + if ($candCount -lt $kv.Value) { + $onlyBaseline.Add([pscustomobject]@{ Line = $kv.Key; Baseline = $kv.Value; Candidate = $candCount }) + } + } + foreach ($kv in $Candidate.GetEnumerator()) { + $baseCount = 0 + [void]$Baseline.TryGetValue($kv.Key, [ref] $baseCount) + if ($baseCount -lt $kv.Value) { + $onlyCandidate.Add([pscustomobject]@{ Line = $kv.Key; Baseline = $baseCount; Candidate = $kv.Value }) + } + } + + [pscustomobject]@{ + BaselineDistinct = $Baseline.Count + CandidateDistinct = $Candidate.Count + BaselineTotal = ($Baseline.Values | Measure-Object -Sum).Sum + CandidateTotal = ($Candidate.Values | Measure-Object -Sum).Sum + MissingInCandidate = @($onlyBaseline | Sort-Object Line) + ExtraInCandidate = @($onlyCandidate | Sort-Object Line) + } +} + +function Split-KnownMtDifferences { + param($Comparison) + + if (-not $ExpectCandidateMultiThreaded -or $knownMtOnly.Count -eq 0) { return $Comparison } + + $remaining = New-Object System.Collections.Generic.List[object] + foreach ($d in $Comparison.ExtraInCandidate) { + $matched = $null + foreach ($k in $knownMtOnly) { + if ($k.Regex.IsMatch([string]$d.Line)) { $matched = $k; break } + } + if ($matched) { + $knownMtDifferences.Add([pscustomobject]@{ Line = $d.Line; Baseline = $d.Baseline; Candidate = $d.Candidate; Reason = $matched.Reason }) + } + else { + $remaining.Add($d) + } + } + + return [pscustomobject]@{ + BaselineDistinct = $Comparison.BaselineDistinct + CandidateDistinct = $Comparison.CandidateDistinct + BaselineTotal = $Comparison.BaselineTotal + CandidateTotal = $Comparison.CandidateTotal + MissingInCandidate = @($Comparison.MissingInCandidate) + ExtraInCandidate = $remaining.ToArray() + } +} + +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$baseStruct = Get-BinlogStructuredCounts -Binlog $BaselineBinlog +$candStruct = Get-BinlogStructuredCounts -Binlog $CandidateBinlog +$sw.Stop() + +if ($null -eq $baseStruct -or $null -eq $candStruct) { + # This is the control that lets the text tiers stop counting target headers, so failing to run it + # has to be loud rather than silent. + $evidenceProblems.Add("Could not read the binlog event stream, so the structural comparison did not run. Attempts: $($structureErrors -join ' | ')") +} +else { + foreach ($dim in @('Targets', 'TargetsByProject', 'Tasks', 'Projects', 'Diagnostics')) { + $name = 'structure/' + $dim.Substring(0, 1).ToLowerInvariant() + $dim.Substring(1) + # Deliberately not passed through Split-KnownMtDifferences: this tier exists precisely to be + # the check that nothing is excused, so a known bug must not be able to suppress it either. + $cmp = Compare-LineCounts -Baseline (ConvertTo-CountMap $baseStruct.$dim) -Candidate (ConvertTo-CountMap $candStruct.$dim) + $sections[$name] = $cmp + Write-Host ("[{0}] {1}: {2} distinct / {3} executions baseline, {4} / {5} candidate; {6} missing, {7} extra" -f ` + $Label, $name, $cmp.BaselineDistinct, $cmp.BaselineTotal, $cmp.CandidateDistinct, $cmp.CandidateTotal, ` + $cmp.MissingInCandidate.Count, $cmp.ExtraInCandidate.Count) + } + Write-Host ("[{0}] structural extraction took {1:n1}s" -f $Label, $sw.Elapsed.TotalSeconds) +} + +foreach ($tier in @( + @{ Name = 'diagnostics'; Verbosity = 'q'; Enabled = $true }, + @{ Name = 'functional'; Verbosity = 'n'; Enabled = $true })) { + + if (-not $tier.Enabled) { continue } + + $v = $tier.Verbosity + $baseLog = Join-Path $workDir "baseline.$v.log" + $candLog = Join-Path $workDir "candidate.$v.log" + + Write-Host "[$Label] replaying at verbosity '$v'..." + Invoke-Replay -Binlog $BaselineBinlog -Verbosity $v -OutFile $baseLog + Invoke-Replay -Binlog $CandidateBinlog -Verbosity $v -OutFile $candLog + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $baseCounts = Get-NormalizedLineCounts -Path $baseLog + $candCounts = Get-NormalizedLineCounts -Path $candLog + $sw.Stop() + + $cmp = Split-KnownMtDifferences -Comparison (Compare-LineCounts -Baseline $baseCounts -Candidate $candCounts) + $sections[$tier.Name] = $cmp + Write-Host ("[{0}] {1}: {2} distinct baseline lines / {3} candidate; {4} missing, {5} extra (normalized in {6:n1}s)" -f ` + $Label, $tier.Name, $cmp.BaselineDistinct, $cmp.CandidateDistinct, $cmp.MissingInCandidate.Count, $cmp.ExtraInCandidate.Count, $sw.Elapsed.TotalSeconds) +} + +# --------------------------------------------------------------------------------------------- +# Deep coverage comparison: which targets ran, which tasks ran, and which assembly each task was +# bound to. Compared as sets, from a diagnostic-verbosity replay. +# --------------------------------------------------------------------------------------------- + +$coverage = @{} +$coverageNames = New-Object System.Collections.Generic.List[string] +if ($DeepCompare) { + $baseLog = Join-Path $workDir 'baseline.diag.log' + $candLog = Join-Path $workDir 'candidate.diag.log' + Write-Host "[$Label] replaying at verbosity 'diag' for target/task coverage..." + Invoke-Replay -Binlog $BaselineBinlog -Verbosity 'diag' -OutFile $baseLog + Invoke-Replay -Binlog $CandidateBinlog -Verbosity 'diag' -OutFile $candLog + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $baseSets = [MtCompareNative]::ExtractSets($baseLog, $deepExtractorNames, $deepExtractors) + $candSets = [MtCompareNative]::ExtractSets($candLog, $deepExtractorNames, $deepExtractors) + $sw.Stop() + Remove-Item -LiteralPath $baseLog, $candLog -Force -ErrorAction SilentlyContinue + + foreach ($name in $deepExtractorNames) { + $missing = @($baseSets[$name] | Where-Object { -not $candSets[$name].Contains($_) } | Sort-Object) + $extra = @($candSets[$name] | Where-Object { -not $baseSets[$name].Contains($_) } | Sort-Object) + $coverageNames.Add($name) + $coverage[$name] = [pscustomobject]@{ + BaselineCount = $baseSets[$name].Count + CandidateCount = $candSets[$name].Count + MissingInCandidate = $missing + ExtraInCandidate = $extra + } + Write-Host ("[{0}] coverage/{1}: baseline {2}, candidate {3}; {4} missing, {5} extra" -f ` + $Label, $name, $baseSets[$name].Count, $candSets[$name].Count, $missing.Count, $extra.Count) + } + Write-Host ("[{0}] coverage extraction took {1:n1}s" -f $Label, $sw.Elapsed.TotalSeconds) +} + +# --------------------------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------------------------- + +$diagnosticsMismatch = $sections['diagnostics'].MissingInCandidate.Count + $sections['diagnostics'].ExtraInCandidate.Count +$functionalMismatch = $sections['functional'].MissingInCandidate.Count + $sections['functional'].ExtraInCandidate.Count +$structureMismatch = 0 +$structureSections = @($sections.Keys | Where-Object { $_ -like 'structure/*' }) +foreach ($name in $structureSections) { + $structureMismatch += $sections[$name].MissingInCandidate.Count + $sections[$name].ExtraInCandidate.Count +} +$coverageMismatch = 0 +foreach ($name in $coverageNames) { + if ($name -notin $deepFailingExtractors) { continue } + $coverageMismatch += $coverage[$name].MissingInCandidate.Count + $coverage[$name].ExtraInCandidate.Count +} + +$passed = ($evidenceProblems.Count -eq 0) -and ($diagnosticsMismatch -eq 0) -and ($functionalMismatch -eq 0) -and ($structureMismatch -eq 0) -and ($coverageMismatch -eq 0) + +$report = [pscustomobject]@{ + Label = $Label + BaselineBinlog = (Resolve-Path $BaselineBinlog).Path + CandidateBinlog = (Resolve-Path $CandidateBinlog).Path + BaselineCommandLine = $baselineCmd + CandidateCommandLine = $candidateCmd + BaselineMultiThreaded = $baselineMt + CandidateMultiThreaded = $candidateMt + EvidenceProblems = @($evidenceProblems) + Sections = $sections + Coverage = $coverage + KnownMtDifferences = $knownMtDifferences.ToArray() + Passed = $passed + GeneratedUtc = (Get-Date).ToUniversalTime().ToString('o') +} + +$jsonPath = Join-Path $OutputDir "log-compare.$Label.json" +$report | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $jsonPath -Encoding utf8 + +$knownMtLineTotal = 0 +foreach ($k in $knownMtDifferences) { $knownMtLineTotal += $k.Candidate } + +$md = New-Object System.Collections.Generic.List[string] +$md.Add("## Log comparison: ``$Label``") +$md.Add('') +$md.Add('| | |') +$md.Add('|---|---|') +$md.Add("| Baseline binlog | ``$($report.BaselineBinlog)`` |") +$md.Add("| Candidate binlog | ``$($report.CandidateBinlog)`` |") +$md.Add("| Baseline ran with -mt | $baselineMt |") +$md.Add("| Candidate ran with -mt | $candidateMt |") +$md.Add("| Diagnostics (errors/warnings) differences | $diagnosticsMismatch |") +$md.Add("| Functional (normal verbosity) differences | $functionalMismatch |") +if ($structureSections.Count -gt 0) { + $md.Add("| Structural differences (target/task/project execution counts) | $structureMismatch |") +} +if ($coverageNames.Count -gt 0) { $md.Add("| Target/task coverage differences | $coverageMismatch |") } +$md.Add("| Known, already-filed -mt differences | $($knownMtDifferences.Count) distinct / $knownMtLineTotal lines |") +$md.Add("| Result | $(if ($passed) { 'PASS' } else { 'FAIL' }) |") +$md.Add('') + +if ($evidenceProblems.Count -gt 0) { + $md.Add('### -mt evidence problems') + $md.Add('') + foreach ($p in $evidenceProblems) { $md.Add("- $p") } + $md.Add('') +} + +foreach ($name in $sections.Keys) { + $s = $sections[$name] + if (($s.MissingInCandidate.Count + $s.ExtraInCandidate.Count) -eq 0) { continue } + $md.Add("### $name differences") + $md.Add('') + if ($s.MissingInCandidate.Count -gt 0) { + $md.Add("Present in baseline but missing (or fewer) in the candidate ($($s.MissingInCandidate.Count)):") + $md.Add('') + foreach ($d in ($s.MissingInCandidate | Select-Object -First $MaxReportedLines)) { + $md.Add("- ``$($d.Line)`` (baseline x$($d.Baseline), candidate x$($d.Candidate))") + } + $md.Add('') + } + if ($s.ExtraInCandidate.Count -gt 0) { + $md.Add("Present in the candidate but missing (or fewer) in baseline ($($s.ExtraInCandidate.Count)):") + $md.Add('') + foreach ($d in ($s.ExtraInCandidate | Select-Object -First $MaxReportedLines)) { + $md.Add("- ``$($d.Line)`` (baseline x$($d.Baseline), candidate x$($d.Candidate))") + } + $md.Add('') + } +} + +foreach ($name in $coverageNames) { + $c = $coverage[$name] + if (($c.MissingInCandidate.Count + $c.ExtraInCandidate.Count) -eq 0) { continue } + $enforced = $name -in $deepFailingExtractors + $md.Add("### coverage/$name differences$(if (-not $enforced) { ' (informational)' })") + $md.Add('') + if (-not $enforced) { + $md.Add('_Target headers are emitted by the text logger only when a target produces output in a contiguous block, so the set of names is partly an interleaving artifact. The non-mt control run shows the same instability, so this signal is reported but not enforced._') + $md.Add('') + } + foreach ($d in ($c.MissingInCandidate | Select-Object -First $MaxReportedLines)) { $md.Add("- only in baseline: ``$d``") } + foreach ($d in ($c.ExtraInCandidate | Select-Object -First $MaxReportedLines)) { $md.Add("- only in candidate: ``$d``") } + $md.Add('') +} + +if ($knownMtDifferences.Count -gt 0) { + $md.Add('### Known -mt-only log differences (already filed; not treated as failures)') + $md.Add('') + foreach ($d in ($knownMtDifferences | Sort-Object -Property @{Expression = { -$_.Candidate } } | Select-Object -First $MaxReportedLines)) { + $md.Add("- x$($d.Candidate) ``$($d.Line)``") + $md.Add(" - $($d.Reason)") + } + $md.Add('') +} + +$mdPath = Join-Path $OutputDir "log-compare.$Label.md" +($md -join [Environment]::NewLine) | Set-Content -LiteralPath $mdPath -Encoding utf8 + +Write-Host "[$Label] reports: $jsonPath" +Write-Host "[$Label] $mdPath" + +# Drop the replayed text logs: they are large and regenerable from the binlogs. +Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue + +if (-not $passed) { + Write-Host "[$Label] FAIL" + foreach ($p in $evidenceProblems) { Write-Host " evidence: $p" } + exit 1 +} +exit 0 + diff --git a/scripts/mt-equivalence/Get-BinlogStructure.ps1 b/scripts/mt-equivalence/Get-BinlogStructure.ps1 new file mode 100644 index 00000000000..d9aa060bf14 --- /dev/null +++ b/scripts/mt-equivalence/Get-BinlogStructure.ps1 @@ -0,0 +1,96 @@ +<# +.SYNOPSIS + Writes the target/task/project execution counts of a binary log to a JSON file. + +.DESCRIPTION + Compiles BinlogStructure.cs with the SDK's own Roslyn and runs it on the SDK's own runtime, so the + program, the compiler and Microsoft.Build all agree on the target framework. + + This deliberately avoids loading Microsoft.Build into the calling PowerShell. The build agents run + an older pwsh than a typical dev box (7.4, on .NET 8) while the SDK's Microsoft.Build targets a + newer framework, so Add-Type there fails with CS1705 - it compiles locally and not on the agent. + It also avoids reading the counts out of a diagnostic-verbosity text replay: engine-assigned + TargetIds are not unique across projects, and a measurable share of TaskStarted events never reach + the text at all, so the text cannot give exact counts. + + The compile is done once per work directory and reused. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $Binlog, + [Parameter(Mandatory = $true)][string] $OutFile, + [Parameter(Mandatory = $true)][string] $WorkDir +) + +Set-StrictMode -Version 3.0 +$ErrorActionPreference = 'Stop' + +function Find-DotnetRoot { + $repoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + $onPath = Get-Command dotnet -ErrorAction SilentlyContinue + foreach ($root in @( + (Join-Path $repoRoot '.dotnet'), + $env:DOTNET_INSTALL_DIR, + $env:DOTNET_ROOT, + $(if ($onPath) { Split-Path -Parent $onPath.Source }))) { + + if (-not $root) { continue } + foreach ($name in 'dotnet.exe', 'dotnet') { + $exe = Join-Path $root $name + if (Test-Path -LiteralPath $exe) { return [pscustomobject]@{ Root = $root; Exe = $exe } } + } + } + throw 'No dotnet installation found (looked in \.dotnet, DOTNET_INSTALL_DIR, DOTNET_ROOT and PATH).' +} + +$dotnet = Find-DotnetRoot + +# Newest SDK that actually carries Microsoft.Build; that is the assembly set the program binds to. +$sdk = Get-ChildItem -LiteralPath (Join-Path $dotnet.Root 'sdk') -Directory -ErrorAction SilentlyContinue | + Sort-Object Name -Descending | + Where-Object { (Test-Path -LiteralPath (Join-Path $_.FullName 'Microsoft.Build.dll')) -and (Test-Path -LiteralPath (Join-Path $_.FullName 'Roslyn\bincore\csc.dll')) } | + Select-Object -First 1 +if (-not $sdk) { throw "No SDK under $(Join-Path $dotnet.Root 'sdk') contains both Microsoft.Build.dll and Roslyn." } + +# Run on the newest shared framework available: it has to be at least what Microsoft.Build targets. +$framework = Get-ChildItem -LiteralPath (Join-Path $dotnet.Root 'shared\Microsoft.NETCore.App') -Directory -ErrorAction SilentlyContinue | + Sort-Object Name -Descending | Select-Object -First 1 +if (-not $framework) { throw "No Microsoft.NETCore.App under $(Join-Path $dotnet.Root 'shared')." } + +$toolDir = Join-Path $WorkDir 'binlogstructure' +$toolDll = Join-Path $toolDir 'BinlogStructure.dll' +$source = Join-Path $PSScriptRoot 'BinlogStructure.cs' + +if ((-not (Test-Path -LiteralPath $toolDll)) -or ((Get-Item $toolDll).LastWriteTimeUtc -lt (Get-Item $source).LastWriteTimeUtc)) { + New-Item -ItemType Directory -Force -Path $toolDir | Out-Null + + # Reference the runtime's own assemblies rather than a targeting pack, which need not be + # installed. Only the managed ones can be referenced, hence the name filter. + $refs = New-Object System.Collections.Generic.List[string] + $refs.Add((Join-Path $sdk.FullName 'Microsoft.Build.dll')) + $refs.Add((Join-Path $sdk.FullName 'Microsoft.Build.Framework.dll')) + foreach ($dll in (Get-ChildItem -LiteralPath $framework.FullName -Filter '*.dll')) { + if ($dll.Name -like '*Native*') { continue } + if ($dll.Name -like 'System*.dll' -or $dll.Name -eq 'netstandard.dll') { $refs.Add($dll.FullName) } + } + + $cscArgs = @( + (Join-Path $sdk.FullName 'Roslyn\bincore\csc.dll'), + '/nologo', '/noconfig', '/nostdlib', '/target:exe', '/optimize+', '/nullable:enable', + "/out:$toolDll", $source) + ($refs | ForEach-Object { "/r:$_" }) + + $cscOutput = & $dotnet.Exe @cscArgs 2>&1 + if ($LASTEXITCODE -ne 0) { throw "compiling BinlogStructure.cs failed: $(($cscOutput | Out-String).Trim())" } + + # csc alone does not emit one, and 'dotnet exec' will not start a framework-dependent app without it. + @{ + runtimeOptions = @{ + tfm = 'net10.0' + framework = @{ name = 'Microsoft.NETCore.App'; version = $framework.Name } + rollForward = 'latestMajor' + } + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $toolDir 'BinlogStructure.runtimeconfig.json') -Encoding utf8 +} + +$runOutput = & $dotnet.Exe exec $toolDll $Binlog $OutFile $sdk.FullName 2>&1 +if ($LASTEXITCODE -ne 0) { throw "reading '$Binlog' failed: $(($runOutput | Out-String).Trim())" } diff --git a/scripts/mt-equivalence/LogNormalizationRules.json b/scripts/mt-equivalence/LogNormalizationRules.json new file mode 100644 index 00000000000..136094d68e3 --- /dev/null +++ b/scripts/mt-equivalence/LogNormalizationRules.json @@ -0,0 +1,160 @@ +{ + "//": "Normalization applied to replayed MSBuild text logs before the two runs are compared as line multisets. Rules are applied in order: 'prefix' strips the node/timestamp prefix, 'drop' removes a line entirely, 'replace' rewrites it, 'setOnly' marks a line as compared for presence rather than for count. Every entry documents why the raw text cannot be compared verbatim.", + + "//prefix": "Stripped from every line before any other rule runs. Whether the file logger emits a node id and timestamp depends on the engine used to replay the binlog (MSBuild.exe does not, 'dotnet msbuild' does), not on the build being tested, so this has to come off first: otherwise every '^'-anchored drop rule below silently stops matching and dropped noise reappears as reported differences.", + + "prefix": [ + { + "pattern": "^\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}\\s+\\d+(?::\\d+)?>", + "reason": "Leading wall-clock timestamp plus node/submission id." + }, + { + "pattern": "^\\s*\\d+(?::\\d+)?>", + "reason": "Node id prefix; -mt changes node assignment." + } + ], + + "//drop": "NOTE: drop patterns are matched after the 'prefix' rules above have run, but before any 'replace' rule. A line therefore still carries its indentation and its trailing '(TaskId:N)', so anchoring a drop pattern with '$' will usually not match.", + + "drop": [ + { + "pattern": "^\\s*(GET|OK|CACHE|NotFound|Retrying) https?://", + "reason": "NuGet HTTP traffic depends on the state of the machine-level package cache, not on the build." + }, + { + "pattern": "^\\s*(Restoring packages for |Restored |Installing |Installed |Acquiring lock for the installation of |Acquired lock for the installation of |Committing restore|Writing assets file to disk|Generating MSBuild file|Assets file has not changed|All projects are up-to-date for restore|Determining projects to restore|Persisting no-op dep graph|Reading project file|Restore completed in|Feeds used:|Package source mapping matches found|Total time taken)", + "reason": "NuGet restore progress varies with package-cache warmth and is not a property of the build engine." + }, + { + "pattern": "^\\s*Installed:\\s*(\\(TaskId:\\d+\\))?\\s*$", + "reason": "Header of NuGet's cold-restore 'Installed:' summary. Only the first of the three builds performs a real restore; the later two are no-ops, so this block appears in exactly one of any two builds being compared - including two identical non-mt builds." + }, + { + "pattern": "^\\s*\\d+ package\\(s\\) to \\S", + "reason": "Body of NuGet's cold-restore 'Installed:' summary (see above)." + }, + { + "pattern": "\\.gdn\\.sarif", + "reason": "Guardian SARIF analysis output injected by the 1ES SDL template. It is an analysis result rather than a build output, and it is not emitted consistently between two identical builds, so the Copy task lines that move it around vary between runs. The files themselves are ignored by ArtifactCompareRules.json for the same reason." + }, + { + "pattern": "X\\.509 certificate chain validation will use", + "reason": "NuGet signing-verification notice emitted only on a cold restore." + }, + { + "pattern": "^\\s*Attempting to create process '[^']*VBCSCompiler", + "reason": "Roslyn compiler-server startup. Whether VBCSCompiler must be started depends on whether a server from an earlier build on the same machine is still alive, so both the message and its count are machine-state dependent." + }, + { + "pattern": "^\\s*Successfully created process with process id \\d+", + "reason": "Roslyn compiler-server startup (see above); also contains a process id." + }, + { + "pattern": "^\\s*Setting DOTNET_[A-Za-z_]+ to '", + "reason": "Emitted by the Roslyn compiler-server client (DOTNET_ROOT, DOTNET_TieredCompilation, ...) on the same code path as the VBCSCompiler startup messages above, so the count tracks server startups rather than compilations. Observed to vary between two identical non-mt builds." + }, + { + "pattern": "^Build started ", + "reason": "Wall-clock start time." + }, + { + "pattern": "^Time Elapsed ", + "reason": "Wall-clock duration." + }, + { + "pattern": "^\\s*BinLogFilePath=", + "reason": "Binlog path differs between the baseline and candidate snapshots." + }, + { + "pattern": "^\\s*BinaryLogger wrote to: ", + "reason": "Binlog path differs between the baseline and candidate snapshots." + }, + { + "pattern": "^\\s*Logging verbosity is set to", + "reason": "Replay artifact, identical by construction." + }, + { + "pattern": "^\\s*DOTNET_CLI_TELEMETRY_SESSIONID", + "reason": "Per-process telemetry session id." + }, + { + "pattern": "^\\s*Creating directory \"", + "reason": "Parallel Copy/MakeDir invocations race to create the same directory: whoever loses the race finds it already present and logs nothing, so both the presence and the count of this message vary between two *identical* builds (observed in the non-mt control run). The resulting directory tree is verified byte-for-byte by the artifact comparison." + }, + { + "pattern": "\\bMSBuildNodeCount\\s*=", + "reason": "-mt turns worker nodes from processes into threads, so the reported node count legitimately differs." + } + ], + + "replace": [ + { + "pattern": "\\s*\\((TaskId|TargetId|ProjectId):\\d+\\)", + "replacement": "", + "reason": "Engine-assigned ids depend on scheduling order." + }, + { + "pattern": "^\\s*Project \"[^\"]+\" \\(\\d+(?::\\d+)?\\) is building \"(?[^\"]+)\"[^\\r\\n]*$", + "replacement": "PROJECT BUILT: ${built}", + "reason": "A shared dependency is really built by whichever requester reaches it first; every other requester is served from the results cache and logs nothing. Both the requester and the exact target subset that ends up being executed are therefore scheduling artifacts. What must still match is the *set of projects* that participated, which this normalization preserves." + }, + { + "pattern": "^\\s*Project \"(?[^\"]+)\" \\(\\d+(?::\\d+)?\\) on node \\d+[^\\r\\n]*$", + "replacement": "PROJECT BUILT: ${built}", + "reason": "Same as above, for the top-level project-started form." + }, + { + "pattern": "^\\s*Project \"(?[^\"]+)\" on node \\d+[^\\r\\n]*$", + "replacement": "PROJECT BUILT: ${built}", + "reason": "Same as above, for the entry-point project-started form." + }, + { + "pattern": "^\\s*Done Building Project \"(?[^\"]+)\"[^\\r\\n]*$", + "replacement": "PROJECT DONE: ${built}", + "reason": "Pairs with the project-started normalization above." + }, + { + "pattern": "\\(in \\d+(\\.\\d+)? (ms|sec|s|min)\\)", + "replacement": "(in T)", + "reason": "Durations." + }, + { + "pattern": "\\b\\d+(\\.\\d+)?\\s?ms\\b", + "replacement": "Tms", + "reason": "Durations." + }, + { + "pattern": "(?", + "reason": "Per-run GUIDs (telemetry ids, temp directory names, and the ids 1ES SDL injection embeds in generated ruleset file names such as Foo__GdnDotnetAnalyzersMerged.ruleset). Anchored on hex-digit lookarounds rather than \\b: '_' is a word character, so \\b does not match between '_' and a hex digit and would miss exactly those embedded ids." + }, + { + "pattern": "(?i)\\\\artifacts\\\\tmp\\\\[^\\\\\"]*", + "replacement": "\\artifacts\\tmp\\", + "reason": "Per-process temp file names." + } + ], + + "setOnly": [ + { + "pattern": "^[A-Za-z_][A-Za-z0-9_.]*:$", + "reason": "MSBuild's file logger re-emits a target header every time output from a different node interleaves, so the number of header lines is a pure scheduling artifact. Compared for presence instead. How many times each target actually executed is compared exactly, and enforced, by the structure/targets and structure/targetsByProject sections, which read TargetStarted events directly from the binlog." + }, + { + "pattern": "^PROJECT (BUILT|DONE): ", + "reason": "See the project-started/finished normalization above: the number of build requests that actually execute (versus being satisfied from the results cache) is scheduling dependent, so these are compared for presence. How many times each project was actually built is compared exactly, and enforced, by the structure/projects section, which reads ProjectStarted events directly from the binlog." + } + ], + + "//knownMtOnly": "Differences that only ever appear on the -mt side, are already fixed or filed, and are therefore reported separately instead of failing the comparison. These entries are applied only to the mt-vs-baseline comparison, never to the control. NOTE: an entry cannot be deleted when its fix merges to main - this pipeline applies -mt to the MSBuild that *drives* the build (the pool's installed Visual Studio MSBuild), not the one the build produces, so a fix only takes effect here once it ships in the VS on the agents.", + "knownMtOnly": [ + { + "pattern": "^Task assembly was loaded from '.*' while the desired location was '.*'\\.$", + "reason": "Under -mt, a task routed to an out-of-process task host is represented in-proc by a TaskHostTask proxy whose own assembly location is always Microsoft.Build.dll, so MSBuild reported a mismatch for most task invocations (1364 extra lines in an official build of this repo). Log noise only: no build output changes, which the byte-level artifact comparison confirms. Fixed by https://github.com/dotnet/msbuild/pull/14550, but the messages are emitted by the installed Visual Studio MSBuild that drives the build - VSEng-MicroBuildVSStable is on MSBuild 18.8.2 while the fix is in main (18.11) - so this entry has to stay until that pool picks up a VS carrying it. Delete it then." + }, + { + "pattern": "^[A-Za-z_][A-Za-z0-9_.]*:$", + "reason": "Collateral of the messages above: MSBuild's file logger only emits a target header when the target logs something at the current verbosity, so targets that are silent at normal verbosity acquire a header in the -mt log only. All 12 target names this suppresses appear in the -mt log only, and every one of their header blocks contains nothing but those messages. What this rule stops comparing is covered by the structure/targets and structure/targetsByProject sections, which take exact execution counts for every target - and every (project, target) pair - from the binlog event stream, are unaffected by verbosity or interleaving, always run, and are enforced. Delete this entry together with the one above." + } + ] +} diff --git a/scripts/mt-equivalence/MtCompareNative.cs b/scripts/mt-equivalence/MtCompareNative.cs new file mode 100644 index 00000000000..1055cde016a --- /dev/null +++ b/scripts/mt-equivalence/MtCompareNative.cs @@ -0,0 +1,197 @@ +// Helper types for the MSBuild -mt build-equivalence comparison scripts. +// Compiled at runtime with Add-Type; PowerShell's own file enumeration + hashing is roughly an +// order of magnitude slower on the ~25k-file artifact trees this compares. +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Threading.Tasks; + +public static class MtCompareNative +{ + public sealed class FileEntry + { + public string RelativePath; + public string FullPath; + public long Length; + public string Hash; + } + + /// + /// Hashes every file under in parallel. + /// Relative paths use forward slashes so they are stable across the two snapshots. + /// + public static List HashTree(string root, int degreeOfParallelism) + { + root = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + int prefix = root.Length + 1; + + var files = new List(Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)); + var results = new FileEntry[files.Count]; + + var options = new ParallelOptions { MaxDegreeOfParallelism = degreeOfParallelism <= 0 ? Environment.ProcessorCount : degreeOfParallelism }; + Parallel.For(0, files.Count, options, i => + { + string path = files[i]; + var entry = new FileEntry + { + FullPath = path, + RelativePath = path.Substring(prefix).Replace('\\', '/'), + Length = -1, + }; + try + { + var info = new FileInfo(path); + entry.Length = info.Length; + // FileShare.ReadWrite: some outputs may still be memory-mapped by a lingering process. + using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 64 * 1024, FileOptions.SequentialScan)) + using (var sha = SHA256.Create()) + { + entry.Hash = BitConverter.ToString(sha.ComputeHash(stream)).Replace("-", string.Empty); + } + } + catch (Exception ex) + { + entry.Hash = "ERROR:" + ex.GetType().Name; + } + results[i] = entry; + }); + + return new List(results); + } + + public sealed class DiffRun + { + public long Start; + public long Length; + } + + /// + /// Returns the contiguous runs of differing bytes between two files, up to . + /// Trailing bytes of the longer file are reported as a final run. + /// + public static List DiffRuns(string leftPath, string rightPath, int maxRuns) + { + byte[] left = File.ReadAllBytes(leftPath); + byte[] right = File.ReadAllBytes(rightPath); + var runs = new List(); + int min = Math.Min(left.Length, right.Length); + + int i = 0; + while (i < min && runs.Count < maxRuns) + { + if (left[i] == right[i]) { i++; continue; } + int start = i; + while (i < min && left[i] != right[i]) { i++; } + runs.Add(new DiffRun { Start = start, Length = i - start }); + } + + if (left.Length != right.Length && runs.Count < maxRuns) + { + runs.Add(new DiffRun { Start = min, Length = Math.Abs(left.Length - right.Length) }); + } + + return runs; + } + + /// + /// Streams a text file, applies prefix/drop/replace normalization and returns a line -> count map. + /// + /// + /// The node-id and timestamp prefix is stripped before anything else, because whether the file + /// logger emits one depends on the replay engine rather than on the build. Leaving it in place + /// makes every '^'-anchored drop rule silently stop matching, which turns dropped noise back into + /// reported differences. + /// + public static Dictionary NormalizedLineCounts( + string path, + System.Text.RegularExpressions.Regex[] prefix, + System.Text.RegularExpressions.Regex[] drop, + System.Text.RegularExpressions.Regex[] replaceFrom, + string[] replaceTo, + System.Text.RegularExpressions.Regex[] setOnly, + HashSet setOnlySeen) + { + var counts = new Dictionary(StringComparer.Ordinal); + + foreach (string rawLine in File.ReadLines(path)) + { + string raw = rawLine; + for (int i = 0; i < prefix.Length; i++) + { + raw = prefix[i].Replace(raw, string.Empty); + } + + bool skip = false; + for (int i = 0; i < drop.Length; i++) + { + if (drop[i].IsMatch(raw)) { skip = true; break; } + } + if (skip) { continue; } + + string line = raw; + for (int i = 0; i < replaceFrom.Length; i++) + { + line = replaceFrom[i].Replace(line, replaceTo[i]); + } + line = line.Trim(); + if (line.Length == 0) { continue; } + + bool countInsensitive = false; + for (int i = 0; i < setOnly.Length; i++) + { + if (setOnly[i].IsMatch(line)) { countInsensitive = true; break; } + } + + if (countInsensitive) + { + // Presence-only: the file logger re-emits a target/context header every time output + // from different nodes interleaves, so the count is a scheduling artifact. + setOnlySeen.Add(line); + counts[line] = 1; + continue; + } + + int existing; + counts[line] = counts.TryGetValue(line, out existing) ? existing + 1 : 1; + } + + return counts; + } + + /// + /// Streams a text file and collects, for each named extractor, the set of values captured by the + /// extractor's "v" group. Used for target/task coverage comparison, where the *set* is meaningful + /// but the count and ordering are scheduling artifacts. + /// + public static Dictionary> ExtractSets( + string path, + string[] names, + System.Text.RegularExpressions.Regex[] patterns) + { + var result = new Dictionary>(StringComparer.Ordinal); + for (int i = 0; i < names.Length; i++) + { + result[names[i]] = new HashSet(StringComparer.Ordinal); + } + + foreach (string raw in File.ReadLines(path)) + { + string line = raw.Trim(); + if (line.Length == 0) { continue; } + + for (int i = 0; i < patterns.Length; i++) + { + var m = patterns[i].Match(line); + if (m.Success) + { + result[names[i]].Add(m.Groups["v"].Value); + break; + } + } + } + + return result; + } +} diff --git a/scripts/mt-equivalence/README.md b/scripts/mt-equivalence/README.md new file mode 100644 index 00000000000..0888f793535 --- /dev/null +++ b/scripts/mt-equivalence/README.md @@ -0,0 +1,316 @@ +# `-mt` build equivalence harness + +Proves that building MSBuild's official artifact set with MSBuild running in multithreaded mode +(`-mt`) produces the same outputs as building it without `-mt`. + +The Azure DevOps pipeline that runs this is +[`azure-pipelines/.vsts-dotnet-mt-equivalence.yml`](../../azure-pipelines/.vsts-dotnet-mt-equivalence.yml). +It runs daily at 02:00 UTC against `main` with `always: true`, so it catches `-mt` regressions +introduced by the whole dependency surface (SDK, arcade, Roslyn, NuGet), not only by commits to this +repo. A red run means `-mt` changed something. + +## What it does + +`Run-MTEquivalence.ps1` runs the *same* official-style build command up to three times **in the same +working tree** (so every absolute path baked into an output is identical) and then compares the +resulting `artifacts` trees: + +| run | command | purpose | +|---|---|---| +| `baseline` | `build.cmd -pack -ci -configuration Release ` | reference | +| `mt` | the same command with `MSBUILD_MT_ENABLED=1` | the thing under test | +| `control` | the same command again, no `-mt` anywhere | scientific control | + +`MSBUILD_MT_ENABLED=1` is the mechanism arcade already provides: `eng/common/tools.ps1` appends `-mt` +to the MSBuild command line when it is set. The build command is otherwise the command the official +build runs, minus four things: + +| dropped | why | +|---|---| +| `-sign` | needs the MicroBuild signing plugin, and an Authenticode signature embeds a trusted-timestamp countersignature, so signed binaries could never be byte-identical between two runs. What `-mt` could actually affect is the *content* being signed, and that is compared unsigned | +| `EnableNgenOptimization` | this is exactly the official pipeline's `$(SkipApplyOptimizationDataArg)`, which it passes whenever OptProf is off. Leaving it on requires a `VisualStudioDropAccessToken` and downloads an IBC drop that both runs would consume identically | +| `GenerateSbom` | SBOM manifests embed a generation timestamp and a per-run GUID, so they can never be byte-compared | + +`-publish` **is** passed. It is the *official-build* publish, not `dotnet publish`, and with +`DotNetPublishUsingPipelines=true` it pushes nothing to any feed — it emits `##vso[artifact.upload]` +logging commands and produces three real outputs that would otherwise never be built: + +| output | compared as | +|---|---| +| `log//AssetManifest/*.xml` — the Build Asset Registry manifest listing every package, blob and PDB the build declares | byte-identical | +| `tmp//SymbolPackages/*.symbols.nupkg` — 10 generated symbol packages | payload byte-identical | +| `tmp//PDBsToPublish/**` — 34 PDBs staged for the symbol server | byte-identical | + +Feed publishing happens in the separate post-build stage, which this pipeline does not run. Note that +those three live under `log/` and `tmp/`, which are otherwise ignored, so `ArtifactCompareRules.json` +carves them out *ahead* of the ignore rules. + +> **`##vso` logging commands from the inner builds are neutralized.** An ADO agent interprets +> `##vso[...]` found in step output, and these are real official-style builds: `-publish` emits +> `##vso[artifact.upload]` for every package, symbol package, PDB and the asset manifest, and arcade's +> initialization emits `##vso[task.setvariable]` / `##vso[task.prependpath]`. Acting on those would +> make this validation pipeline publish build assets only the official build may publish, mutate the +> outer job's `TEMP`/`TMP`/`PATH`, and fail the job when the agent's *asynchronous* upload races the +> snapshot move. The orchestrator rewrites the marker so the lines stay readable but are inert. The +> verdict's own `##vso[task.logissue]` messages are unaffected. + +`VisualStudioDropName` is also passed (mirroring the official job's `VisualStudio.DropName`): without +it `AfterSigning.proj` fails outright. It is only embedded into the generated VS insertion manifests — +nothing is uploaded — and it is identical across the runs being compared. + +Everything else — `-pack`, `-ci`, `OfficialBuildId`, `RepositoryName`, `TeamName`, +`DotNetPublishUsingPipelines`, `SuppressFinalPackageVersion`, `IsExperimental` — is passed exactly as +the official job passes it, so the same projects, targets and packaging run. + +### Use `vs`, not `dotnet` + +The official build runs `msbuildEngine: vs`, and only that engine produces the **VS insertion +outputs**: `artifacts/VSSetup` (`.vsix`, `.vsman`, `.vsmand`), the `VS.ExternalAPIs.MSBuild` package +and the arm64 flavours. On this repo that is 28 011 artifact files versus 23 329 with `dotnet` — so +validating with `dotnet` silently skips the entire insertion surface. `vs` is the default here for +that reason; `-MSBuildEngine dotnet` remains available for machines without a new enough VS. + +**The control run is what makes a difference interpretable.** It measures how much this repo's build +output varies between two *identical* builds. Any difference that also shows up in the control is +pre-existing build non-determinism, not something `-mt` caused. Without it, an `mt`-vs-`baseline` +diff cannot be attributed to anything. + +### Which MSBuild's `-mt` this validates + +`MSBUILD_MT_ENABLED=1` makes arcade pass `-mt` to **the MSBuild that drives the build** — the pool's +Visual Studio MSBuild with `-MSBuildEngine vs`, or the SDK pinned in `global.json` with `dotnet`. So +this pipeline answers *"can the official build be flipped to `-mt` today?"*. + +That is deliberately a different question from the one `.vsts-dotnet-ci.yml` answers. CI passes +`-stage2Arguments /mt`, which applies `-mt` to a stage-2 build driven by the **freshly built** MSBuild, +answering *"does the `-mt` implementation in this PR break a build?"*. Both are worth having; do not +read a green run here as coverage for in-development `-mt` changes. + +A practical consequence: the MSBuild on the agent must itself support `-mt`, or only the `mt` build +fails (with MSB1001 for an unrecognized switch). The orchestrator prints an explicit hint in that case. +A second one: a fix merged into this repo does not change what this pipeline sees until it ships in +the Visual Studio installed on the pool. + +## What it compares + +### 1. Artifacts, byte for byte — `Compare-Artifacts.ps1` + +Hashes every file under `artifacts` on both sides and classifies each path with an ordered rule set +([`ArtifactCompareRules.json`](ArtifactCompareRules.json)): + +| disposition | meaning | +|---|---| +| `Compare` (default) | must be byte-identical; a difference fails the run | +| `ComparePayload` | zip container: the *entries* must be byte-identical, but the zip's per-entry timestamps may differ. A rule may add `normalizeEntries` to strip one documented per-build value out of specific entries (see `.vsix` below) while still comparing everything else in them | +| `Informational` | non-deterministic by construction; reported with a written reason, does not fail | +| `Ignore` | excluded (logs, temp, SBOM manifests, dev-environment helper files) | + +`Ignore` is the only disposition that removes a path from the diff lists entirely, so the report +accounts for it explicitly: every run lists each `Ignore` rule with how many paths it matched, how +many of those actually differed, and a sample of the paths. A rule matching nothing is called out as +dead, and a rule with a large `differing` count is visibly doing real work. Review that table when +changing the rules. + +Note that patterns are matched with PowerShell `-like`, where `*` also crosses directory separators — +so a pattern must not be prefixed with `**/` if it needs to match a root-relative path such as +`VSSetup/Release/...`. + +Two reclassifications are derived rather than hard-coded, so the rule file stays small and honest: + +* **MVID-only PE differences.** If the only differing bytes in an assembly are the COFF + `TimeDateStamp`, the optional-header `CheckSum`, and a single 16-byte window, the assembly was + compiled with `/deterministic-` and its MVID is a random GUID. Under `/deterministic+` the MVID is + a hash of the emitted content, so an MVID-only delta cannot be caused by the build engine. +* **Payload-identical zips.** A `.nupkg` whose entries all hash equal but whose entry timestamps + differ is reported as payload-identical (NuGet stamps entries with the wall-clock last-write time + of the packed file). + +Differing files are triaged automatically: PE images are reported per PE section, zips per entry, +text files by first differing line. + +### 2. Logs, functionally — `Compare-Binlogs.ps1` + +Binary logs can never be byte-identical (timestamps, durations, node ids, event ordering), so they +are compared functionally: + +1. **`-mt` evidence.** The MSBuild command line is read back out of each binlog (it is recorded in + the first few KB, so this costs one gzip read) and the script asserts that the `mt` run really did + run with `-mt` and the baseline did not. Without this the whole comparison could silently become + vacuous. +2. **Structure.** The binlog event stream is read directly and the two runs must agree, exactly, on + how many times every target ran, every `(project, target)` pair ran, every task ran and every + project was built, plus the multiset of warning and error codes. This costs about 2.5 seconds per + binlog and needs no replay. It is the check that makes the noise tolerance of the text tiers below + safe: those deliberately stop counting target headers, and this is what still pins down what + actually executed. A target that runs but logs nothing is invisible to a text comparison at any + verbosity, and is caught here. +3. **Diagnostics.** Both logs are replayed at quiet verbosity; the multiset of errors and warnings + must be identical. +4. **Functional.** Both are replayed at normal verbosity, normalized with + [`LogNormalizationRules.json`](LogNormalizationRules.json), and compared as line multisets. +5. **Coverage** (`-DeepCompare`). Both are replayed at diagnostic verbosity and the sets of executed + tasks and of task→assembly bindings must match. Target coverage is reported but never fails, + because it is extracted from text headers and is unstable for the same reason; tier 2 covers that + ground precisely and always. What this tier still adds is the task→assembly binding. + +Every normalization rule carries a `reason` explaining why the raw text cannot be compared verbatim. + +> **Editing `drop` patterns:** the `prefix` rules (node id and timestamp) are stripped first, but +> `drop` still runs before any `replace` rule, so the line keeps its indentation and its trailing +> `(TaskId:N)`. Anchoring a `drop` pattern with `$` will usually not match. + +### 3. Netting the `-mt` differences against the control + +When the control run is present, `Run-MTEquivalence.ps1` computes the verdict that actually matters: +**differences attributable to `-mt`**. Any artifact path or log line that differs in the `mt` +comparison *and* also differs in the `control` comparison has just been demonstrated, in that same +run, to differ without `-mt` being involved, so it is reported as explained by the control rather than +counted against `-mt`. + +This keeps the pipeline strict about `-mt` while staying robust to pre-existing non-determinism that +the static rule set has not seen yet — for example, outputs that only exist on a signing-enabled +MicroBuild agent. Without a control run (`-SkipControl`) the verdict falls back to the raw +`mt`-vs-`baseline` result, which is stricter but attributes nothing. + +The run always fails if the `-mt` evidence check fails, regardless of everything else: a comparison +that cannot prove `-mt` was actually on is worthless. + +## Investigating a failure + +The artifact snapshots are far too large to publish (~4.5 GB each, three of them), and a regression +that only reproduces intermittently may never reproduce on a developer machine. So every run publishes +a bounded evidence bundle as the `mt-equivalence-reports` pipeline artifact: + +| in the artifact | when | what it is | +|---|---|---| +| `binlogs/{baseline,mt,control}.binlog` | always (~28 MB) | the binary log of each build — traces a differing file back to the task and target that wrote it | +| `binlogs/.FAILED.binlog` | when a build itself fails | the failing build's log, rescued before cleanup destroys it | +| `evidence//` | when files differ | **both versions** of every differing file, so the bytes can be diffed offline | +| `evidence-manifest.json` | always | what was collected, and anything a size cap excluded | +| `verdict.json` | always | machine-readable verdict and the unexplained-difference lists | + +Differences are collected in priority order — attributed to `-mt` first, then the rest of the `mt` +comparison, then the `control` comparison — so when a cap is hit (`-MaxFiles 200`, +`-MaxTotalBytes 500MB` by default) the most important evidence is what survives. The manifest always +records the true totals. + +A typical investigation: open `summary.md` for the verdict, find the offending path, diff +`evidence/baseline/` against `evidence/mt/`, then open `binlogs/mt.binlog` to see which +task produced it. + +The replayed text logs used by the log comparison are *not* published: they run to tens of megabytes +and are regenerable from the binlogs in seconds. + +### Exit codes + +| situation | verdict | +|---|---| +| `-mt` could not be proven to have run with `-mt` | **fail** (exit 1) — the comparison proves nothing | +| a difference is attributable to `-mt` | **fail** (exit 1) | +| the control run shows non-determinism the rules do not explain | **warning**, exit unaffected — real, worth fixing, but not an `-mt` regression | +| otherwise | pass (exit 0) | + +Control-run problems are deliberately a warning rather than a failure: failing the pipeline for +pre-existing build non-determinism would train people to ignore a red run, which is exactly when the +`-mt` signal matters. In Azure DevOps both are also emitted as `##vso[task.logissue]` so they show up +on the run summary. + +### Known `-mt`-only log differences + +`LogNormalizationRules.json` has a `knownMtOnly` section for differences that are already fixed or +filed. They are reported in their own section of the report instead of failing the comparison, and are +applied **only** to the `mt` comparison, never to the control. + +An entry is cleared when its fix **reaches the agents**, not when it merges. Because `-mt` is applied +to the MSBuild that drives the build, a fix in this repo only changes what this pipeline observes once +it ships in the Visual Studio installed on the pool. Merging +[#14550](https://github.com/dotnet/msbuild/pull/14550) into `main` (18.11) left its entry firing +unchanged, because `VSEng-MicroBuildVSStable` drives the build with MSBuild 18.8.2. + +## Running it locally + +```powershell +# Full run: three builds plus all comparisons (about 1.5-2 hours on a dev box). +pwsh scripts/mt-equivalence/Run-MTEquivalence.ps1 -WorkDir D:\mtcmp -DeepLogCompare + +# Faster: skip the control run. +pwsh scripts/mt-equivalence/Run-MTEquivalence.ps1 -WorkDir D:\mtcmp -SkipControl + +# Iterate on the comparison rules against snapshots you already produced. +pwsh scripts/mt-equivalence/Run-MTEquivalence.ps1 -WorkDir D:\mtcmp -SkipBuilds -DeepLogCompare +``` + +On a machine without the Visual Studio version pinned in `global.json`, add `-MSBuildEngine dotnet`. + +Each snapshot of `artifacts` is about 4.5 GB, and up to three are kept at once, so budget ~15 GB of +free space for `-WorkDir`. + +Reports land in `\reports`: + +* `summary.md` – everything in one file (this is what the pipeline uploads as the build summary) +* `artifact-compare.