CI Improvements - Parallelize integration tests, cache ghcup, fix tag caching

.github/workflows/badges.yml

@@ -11,7 +11,7 @@ jobs:
     env:
       GH_TOKEN: ${{ github.token }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Update badge
         run: |
           CLI_DOWNLOADS=$(gh api /repos/fossas/fossa-cli/releases --paginate --cache 1h | jq '.[] | .assets | .[] | .download_count' | jq --slurp 'add')

.github/workflows/bench.yml

@@ -13,14 +13,16 @@ jobs:
     steps:
     - uses: dtolnay/rust-toolchain@stable
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         lfs: true
         fetch-tags: true
 
     - name: Get latest release tag
       id: latest-tag
-      run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
+      run: |
+        tag="$(git tag --sort=-v:refname 2>/dev/null | head -1 || true)"
+        echo "tag=${tag:-none}" >> "$GITHUB_OUTPUT"
 
     # Adding the "git config ..." line ensures git doesn't fail during our build.
     - name: Configure Git
@@ -55,15 +57,15 @@ jobs:
 
     # Benchmarks build at a different optimization level, so they use
     # their own cache prefix to avoid collisions with the main build.
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache cabal store
       with:
         path: ${{ steps.setup-haskell.outputs.cabal-store || '~/.local/state/cabal' }}
         key: ${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-cabal-cache-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
         restore-keys: |
           ${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-cabal-cache-
 
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache dist-newstyle
       with:
         path: ${{ github.workspace }}/dist-newstyle

.github/workflows/build-all.yml

@@ -54,22 +54,35 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         lfs: true
         fetch-depth: 2
         fetch-tags: true
 
     - name: Get latest release tag
       id: latest-tag
-      run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
+      run: |
+        tag="$(git tag --sort=-v:refname 2>/dev/null | head -1 || true)"
+        echo "tag=${tag:-none}" >> "$GITHUB_OUTPUT"
 
     - name: Install MacOS binary dependencies
       if: ${{ contains(matrix.os, 'macos') }}
       run: |
         brew install jq
 
     # Set up Haskell.
+    # Cache ghcup installations so the setup action skips downloading GHC (~500 MB).
+    - uses: actions/cache@v5
+      if: ${{ !contains(matrix.os-name, 'Linux') }}
+      name: Cache ghcup
+      with:
+        path: |
+          ${{ runner.os == 'Windows' && 'C:\ghcup\bin' || '~/.ghcup/bin' }}
+          ${{ runner.os == 'Windows' && format('C:\ghcup\ghc\{0}', matrix.ghc) || format('~/.ghcup/ghc/{0}', matrix.ghc) }}
+          ${{ runner.os == 'Windows' && 'C:\ghcup\cache' || '~/.ghcup/cache' }}
+        key: ${{ matrix.os-name }}-ghcup-${{ matrix.ghc }}-3.10.3.0
+
     - uses: haskell-actions/setup@v2
       id: setup-haskell
       name: Setup ghc/cabal (non-alpine)
@@ -129,15 +142,15 @@ jobs:
     # The home directory inside a github container run during a step is different than one run outside of it.
     # This is why there is special logic for 'LinuxARM'.
     # Its builds run inside a container but are cached by an action outside of it.
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache cabal store
       with:
         path: ${{ steps.setup-haskell.outputs.cabal-store || ( matrix.os == 'LinuxARM' && format('{0}/_github_home/.local/state/cabal', runner.temp) || '~/.local/state/cabal') }}
         key: ${{ matrix.os-name }}-${{ matrix.ghc }}-cabal-cache-${{ hashFiles('spectrometer.cabal', matrix.project-file, 'cabal.project.common') }}
         restore-keys: |
           ${{ matrix.os-name }}-${{ matrix.ghc }}-cabal-cache-
 
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache dist-newstyle
       with:
         path: ${{ github.workspace }}/dist-newstyle
@@ -249,7 +262,7 @@ jobs:
            exit 1
         fi
 
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v5
       with:
         name: ${{ matrix.os-name }}-binaries
         path: release
@@ -263,7 +276,7 @@ jobs:
       contents: write
 
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
 
     # Sets VERSION from git's tag or sha.
     # refer to: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-of-setting-an-output-parameter
@@ -438,7 +451,7 @@ jobs:
     # need to run for tagged release versions.
     - name: Upload release archives
       if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: release-archives
         path: release

.github/workflows/dependency-scan.yml

@@ -9,7 +9,7 @@ jobs:
       FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Install and set GHC and Cabal versions
         # The first line works around a ghcup issue:

.github/workflows/install-script-test.yml

@@ -8,7 +8,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest-large]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: install script performs installation
         shell: bash
@@ -61,7 +61,7 @@ jobs:
   test-macos-arm:
     runs-on: "macos-latest"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: install latest script can install a specific version
         shell: bash
         run: |
@@ -80,7 +80,7 @@ jobs:
   test-windows:
     runs-on: "windows-latest"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: get latest release version from github
         shell: pwsh

.github/workflows/integrations-test.yml

@@ -1,41 +1,58 @@
 name: Integration Tests
 
-# Only run workflow manually
-# Refer to https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#workflow_dispatch
 on:
   push:
   workflow_dispatch:
   schedule:
     - cron: "0 5 * * *" # At 05:00 on every day.
 
+# Cancel in-progress runs for the same branch so stale runs don't block the runner.
+concurrency:
+  group: integration-${{ github.ref }}
+  cancel-in-progress: true
+
+# Single source of truth for integration test groups. Used by both the
+# matrix strategy and the build job's coverage validation step.
+env:
+  TEST_MATRIX: >-
+    [
+      {"group": "jvm", "matches": "Analysis.Maven Analysis.Gradle Analysis.Scala Analysis.Clojure"},
+      {"group": "compiled", "matches": "Analysis.Go Analysis.Rust Analysis.Erlang Analysis.Elixir"},
+      {"group": "scripting-and-other", "matches": "Analysis.Python Analysis.Ruby Analysis.Swift Analysis.Cocoapods Analysis.Pnpm Analysis.Nuget Analysis.Carthage Analysis.LicenseScanner Container.Analysis Reachability.Upload"}
+    ]
+
 jobs:
-  integration-test:
-    name: integration-test
+  # Build once, then run test groups in parallel.
+  # The build job populates the cabal and dist-newstyle caches.
+  # Each test job restores those caches so it doesn't need to rebuild.
+  build:
+    name: build
     runs-on: "fossa-cli-integration-runner"
-    # Be sure to update the env below too
     container: fossa/haskell-static-alpine:ghc-9.8.4
-
+    outputs:
+      latest-tag: ${{ steps.latest-tag.outputs.tag }}
+      test-matrix: ${{ env.TEST_MATRIX }}
     env:
       GHC_VERSION: '9.8.4'
 
     steps:
     - uses: dtolnay/rust-toolchain@stable
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         lfs: true
         fetch-depth: 2
         fetch-tags: true
 
     - name: Get latest release tag
       id: latest-tag
-      run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
+      run: |
+        tag="$(git tag --sort=-v:refname 2>/dev/null | head -1 || true)"
+        echo "tag=${tag:-none}" >> "$GITHUB_OUTPUT"
 
     - name: Ensures git ownership check does not lead to compile error (we run git during compile for version tagging, etc.)
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
-    # adduser cannot add users to group: https://unix.stackexchange.com/a/397733
-    # so we edit /etc/group directly
     - name: Create nixbuild users/group
       run: |
         addgroup nixbld
@@ -60,19 +77,15 @@ jobs:
 
     - uses: Swatinem/rust-cache@v2
 
-    # Cache cabal store and dist-newstyle keyed on spectrometer.cabal + project file.
-    # cabal build handles staleness internally — if a transitive dep changed,
-    # it recompiles only what's needed. This avoids the 8-min cabal update +
-    # dry-run solver step that the plan-hash approach requires.
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache cabal store
       with:
-        path: ${{ steps.setup-haskell.outputs.cabal-store || '~/.local/state/cabal' }}
+        path: ~/.local/state/cabal
         key: ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
         restore-keys: |
           ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-
 
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       name: Cache dist-newstyle
       with:
         path: ${{ github.workspace }}/dist-newstyle
@@ -98,8 +111,127 @@ jobs:
         cabal update
         $RUN_CMD || $RUN_CMD
 
-    # This is set up to run integration tests in parallel.
-    # If that becomes a problem disable it by removing the "+RTS -N -RTS" test options.
-    - name: Run all integration tests
+    # Verify every integration test is covered by at least one group pattern.
+    # Counts tests from an unfiltered --dry-run vs the union of all --match
+    # patterns. Fails if any tests would be silently skipped.
+    - name: Validate test groups cover all tests
+      run: |
+        TOTAL=$(cabal test --project-file=cabal.project.ci.linux \
+          --test-show-details=direct \
+          integration-tests \
+          --test-option=--dry-run 2>&1 | grep -c "^  " || echo 0)
+
+        ALL_PATTERNS=$(echo '${{ env.TEST_MATRIX }}' | jq -r '.[].matches' | tr ' ' '\n' | sort -u | tr '\n' ' ')
+        MATCH_ARGS=""
+        for pattern in $ALL_PATTERNS; do
+          MATCH_ARGS="$MATCH_ARGS --test-option=--match --test-option=$pattern"
+        done
+        MATCHED=$(cabal test --project-file=cabal.project.ci.linux \
+          --test-show-details=direct \
+          $MATCH_ARGS \
+          integration-tests \
+          --test-option=--dry-run 2>&1 | grep -c "^  " || echo 0)
+
+        echo "Total tests: $TOTAL"
+        echo "Matched tests: $MATCHED"
+        if [ "$TOTAL" -ne "$MATCHED" ]; then
+          echo "ERROR: $((TOTAL - MATCHED)) integration tests are not covered by any group pattern."
+          echo "Update TEST_MATRIX in integrations-test.yml to include the missing patterns."
+          exit 1
+        fi
+
+    # Upload artifacts that test jobs need so they don't have to rebuild
+    # Rust or re-download vendor binaries.
+    - uses: actions/upload-artifact@v5
+      with:
+        name: vendor-bins
+        path: vendor-bins/
+        if-no-files-found: error
+
+    - uses: actions/upload-artifact@v5
+      with:
+        name: rust-release-bins
+        path: |
+          target/release/berkeleydb
+          target/release/millhone
+        if-no-files-found: error
+
+  integration-test:
+    name: test-${{ matrix.group }}
+    needs: build
+    runs-on: "fossa-cli-integration-runner"
+    container: fossa/haskell-static-alpine:ghc-9.8.4
+    env:
+      GHC_VERSION: '9.8.4'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.build.outputs.test-matrix) }}
+
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        lfs: true
+        fetch-depth: 2
+
+    - name: Ensures git ownership check does not lead to compile error
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+    - name: Create nixbuild users/group
+      run: |
+        addgroup nixbld
+        adduser -D nixbld-1
+        adduser -D nixbld-2
+        adduser -D nixbld-3
+        sed 's/nixbld:x:\([[:digit:]]*\):$/nixbld:x:\1:nixbld-1,nixbld-2,nixbld-3/' /etc/group > group-changed
+        mv group-changed /etc/group
+
+    - uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:nixos-25.05
+        extra_nix_config: "build-users-group = nixbld"
+
+    # Restore caches populated by the build job.
+    - uses: actions/cache@v5
+      name: Cache cabal store
+      with:
+        path: ~/.local/state/cabal
+        key: ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-
+
+    - uses: actions/cache@v5
+      name: Cache dist-newstyle
+      with:
+        path: ${{ github.workspace }}/dist-newstyle
+        key: ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ needs.build.outputs.latest-tag }}-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ needs.build.outputs.latest-tag }}-
+          ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-
+
+    # extra-source-files (vendor-bins/*, target/release/*) must exist on
+    # disk or cabal recompiles EmbeddedBinary.hs (fails under -Werror).
+    - uses: actions/download-artifact@v5
+      with:
+        name: vendor-bins
+        path: vendor-bins/
+
+    - uses: actions/download-artifact@v5
+      with:
+        name: rust-release-bins
+        path: target/release/
+
+    - name: Run integration tests (${{ matrix.group }})
       run: |
-        cabal test --project-file=cabal.project.ci.linux --test-show-details=direct --test-option=--times integration-tests --test-options="+RTS -N -RTS"
+        cabal update
+        MATCH_ARGS=""
+        for pattern in ${{ matrix.matches }}; do
+          MATCH_ARGS="$MATCH_ARGS --test-option=--match --test-option=$pattern"
+        done
+        cabal test --project-file=cabal.project.ci.linux \
+          --test-show-details=direct \
+          --test-option=--times \
+          $MATCH_ARGS \
+          integration-tests \
+          --test-options="+RTS -N -RTS"