From 276ec9d4cf0537d9880d9e560f13da947b52ef21 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 10:42:38 -0700 Subject: [PATCH 1/6] Add separate cache for unit tests. Signed-off-by: Spencer Schrock --- .github/workflows/main.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16457560133..e5d89458100 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,36 +37,35 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1 + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - + - name: Setup Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache builds # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 with: path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-unit-test-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go- + ${{ runner.os }}-go-unit-test- - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0 - with: - go-version: ${{ env.GO_VERSION }} - check-latest: true - cache: true + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Run unit-tests run: make unit-test - name: Upload codecoverage - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 2.1.0 + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4 with: files: ./unit-coverage.out verbose: true From a16b0e7a78994a64862ff16a9d26cf0e867234cb Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 11:08:49 -0700 Subject: [PATCH 2/6] share cache with gitlab tests too. Signed-off-by: Spencer Schrock --- .github/workflows/gitlab.yml | 30 +++++++++++++++++++++++------- .github/workflows/main.yml | 4 ++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 964c47f2834..ef86f047f89 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -24,6 +24,9 @@ on: branches: - main +env: + GO_VERSION_FILE: go.mod # no good way of getting a mutual version between go.mod and tools/go.mod + jobs: gitlab-integration-trusted: runs-on: ubuntu-latest @@ -33,18 +36,31 @@ jobs: uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - + - name: Setup Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: ${{ env.GO_VERSION_FILE }} + check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 + with: + path: | + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-tests- - name: Clone the code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} # head SHA if PR, else fallback to push SHA - - name: setup-go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version: '1.19' - check-latest: true - - name: Prepare test env run: | go mod download diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5d89458100..6f81f9d4748 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,9 +57,9 @@ jobs: path: | ${{ steps.go-cache-paths.outputs.go-build }} ${{ steps.go-cache-paths.outputs.go-mod }} - key: ${{ runner.os }}-go-unit-test-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go-unit-test- + ${{ runner.os }}-go-tests- - name: Clone the code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Run unit-tests From 151b28e2d029dee1af996db2be8b5af68091a0df Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 11:12:34 -0700 Subject: [PATCH 3/6] share cache with github integration tests. Signed-off-by: Spencer Schrock --- .github/workflows/integration.yml | 40 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f2e88119a0b..ca3a27202b9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,6 +23,9 @@ on: permissions: contents: read +env: + GO_VERSION_FILE: go.mod # no good way of getting a mutual version between go.mod and tools/go.mod + jobs: approve: runs-on: ubuntu-latest @@ -41,27 +44,40 @@ jobs: needs: [approve] steps: - name: Harden Runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1 + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - - name: pull_request actions/checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: setup-go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0 + - name: Setup Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: - go-version: '1.19' + go-version-file: ${{ env.GO_VERSION_FILE }} check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 + with: + path: | + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-tests- + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Prepare test env run: | go mod download - name: Run GITHUB_TOKEN E2E #using retry because the GitHub token is being throttled. - uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd + uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd # v2.8.3 env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -71,7 +87,7 @@ jobs: command: make e2e-gh-token - name: codecov - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 2.1.0 + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4 with: files: "*e2e-coverage.out" verbose: true From ab76f221ceff795d9631f6fdebb84d4fae569849 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 11:19:29 -0700 Subject: [PATCH 4/6] explicitly download modules in unit test job Signed-off-by: Spencer Schrock --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f81f9d4748..11e15bdf2b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,6 +62,9 @@ jobs: ${{ runner.os }}-go-tests- - name: Clone the code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: Prepare test env + run: | + go mod download - name: Run unit-tests run: make unit-test - name: Upload codecoverage From 52c878f51bf5247b3de385eb7e20e5bc32c11a61 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 11:30:13 -0700 Subject: [PATCH 5/6] checkout needs to be before the go.mod is read. Signed-off-by: Spencer Schrock --- .github/workflows/gitlab.yml | 8 ++++---- .github/workflows/integration.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index ef86f047f89..6ba61e9405f 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -36,6 +36,10 @@ jobs: uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} # head SHA if PR, else fallback to push SHA - name: Setup Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: @@ -56,10 +60,6 @@ jobs: key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-tests- - - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} # head SHA if PR, else fallback to push SHA - name: Prepare test env run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ca3a27202b9..c4e70ccf88d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -47,6 +47,10 @@ jobs: uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Setup Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: @@ -67,10 +71,6 @@ jobs: key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-tests- - - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - ref: ${{ github.event.pull_request.head.sha }} - name: Prepare test env run: | From 701e044b35f8c1d54dab34b49c895c516975f06f Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 11:31:32 -0700 Subject: [PATCH 6/6] checkout needs to be before the go.sum files are hashed. Signed-off-by: Spencer Schrock --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11e15bdf2b9..a995caa1c77 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,6 +40,8 @@ jobs: uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Setup Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: @@ -60,8 +62,6 @@ jobs: key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-tests- - - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Prepare test env run: | go mod download