Merge branch 'master' into change-output-base

Author: cgundy

.claude/CLAUDE.md

@@ -8,11 +8,13 @@ After changing Rust code (`*.rs`) follow these steps in order:
 
 1. **Format** by running the following from the root of the repository:
    ```
+   cd "$(git rev-parse --show-toplevel)"
    rustfmt <MODIFIED_RUST_FILES>
    ```
    where `<MODIFIED_RUST_FILES>` is a space separated list of paths of all modified Rust files relative to the root of the repository.
 2. **Lint** by running the following from the root of the repository:
    ```
+   cd "$(git rev-parse --show-toplevel)"
    cargo clippy --all-features <CRATES> -- \
        -D warnings \
        -D clippy::all \
@@ -30,7 +32,8 @@ After changing Rust code (`*.rs`) follow these steps in order:
    Fix any linting errors.
 3. **Build** the directly affected bazel targets by running the following from the root of the repository:
    ```
-   TARGETS="$(bazel query 'kind(rule, rdeps(//..., set(<MODIFIED_FILES>), 1))' --keep_going 2>/dev/null)"
+   cd "$(git rev-parse --show-toplevel)"
+   TARGETS="$(bazel query 'kind(rule, rdeps(//..., set(<MODIFIED_FILES>), 1))' --keep_going 2>/dev/null)" || true
    if [ -n "$TARGETS" ]; then
        bazel build $TARGETS
    fi
@@ -40,11 +43,26 @@ After changing Rust code (`*.rs`) follow these steps in order:
    Fix all build errors.
 4. **Test** the directly affected bazel tests by running the following from the root of the repository:
    ```
-   TESTS="$(bazel query 'kind(".*_test|test_suite", kind(rule, rdeps(//..., set(<MODIFIED_FILES>), 2)))' --keep_going 2>/dev/null)"
+   cd "$(git rev-parse --show-toplevel)"
+   TESTS="$(bazel query 'kind(".*_test|test_suite", kind(rule, rdeps(//..., set(<MODIFIED_FILES>), 2))) except attr(tags, "manual", //...)' --keep_going 2>/dev/null)" || true
    if [ -n "$TESTS" ]; then
        bazel test --test_output=errors $TESTS
    fi
    ```
    (Use a depth of 2 in `rdeps` because tests usually depend on source files indirectly through a `rust_library` for example).
 
-   Fix all test failures.
+   Always run tests, even if they're system-tests, i.e. their label starts with `//rs/tests/`.
+
+   Fix all test failures.
+
+# Pull Requests
+
+When asked to create a PR, always create it in draft mode.
+
+When updating a PR prefer to push new commits to the PR branch instead of force-pushing over the existing commits.
+
+After the PR has been created or updated, request a review from the GitHub Copilot bot using:
+```
+gh api repos/dfinity/ic/pulls/<PULL_REQUEST_NUMBER>/requested_reviewers --method POST --raw-field 'reviewers[]=copilot-pull-request-reviewer[bot]'
+```
+where `<PULL_REQUEST_NUMBER>` is the number of the Pull Request.

.claude/skills/fix-flaky-tests/SKILL.md

@@ -60,6 +60,12 @@ This guide explains how to find flaky tests to fix and how to debug them. Flaky
 
 3. Analyze the source code of `label` and the logs in `<LOG_DIR>` to determine the root cause of the flakiness.
 
+   Ignore failures containing the error:
+   ```
+   Retried too many times: sending a request to Farm
+   ```
+   as those are due to infrastructure issues unrelated to the test code.
+
 4. Once you have determined the root cause,
    fix the test taking `.claude/CLAUDE.md` into account.
 
@@ -76,7 +82,12 @@ This guide explains how to find flaky tests to fix and how to debug them. Flaky
       and `<date>` with the current date in `YYYY-MM-DD` format,
       and commit your fix to that branch.
 
-   2. Submit a draft PR using `gh` with the fix.
+   2. Push the branch to `origin` (assuming it's `git@github.com:dfinity/ic.git`) using:
+      ```
+      git push --set-upstream origin HEAD
+      ```
+
+   3. Submit a draft PR using `gh` with the fix.
       Name it: `fix: deflake <label>`.
       Include the root cause analysis in the PR description
       and mention the PR was created following the steps in `.claude/skills/fix-flaky-tests/SKILL.md`.

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "ghcr.io/dfinity/ic-dev@sha256:fe06783d9cf8e9fc901a5996d9fc8b726f15769f2fd6bd86969d1fbbf77ae025",
+  "image": "ghcr.io/dfinity/ic-dev@sha256:a98782800df599b3d1501f0da9fbe1dff7a8a401a86c0371eaab7ae0503d3696",
   "remoteUser": "ubuntu",
   "privileged": true,
   "runArgs": [

.github/actions/netrc/action.yaml

@@ -14,6 +14,7 @@ runs:
         echo "machine api.github.com login x-access-token password ${{ github.token }}" >> ~/.netrc
         echo "Current GitHub API rate limits:"
 
-        # Show how close we are to the limits
-        curl -s --netrc https://api.github.com/rate_limit | \
-          jq -r '.resources.core | "Rate limit remaining: \(.remaining)/\(.limit) (resets at \(.reset | strftime("%Y-%m-%d %H:%M:%S %Z")))"'
+        # Show how close we are to the limits (ignore errors from curl or invalid JSON)
+        curl -sf --netrc https://api.github.com/rate_limit 2>/dev/null | \
+          jq -r '.resources.core | "Rate limit remaining: \(.remaining)/\(.limit) (resets at \(.reset | strftime("%Y-%m-%d %H:%M:%S %Z")))"' 2>/dev/null \
+          || echo "Could not retrieve rate limit information"

.github/workflows/api-bn-recovery-test.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:18d23aef1f5e9e7e1eef94c32563f8ed15531ae79065bb00bb5206a643fc49fe
+      image: ghcr.io/dfinity/ic-build@sha256:0a0cff0b12c7586c2f312c739176924edf0c9e71df92f382eba836913da9f1c6
       options: >-
         -e NODE_NAME --privileged --cgroupns host
         --mount type=tmpfs,target="/home/buildifier/.local/share/containers"

.github/workflows/bazel-dependency-submission.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - 'master'
-      - 'master-private'
 
 permissions:
   contents: write

.github/workflows/ci-kickoff-manual.yml

@@ -54,6 +54,7 @@ jobs:
       commit-sha: ${{ inputs.commit-sha }}
     permissions:
       contents: read
+      id-token: write
       pull-requests: read
 
   ci-pr-only:

.github/workflows/ci-kickoff.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - master
       - 'dev-gh-*'
+      - 'public-hotfix-*'
   pull_request:
     branches-ignore:
       - hotfix-* # This is to ensure that this workflow is not triggered twice on ic-private, as it's already triggered from release-testing
@@ -17,17 +18,18 @@ concurrency:
 jobs:
   ci-main:
     name: CI Main
-    # if in the ic or ic-private repo:
-    # run on all events except for forked PRs
+    # for dfinity/ic: run on all events except for forked PRs
+    # for dfinity/ic-private: only run on PRs
     if: |
-      contains(fromJson('["dfinity/ic","dfinity/ic-private"]'), github.repository) && 
-      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+      (github.repository == 'dfinity/ic' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) ||
+      (github.repository == 'dfinity/ic-private' && github.event_name == 'pull_request')
     uses: ./.github/workflows/ci-main.yml
     secrets: inherit
     with:
       commit-sha: ${{ github.sha }}
     permissions:
       contents: read
+      id-token: write
       pull-requests: read
 
   ci-pr-only:

.github/workflows/ci-main.yml

@@ -10,6 +10,7 @@ on:
 
 permissions:
   contents: read
+  id-token: write
   pull-requests: read
 
 env:
@@ -27,7 +28,7 @@ jobs:
       labels: dind-large
       group: dm2
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:18d23aef1f5e9e7e1eef94c32563f8ed15531ae79065bb00bb5206a643fc49fe
+      image: ghcr.io/dfinity/ic-build@sha256:0a0cff0b12c7586c2f312c739176924edf0c9e71df92f382eba836913da9f1c6
       options: >-
          -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 90
@@ -38,14 +39,31 @@ jobs:
         with:
           fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
           ref: ${{ inputs.commit-sha }}
+      - name: Filter fuzzer related files
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          ref: ${{ inputs.commit-sha }}
+          filters: |
+            fuzzers:
+              - ".github/workflows/ci-main.yml"
+              - "ci/container/TAG"
+              - "bin/fuzzing/*.sh"
+              - "**/BUILD.bazel"
+              - "**/*.bzl"
+              - "**/*.rs"
       - uses: ./.github/actions/netrc
+        if: github.event_name != 'pull_request' || steps.filter.outputs.fuzzers == 'true'
       - name: Run Libfuzzer targets
         uses: ./.github/actions/bazel
+        if: github.event_name != 'pull_request' || steps.filter.outputs.fuzzers == 'true'
         with:
           invocation-names: libfuzzer
           run: ./bin/fuzzing/run-all-fuzzers.sh --libfuzzer 100
       - name: Run AFL targets
         uses: ./.github/actions/bazel
+        if: github.event_name != 'pull_request' || steps.filter.outputs.fuzzers == 'true'
         with:
           invocation-names: afl
           run: ./bin/fuzzing/run-all-fuzzers.sh --afl 10
@@ -66,7 +84,7 @@ jobs:
 
           # List of "protected" branches, i.e. branches (not necessarily "protected" in the GitHub sense) where we need
           # the full build to occur (including versioning)
-          protected_branches=("^master$" "^rc--" "^hotfix-" "^master-private$")
+          protected_branches=("^master$" "^rc--" "^hotfix-" "^public-hotfix-")
           for pattern in "${protected_branches[@]}"; do
               if [[ "$BRANCH_NAME" =~ $pattern ]]; then
                   is_protected_branch="true"
@@ -123,7 +141,6 @@ jobs:
               skip_buf_checks="false"
           fi
 
-
           echo "| config | value |" >> "$GITHUB_STEP_SUMMARY"
           echo "| --- | --- |" >> "$GITHUB_STEP_SUMMARY"
 
@@ -187,7 +204,7 @@ jobs:
               )
             if [[ "$CI_EVENT_NAME" == 'merge_group' ]]; then
                 bazel_args+=( --test_timeout_filters=short,moderate --flaky_test_attempts=3 )
-            elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
+            elif [[ $BRANCH_NAME =~ ^(hotfix-|public-hotfix-) ]]; then
                 bazel_args+=( --test_timeout_filters=short,moderate )
             else
                 bazel_args+=( --keep_going )
@@ -249,12 +266,18 @@ jobs:
             bazel_do build
             bazel_do test
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        if: needs.config.outputs.release-build == 'true'
+        with:
+          role-session-name: "GitHub-Actions_ic_upload-artifacts"
+          role-to-assume: ${{ vars.AWS_ASSUME_ROLE }}
+          aws-region: eu-central-1
+
       - name: Upload artifacts
         uses: ./.github/actions/upload-artifacts
         if: needs.config.outputs.release-build == 'true'
         env:
-          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
-          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
           CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
           CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
@@ -314,13 +337,19 @@ jobs:
               --test_tag_filters=test_all_platforms \
               //...
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        if: needs.config.outputs.release-build == 'true'
+        with:
+          role-session-name: "GitHub-Actions_ic_upload-artifacts"
+          role-to-assume: ${{ vars.AWS_ASSUME_ROLE }}
+          aws-region: eu-central-1
+
       - name: Upload artifacts
         # NOTE: GHA output quirk, 'true' is a string
         if: ${{ needs.config.outputs.full_macos_build == 'true' && needs.config.outputs.release-build == 'true' }}
         uses: ./.github/actions/upload-artifacts
         env:
-          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
-          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
           CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
           CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
@@ -452,11 +481,16 @@ jobs:
 
           echo bundledir="$bundledir" >> "$GITHUB_OUTPUT"
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        with:
+          role-session-name: "GitHub-Actions_ic_upload-artifacts"
+          role-to-assume: ${{ vars.AWS_ASSUME_ROLE }}
+          aws-region: eu-central-1
+
       - name: Upload artifacts
         uses: ./.github/actions/upload-artifacts
         env:
-          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
-          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
           CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
           CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
@@ -670,7 +704,8 @@ jobs:
           contains(github.event.pull_request.labels.*.name, 'CI_RUN_CARGO_JOBS') ||
           env.BRANCH_NAME == 'master' ||
           startsWith(env.BRANCH_NAME, 'rc--') ||
-          startsWith(env.BRANCH_NAME, 'hotfix-')
+          startsWith(env.BRANCH_NAME, 'hotfix-') ||
+          startsWith(env.BRANCH_NAME, 'public-hotfix-')
         shell: bash
         run: |
           set -eExuo pipefail
@@ -732,6 +767,7 @@ jobs:
           steps.filter.outputs.container-run == 'true' ||
           env.BRANCH_NAME == 'master' ||
           startsWith(env.BRANCH_NAME, 'rc--') ||
-          startsWith(env.BRANCH_NAME, 'hotfix-')
+          startsWith(env.BRANCH_NAME, 'hotfix-') ||
+          startsWith(env.BRANCH_NAME, 'public-hotfix-')
         run: |
           ./ci/container/container-run.sh ${{ matrix.command }}

.github/workflows/ci-pr-only.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: &dind-small-setup
       labels: dind-small
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:18d23aef1f5e9e7e1eef94c32563f8ed15531ae79065bb00bb5206a643fc49fe
+      image: ghcr.io/dfinity/ic-build@sha256:0a0cff0b12c7586c2f312c739176924edf0c9e71df92f382eba836913da9f1c6
       options: >-
          -e NODE_NAME --mount type=tmpfs,target="/tmp/containers"
     steps: