chore: update CI (#240)

dbarrosop · web-flow · commit c2b5a499af5f · 2025-05-30T11:39:15.000+02:00
### **PR Type** Enhancement ___ ### **Description** • Enhanced CI security with permission checks and safe_to_test label • Added AI-powered PR review workflow with Claude integration • Migrated to Blacksmith runners and S3-based Nix cache • Improved AWS integration with OIDC and ECR authentication ___ ### **Changes walkthrough** 📝 <table><thead><tr><th></th><th align="left">Relevant files</th></tr></thead><tbody><tr><td><strong>Enhancement</strong></td><td><table> <tr> <td> <details> <summary><strong>checks.yaml</strong><dd><code>Enhanced security with permission checks</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> .github/workflows/checks.yaml • Changed trigger from <code>pull_request</code> to <code>pull_request_target</code> for <br>security<br> • Added permission check job requiring <code>safe_to_test</code> label or <br>trusted user<br> • Added secrets for AWS account, Nix cache keys<br> • Added <br>job to automatically remove <code>safe_to_test</code> label after use </details> </td> <td><a href="https://github.com/nhost/hasura-storage/pull/240/files#diff-4af11422a4987e947e5a47adead7a30d32cdb2db82e2d3fe36f8e6cbe84d5ac5">+36/-2</a>&nbsp; &nbsp; </td> </tr> <tr> <td> <details> <summary><strong>gen_ai_review.yaml</strong><dd><code>Added AI review workflow</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> .github/workflows/gen_ai_review.yaml • New workflow for AI-powered PR reviews using Claude Sonnet model<br> • <br>Triggers on PR events and issue comments<br> • Configured with OpenAI and <br>Anthropic API keys<br> • Ignores generated files and vendor directories </details> </td> <td><a href="https://github.com/nhost/hasura-storage/pull/240/files#diff-d1e4c772e0acb5ce4891df2dd94ba58ffaf6393e8f75493ec7e10cbce1c4992c">+27/-0</a>&nbsp; &nbsp; </td> </tr> <tr> <td> <details> <summary><strong>wf_build_artifacts.yaml</strong><dd><code>Migrated to Blacksmith runners and S3 cache</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> .github/workflows/wf_build_artifacts.yaml • Migrated from Ubuntu runners to Blacksmith ARM/x64 runners<br> • <br>Replaced Magic Nix Cache with S3-based cache using AWS credentials<br> • <br>Added AWS OIDC configuration and ECR integration<br> • Improved <br>architecture detection and artifact naming </details> </td> <td><a href="https://github.com/nhost/hasura-storage/pull/240/files#diff-63d6f3e2e5894df4d88cf2c82b3d0c90e7b0b7c8ca9da25a3d46f35f15270fb0">+28/-22</a>&nbsp; </td> </tr> <tr> <td> <details> <summary><strong>wf_check.yaml</strong><dd><code>Enhanced checks with AWS integration</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> .github/workflows/wf_check.yaml • Migrated to Blacksmith ARM runner with reduced timeout<br> • Added AWS <br>OIDC and ECR login configuration<br> • Replaced Magic Nix Cache with <br>S3-based cache<br> • Enhanced build verification for both Go and Node <br>components </details> </td> <td><a href="https://github.com/nhost/hasura-storage/pull/240/files#diff-c2ecea6736037ba6304681d744d612d44d7681788d2fbc58ce223cf52ecefa43">+47/-25</a>&nbsp; </td> </tr> </table></td></tr><tr><td><strong>Configuration changes</strong></td><td><table> <tr> <td> <details> <summary><strong>release.yaml</strong><dd><code>Updated release workflow secrets</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> .github/workflows/release.yaml • Added secrets for AWS account and Nix cache keys<br> • Removed <code>DOCKER</code> <br>input parameter from build artifacts call </details> </td> <td><a href="https://github.com/nhost/hasura-storage/pull/240/files#diff-e426ed45842837026e10e66af23d9c7077e89eacbe6958ce7cb991130ad05ada">+9/-1</a>&nbsp; &nbsp; &nbsp; </td> </tr> </table></td></tr></tr></tbody></table> ___ > <details> <summary> Need help?</summary><li>Type <code>/help how to ...</code> in the comments thread for any questions about PR-Agent usage.</li><li>Check out the <a href="https://qodo-merge-docs.qodo.ai/usage-guide/">documentation</a> for more information.</li></details>
diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml
@@ -1,22 +1,56 @@
 ---
 name: "check and build"
 on:
-  pull_request:
+  pull_request_target:
   push:
     branches:
       - main
 
 jobs:
+  check-permissions:
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "github.event_name: ${{ github.event_name }}"
+          echo "github.event.pull_request.author_association: ${{ github.event.pull_request.author_association }}"
+      - name: "This task will run and fail if user has no permissions and label safe_to_test isn't pressent"
+        if: "github.event_name == 'pull_request_target' && ! ( contains(github.event.pull_request.labels.*.name, 'safe_to_test') || contains(fromJson('[\"OWNER\", \"MEMBER\", \"COLLABORATOR\"]'), github.event.pull_request.author_association) )"
+        run: |
+          exit 1
+
   tests:
     uses: ./.github/workflows/wf_check.yaml
+    needs:
+      - check-permissions
     with:
       NAME: hasura-storage
       GIT_REF: ${{ github.event.pull_request.head.sha }}
+    secrets:
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_PRODUCTION_CORE_ACCOUNT_ID }}
+      NIX_CACHE_PUB_KEY: ${{ secrets.NIX_CACHE_PUB_KEY }}
+      NIX_CACHE_PRIV_KEY: ${{ secrets.NIX_CACHE_PRIV_KEY }}
 
   build_artifacts:
     uses: ./.github/workflows/wf_build_artifacts.yaml
+    needs:
+      - check-permissions
     with:
       NAME: hasura-storage
       VERSION: 0.0.0-dev # hardcoded to avoid rebuilding
-      DOCKER: true
       GIT_REF: ${{ github.event.pull_request.head.sha }}
+    secrets:
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_PRODUCTION_CORE_ACCOUNT_ID }}
+      NIX_CACHE_PUB_KEY: ${{ secrets.NIX_CACHE_PUB_KEY }}
+      NIX_CACHE_PRIV_KEY: ${{ secrets.NIX_CACHE_PRIV_KEY }}
+
+  remove_label:
+    runs-on: ubuntu-latest
+    needs:
+      - check-permissions
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: |
+            safe_to_test
+    if: contains(github.event.pull_request.labels.*.name, 'safe_to_test')
diff --git a/.github/workflows/gen_ai_review.yaml b/.github/workflows/gen_ai_review.yaml
@@ -0,0 +1,27 @@
+---
+name: "gen: AI review"
+on:
+  pull_request:
+    types: [opened, reopened, ready_for_review]
+  issue_comment:
+jobs:
+  pr_agent_job:
+    if: ${{ github.event.sender.type != 'Bot' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      issues: write
+      pull-requests: write
+    name: Run pr agent on every pull request, respond to user comments
+    steps:
+      - name: PR Agent action step
+        id: pragent
+        uses: Codium-ai/pr-agent@v0.29
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          config.max_model_tokens: 100000
+          config.model: "anthropic/claude-sonnet-4-20250514"
+          config.model_turbo: "anthropic/claude-sonnet-4-20250514"
+          ignore.glob: "['vendor/**','**/client_gen.go','**/models_gen.go','**/generated.go','**/*.gen.go']"
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -9,13 +9,21 @@ jobs:
     uses: ./.github/workflows/wf_check.yaml
     with:
       NAME: hasura-storage
+    secrets:
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_PRODUCTION_CORE_ACCOUNT_ID }}
+      NIX_CACHE_PUB_KEY: ${{ secrets.NIX_CACHE_PUB_KEY }}
+      NIX_CACHE_PRIV_KEY: ${{ secrets.NIX_CACHE_PRIV_KEY }}
+
 
   build_artifacts:
     uses: ./.github/workflows/wf_build_artifacts.yaml
     with:
       NAME: hasura-storage
       VERSION: ${{ github.ref_name }}
-      DOCKER: true
+    secrets:
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_PRODUCTION_CORE_ACCOUNT_ID }}
+      NIX_CACHE_PUB_KEY: ${{ secrets.NIX_CACHE_PUB_KEY }}
+      NIX_CACHE_PRIV_KEY: ${{ secrets.NIX_CACHE_PRIV_KEY }}
 
   push_docker_image:
     uses: ./.github/workflows/wf_docker_push_image.yaml
diff --git a/.github/workflows/wf_build_artifacts.yaml b/.github/workflows/wf_build_artifacts.yaml
@@ -15,20 +15,23 @@ on:
         type: string
         default: "."
         required: false
-      DOCKER:
-        type: boolean
+    secrets:
+      AWS_ACCOUNT_ID:
+        required: true
+      NIX_CACHE_PUB_KEY:
+        required: true
+      NIX_CACHE_PRIV_KEY:
         required: true
 
 jobs:
   artifacts:
-    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
       id-token: write
       contents: write
 
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-arm64-22.04-2cores]
+        os: [blacksmith-2vcpu-ubuntu-2404-arm, blacksmith-2vcpu-ubuntu-2404]
       fail-fast: true
 
     runs-on: ${{ matrix.os }}
@@ -41,6 +44,12 @@ jobs:
         fetch-depth: 0
         ref: ${{ inputs.GIT_REF }}
 
+    - name: Configure aws
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-nhost-${{ github.event.repository.name }}
+        aws-region: eu-central-1
+
     - uses: cachix/install-nix-action@v31
       with:
         install_url: "https://releases.nixos.org/nix/nix-2.22.3/install"
@@ -49,20 +58,15 @@ jobs:
           experimental-features = nix-command flakes
           sandbox = false
           access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-          substituters = https://cache.nixos.org/?priority=40
-          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-
-    - uses: DeterminateSystems/magic-nix-cache-action@v9
-      with:
-        diagnostic-endpoint: ""
-        use-flakehub: false
+          substituters = https://cache.nixos.org/?priority=40 s3://nhost-nix-cache?region=eu-central-1&priority=50
+          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ secrets.NIX_CACHE_PUB_KEY }}
 
     - name: Compute common env vars
       id: vars
       run: |
         echo "VERSION=$(make get-version VER=${{ inputs.VERSION }})" >> $GITHUB_OUTPUT
-        echo "ARCHITECTURE=$(uname -m)" >> $GITHUB_OUTPUT
-
+        ARCH=$([ "${{ runner.arch }}" == "X64" ] && echo "x86_64" || echo "aarch64")
+        echo "ARCH=${ARCH}" >> $GITHUB_OUTPUT
     - name: "Build artifact"
       run: |
         make build
@@ -71,24 +75,26 @@ jobs:
     - name: "Push artifact to artifact repository"
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ inputs.NAME }}-artifact-${{ steps.vars.outputs.ARCHITECTURE }}-${{ steps.vars.outputs.VERSION }}
-        path: result/bin/hasura-storage
+        name: ${{ inputs.NAME }}-artifact-${{ steps.vars.outputs.ARCH }}-${{ steps.vars.outputs.VERSION }}
+        path: result.zip
         retention-days: 7
 
     - name: "Build docker image"
       run: |
+        sudo chmod 755 /run/containers
+        sudo mkdir -p "/run/containers/$(id -u runner)"
+        sudo chown runner: "/run/containers/$(id -u runner)"
         make build-docker-image
-      if: ${{ ( inputs.DOCKER ) }}
 
     - name: "Push docker image to artifact repository"
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ inputs.NAME }}-docker-image-${{ steps.vars.outputs.ARCHITECTURE }}-${{ steps.vars.outputs.VERSION }}
+        name: ${{ inputs.NAME }}-docker-image-${{ steps.vars.outputs.ARCH }}-${{ steps.vars.outputs.VERSION }}
         path: result
         retention-days: 7
-      if: ${{ ( inputs.DOCKER ) }}
 
-    # - name: "Cache build environment"
-    #   run: |
-    #    # DEPENDENCIES=($(nix-store --query --requisites --include-outputs ${{ steps.verify-build.outputs.DERIVATION_PATH }}))
-    #    nix copy --to http://127.0.0.1:37515 "${DEPENDENCIES[@]}" || exit 0
+    - name: "Cache build"
+      run: |
+        nix store sign --key-file <(echo "${{ secrets.NIX_CACHE_PRIV_KEY }}") --all
+        find /nix/store -maxdepth 1 -name "*-*" -type d -exec nix copy --to s3://nhost-nix-cache\?region=eu-central-1 {} \;
+      if: always()
diff --git a/.github/workflows/wf_check.yaml b/.github/workflows/wf_check.yaml
@@ -8,24 +8,42 @@ on:
       GIT_REF:
         type: string
         required: false
+    secrets:
+      AWS_ACCOUNT_ID:
+        required: true
+      NIX_CACHE_PUB_KEY:
+        required: true
+      NIX_CACHE_PRIV_KEY:
+        required: true
 
 jobs:
   tests:
-    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
+    timeout-minutes: 30
+
     permissions:
       id-token: write
       contents: write
 
-
-    runs-on: ubuntu-22.04
-    timeout-minutes: 180
-
     steps:
     - name: "Check out repository"
       uses: actions/checkout@v4
       with:
         fetch-depth: 0
         ref: ${{ inputs.GIT_REF }}
+        actions: read
+
+    - name: Configure aws
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-nhost-${{ github.event.repository.name }}
+        aws-region: eu-central-1
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+      with:
+        mask-password: 'true'
 
     - uses: cachix/install-nix-action@v31
       with:
@@ -35,39 +53,43 @@ jobs:
           experimental-features = nix-command flakes
           sandbox = false
           access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-          substituters = https://cache.nixos.org/?priority=40
-          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-
-    - uses: DeterminateSystems/magic-nix-cache-action@v9
-      with:
-        diagnostic-endpoint: ""
-        use-flakehub: false
+          substituters = https://cache.nixos.org/?priority=40 s3://nhost-nix-cache?region=eu-central-1&priority=50
+          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ secrets.NIX_CACHE_PUB_KEY }}
 
     - name: "Verify if we need to build"
       id: verify-build
       run: |
-        export drvPath=$(make check-dry-run)
-        echo "drvPath - $drvPath"
-        nix store verify --no-trust --store http://127.0.0.1:37515 $drvPath \
-          && export BUILD_NEEDED=no \
-          || export BUILD_NEEDED=yes
+        export drvPathGo=$(make check-dry-run-go)
+        export drvPathNode=$(make check-dry-run-node)
+
+        nix store verify --no-trust --store s3://nhost-nix-cache\?region=eu-central-1 $drvPathGo \
+          && export GO_BUILD_NEEDED=no \
+          || export GO_BUILD_NEEDED=yes
+
+        nix store verify --no-trust --store s3://nhost-nix-cache\?region=eu-central-1 $drvPathNode \
+          && export NODE_BUILD_NEEDED=no \
+          || export NODE_BUILD_NEEDED=yes
+
+        # Set BUILD_NEEDED to yes if either Go or Node build is needed
+        if [ "$GO_BUILD_NEEDED" = "yes" ] || [ "$NODE_BUILD_NEEDED" = "yes" ]; then
+          export BUILD_NEEDED=yes
+        else
+          export BUILD_NEEDED=no
+        fi
 
         echo BUILD_NEEDED=$BUILD_NEEDED >> $GITHUB_OUTPUT
-        echo DERIVATION_PATH=$drvPath >> $GITHUB_OUTPUT
 
     - name: "Start containters for integration tests"
       run: |
-        make build-docker-image
-        nix develop -c make dev-env-up
+        nix develop .\#${{ inputs.NAME }} -c make dev-env-up
       if: ${{ steps.verify-build.outputs.BUILD_NEEDED == 'yes' }}
 
     - name: "Run checks"
       run: make check
       if: ${{ steps.verify-build.outputs.BUILD_NEEDED == 'yes' }}
 
-    - name: "Cache checks"
+    - name: "Cache build"
       run: |
-       # DEPENDENCIES=($(nix-store --query --requisites --include-outputs ${{ steps.verify-build.outputs.DERIVATION_PATH }}))
-       # nix copy --to http://127.0.0.1:37515 "${DEPENDENCIES[@]}" || exit 0
-       nix copy --to http://127.0.0.1:37515 ${{ steps.verify-build.outputs.DERIVATION_PATH }}|| exit 0
-      if: ${{ steps.verify-build.outputs.BUILD_NEEDED == 'yes' }}
+        nix store sign --key-file <(echo "${{ secrets.NIX_CACHE_PRIV_KEY }}") --all
+        find /nix/store -maxdepth 1 -name "*-*" -type d -exec nix copy --to s3://nhost-nix-cache\?region=eu-central-1 {} \;
+      if: always()
diff --git a/flake.nix b/flake.nix
@@ -104,6 +104,8 @@
               ccls
             ] ++ buildInputs ++ nativeBuildInputs ++ checkDeps;
           };
+
+          hasura-storage = default;
         };
 
         packages = flake-utils.lib.flattenTree rec {
