Merge pull request #211 from PolicyEngine/auto/update-policyengine-us… #211
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to GCP Cloud Run | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| skip_staging: | |
| description: 'Skip staging and deploy directly to production' | |
| required: false | |
| default: false | |
| type: boolean | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "src/**" | |
| - "docs/**" | |
| - ".github/workflows/deploy.yml" | |
| - "terraform/**" | |
| - "alembic/**" | |
| - "Dockerfile" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| permissions: | |
| id-token: write | |
| contents: write | |
| concurrency: | |
| group: deploy-main | |
| cancel-in-progress: false | |
| jobs: | |
| # ── Stage 1: Test ────────────────────────────────────────────── | |
| test: | |
| name: Test | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'push' || github.event.head_commit.message != 'Update package version' }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync --extra dev | |
| - name: Run tests | |
| run: uv run pytest -v -m "not integration and not staging" | |
| # ── Stage 2: Build + Migrate + Infra (parallel) ─────────────── | |
| migrate-staging: | |
| name: Migrate staging database | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: ${{ !inputs.skip_staging }} | |
| environment: staging | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync | |
| - name: Run database migrations | |
| env: | |
| SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} | |
| run: uv run alembic upgrade head | |
| migrate-production: | |
| name: Migrate production database | |
| runs-on: ubuntu-latest | |
| needs: test | |
| environment: production | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync | |
| - name: Run database migrations | |
| env: | |
| SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} | |
| run: uv run alembic upgrade head | |
| build: | |
| name: Build Docker image | |
| runs-on: ubuntu-latest | |
| needs: test | |
| environment: production | |
| env: | |
| IMAGE_URL: ${{ vars.GCP_REGION }}-docker.pkg.dev/${{ vars.GCP_PROJECT_ID }}/${{ vars.PROJECT_NAME }}/${{ vars.PROJECT_NAME }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Configure Docker for Artifact Registry | |
| run: gcloud auth configure-docker ${{ vars.GCP_REGION }}-docker.pkg.dev | |
| - name: Build and push Docker image | |
| run: | | |
| docker build -t $IMAGE_URL:${{ github.sha }} . | |
| docker tag $IMAGE_URL:${{ github.sha }} $IMAGE_URL:latest | |
| docker push $IMAGE_URL:${{ github.sha }} | |
| docker push $IMAGE_URL:latest | |
| infra: | |
| name: Apply infrastructure | |
| runs-on: ubuntu-latest | |
| needs: test | |
| environment: production | |
| env: | |
| TF_IN_AUTOMATION: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: 1.6.0 | |
| - name: Terraform init | |
| working-directory: ./terraform | |
| run: terraform init -input=false | |
| - name: Terraform plan | |
| working-directory: ./terraform | |
| env: | |
| TF_VAR_supabase_url: ${{ secrets.SUPABASE_URL }} | |
| TF_VAR_supabase_key: ${{ secrets.SUPABASE_KEY }} | |
| TF_VAR_supabase_secret_key: ${{ secrets.SUPABASE_SECRET_KEY }} | |
| TF_VAR_supabase_db_url: ${{ secrets.SUPABASE_DB_URL }} | |
| TF_VAR_logfire_token: ${{ secrets.LOGFIRE_TOKEN }} | |
| TF_VAR_logfire_environment: prod | |
| TF_VAR_modal_token_id: ${{ secrets.MODAL_TOKEN_ID }} | |
| TF_VAR_modal_token_secret: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| TF_VAR_modal_environment: main | |
| run: terraform plan -out=tfplan -input=false | |
| - name: Terraform apply | |
| working-directory: ./terraform | |
| run: terraform apply -input=false tfplan | |
| # ── Stage 3: Deploy to staging ───────────────────────────────── | |
| setup-modal-environments: | |
| name: Setup Modal environments | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: ${{ !inputs.skip_staging }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync | |
| - name: Create staging environment | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| run: | | |
| chmod +x .github/scripts/*.sh | |
| .github/scripts/modal-setup-environments.sh | |
| deploy-staging-modal: | |
| name: Deploy Modal to staging | |
| runs-on: ubuntu-latest | |
| needs: [migrate-staging, setup-modal-environments] | |
| if: ${{ !inputs.skip_staging }} | |
| environment: staging | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync | |
| - name: Sync secrets to staging | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} | |
| SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} | |
| SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} | |
| LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| chmod +x .github/scripts/*.sh | |
| .github/scripts/modal-sync-secrets.sh staging staging | |
| - name: Extract package versions | |
| id: versions | |
| run: | | |
| chmod +x .github/scripts/modal-extract-versions.sh | |
| .github/scripts/modal-extract-versions.sh . | |
| - name: Deploy versioned Modal simulation app to staging | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| POLICYENGINE_US_VERSION: ${{ steps.versions.outputs.us_version }} | |
| POLICYENGINE_UK_VERSION: ${{ steps.versions.outputs.uk_version }} | |
| run: | | |
| chmod +x .github/scripts/modal-deploy-versioned.sh | |
| .github/scripts/modal-deploy-versioned.sh staging | |
| - name: Deploy agent sandbox to staging | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| run: | | |
| uv run modal deploy --env=staging src/policyengine_api/agent_sandbox.py | |
| deploy-staging-cloudrun: | |
| name: Deploy Cloud Run staging | |
| runs-on: ubuntu-latest | |
| needs: [build, infra, deploy-staging-modal] | |
| if: ${{ !inputs.skip_staging }} | |
| environment: staging | |
| outputs: | |
| staging_url: ${{ steps.get-staging-url.outputs.url }} | |
| env: | |
| IMAGE_URL: ${{ vars.GCP_REGION }}-docker.pkg.dev/${{ vars.GCP_PROJECT_ID }}/${{ vars.PROJECT_NAME }}/${{ vars.PROJECT_NAME }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Deploy staging revision (no traffic) | |
| run: | | |
| gcloud run deploy ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --image=$IMAGE_URL:${{ github.sha }} \ | |
| --tag=staging \ | |
| --no-traffic \ | |
| --update-env-vars=MODAL_ENVIRONMENT=staging,LOGFIRE_ENVIRONMENT=staging,SUPABASE_URL=${{ secrets.SUPABASE_URL }},SUPABASE_KEY=${{ secrets.SUPABASE_KEY }},SUPABASE_SECRET_KEY=${{ secrets.SUPABASE_SECRET_KEY }},SUPABASE_DB_URL=${{ secrets.SUPABASE_DB_URL }} | |
| - name: Get staging URL | |
| id: get-staging-url | |
| run: | | |
| STAGING_URL=$(gcloud run services describe ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --format=json | jq -r '.status.traffic[] | select(.tag=="staging") | .url') | |
| echo "url=$STAGING_URL" >> "$GITHUB_OUTPUT" | |
| echo "Staging URL: $STAGING_URL" | |
| - name: Health check staging | |
| run: | | |
| chmod +x .github/scripts/*.sh | |
| .github/scripts/health-check.sh "${{ steps.get-staging-url.outputs.url }}/health" | |
| # ── Stage 4: Integration tests against staging ───────────────── | |
| integration-tests: | |
| name: Integration tests (staging) | |
| runs-on: ubuntu-latest | |
| needs: deploy-staging-cloudrun | |
| if: ${{ !inputs.skip_staging }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync --extra dev | |
| - name: Run staging integration tests | |
| env: | |
| API_BASE_URL: ${{ needs.deploy-staging-cloudrun.outputs.staging_url }} | |
| run: uv run pytest tests/test_staging_api.py -v | |
| # ── Stage 5: Deploy to production ────────────────────────────── | |
| deploy-prod-modal: | |
| name: Deploy Modal to production | |
| runs-on: ubuntu-latest | |
| needs: [migrate-production, integration-tests] | |
| if: | | |
| always() && | |
| needs.migrate-production.result == 'success' && | |
| (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') | |
| environment: production | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Setup Python | |
| run: uv python install 3.13 | |
| - name: Sync dependencies | |
| run: uv sync | |
| - name: Sync secrets to production | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} | |
| SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} | |
| SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} | |
| LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| chmod +x .github/scripts/*.sh | |
| .github/scripts/modal-sync-secrets.sh main prod | |
| - name: Extract package versions | |
| id: prod-versions | |
| run: | | |
| chmod +x .github/scripts/modal-extract-versions.sh | |
| .github/scripts/modal-extract-versions.sh . | |
| - name: Deploy versioned Modal simulation app to production | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| POLICYENGINE_US_VERSION: ${{ steps.prod-versions.outputs.us_version }} | |
| POLICYENGINE_UK_VERSION: ${{ steps.prod-versions.outputs.uk_version }} | |
| run: | | |
| chmod +x .github/scripts/modal-deploy-versioned.sh | |
| .github/scripts/modal-deploy-versioned.sh main | |
| - name: Deploy agent sandbox to production | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| run: | | |
| uv run modal deploy --env=main src/policyengine_api/agent_sandbox.py | |
| - name: Validate Modal secrets | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| POLICYENGINE_US_VERSION: ${{ steps.prod-versions.outputs.us_version }} | |
| POLICYENGINE_UK_VERSION: ${{ steps.prod-versions.outputs.uk_version }} | |
| run: | | |
| US_SAFE="${POLICYENGINE_US_VERSION//./-}" | |
| UK_SAFE="${POLICYENGINE_UK_VERSION//./-}" | |
| APP_NAME="policyengine-v2-us${US_SAFE}-uk${UK_SAFE}" | |
| echo "Validating Modal secrets on ${APP_NAME}..." | |
| result=$(uv run python -c " | |
| import modal | |
| f = modal.Function.from_name('${APP_NAME}', 'validate_secrets') | |
| result = f.remote() | |
| import json | |
| print(json.dumps(result)) | |
| ") | |
| echo "$result" | |
| if echo "$result" | grep -q '"status": "error"'; then | |
| echo "::error::Modal secrets validation failed" | |
| exit 1 | |
| fi | |
| echo "Modal secrets validated" | |
| deploy-prod-cloudrun: | |
| name: Deploy to Cloud Run production | |
| runs-on: ubuntu-latest | |
| needs: [deploy-prod-modal, build, infra] | |
| if: | | |
| always() && | |
| needs.deploy-prod-modal.result == 'success' && | |
| needs.build.result == 'success' && | |
| needs.infra.result == 'success' | |
| environment: production | |
| env: | |
| IMAGE_URL: ${{ vars.GCP_REGION }}-docker.pkg.dev/${{ vars.GCP_PROJECT_ID }}/${{ vars.PROJECT_NAME }}/${{ vars.PROJECT_NAME }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Deploy canary (no traffic) | |
| run: | | |
| gcloud run deploy ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --image=$IMAGE_URL:${{ github.sha }} \ | |
| --tag=canary \ | |
| --no-traffic \ | |
| --update-env-vars=MODAL_ENVIRONMENT=main,LOGFIRE_ENVIRONMENT=prod | |
| - name: Smoke test canary | |
| run: | | |
| CANARY_URL=$(gcloud run services describe ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --format=json | jq -r '.status.traffic[] | select(.tag=="canary") | .url') | |
| echo "Canary URL: $CANARY_URL" | |
| chmod +x .github/scripts/*.sh | |
| .github/scripts/health-check.sh "$CANARY_URL/health" | |
| - name: Shift traffic to new revision | |
| run: | | |
| gcloud run services update-traffic ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --to-latest | |
| - name: Get production URL | |
| run: | | |
| API_URL=$(gcloud run services describe ${{ vars.API_SERVICE_NAME }} \ | |
| --region=${{ vars.GCP_REGION }} \ | |
| --format='value(status.url)') | |
| echo "API: $API_URL" | |
| echo "Health: $API_URL/health" | |
| echo "Docs: $API_URL/docs" |