Publish canonical release manifests for US data releases #200
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR checks | |
| on: | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: pr-checks-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-fork: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if PR is from fork | |
| run: | | |
| if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then | |
| echo "::error::PRs must be from branches in PolicyEngine/policyengine-us-data, not forks." | |
| exit 1 | |
| fi | |
| check-lock-freshness: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v5 | |
| - name: Check lock file is up-to-date | |
| run: | | |
| uv lock --locked || { | |
| echo "::error::uv.lock is outdated. Run 'uv lock' and commit." | |
| exit 1 | |
| } | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: pip install ruff>=0.9.0 | |
| - run: ruff format --check . | |
| check-changelog: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v5 | |
| - run: uv sync --dev | |
| - name: Check for changelog fragment | |
| run: uv run towncrier check --compare-with origin/main | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v5 | |
| - run: uv sync --dev | |
| - name: Run unit tests with coverage | |
| env: | |
| HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} | |
| run: > | |
| uv run pytest tests/unit/ | |
| --cov=policyengine_us_data | |
| --cov-report=xml | |
| -v | |
| - name: Upload coverage to Codecov | |
| if: always() | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: coverage.xml | |
| flags: unit | |
| fail_ci_if_error: false | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| smoke-test: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - run: python -m pip install . | |
| - run: python -c "import policyengine_us_data; print('OK')" | |
| - run: python -c "from policyengine_core.data import Dataset; print('OK')" | |
| docs-build: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| - uses: astral-sh/setup-uv@v5 | |
| - run: uv sync --dev | |
| - name: Test documentation builds | |
| run: uv run make documentation | |
| decide-test-scope: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| outputs: | |
| run_integration: ${{ steps.check.outputs.run_integration }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check changed files for integration scope | |
| id: check | |
| run: | | |
| CHANGED=$(git diff --name-only origin/main...HEAD) | |
| if echo "$CHANGED" | grep -qE '^(policyengine_us_data/|modal_app/|tests/integration/)'; then | |
| echo "run_integration=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_integration=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint, decide-test-scope] | |
| if: needs.decide-test-scope.outputs.run_integration == 'true' | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - run: pip install modal | |
| - name: Build datasets and run integration tests on Modal | |
| run: | | |
| STAGE_ARGS="" | |
| if git diff --name-only origin/main...HEAD | grep -qx 'pyproject.toml'; then | |
| VERSION=$(python -c "from pathlib import Path; import tomllib; print(tomllib.load(Path('pyproject.toml').open('rb'))['project']['version'])") | |
| STAGE_ARGS="--upload --stage-only --run-id=${VERSION}" | |
| { | |
| echo "## Release Artifact Staging" | |
| echo "" | |
| echo "- package version: \`${VERSION}\`" | |
| echo "- staged HF prefix: \`staging/${VERSION}/\`" | |
| echo "- promote with: \`uv run python policyengine_us_data/storage/upload_completed_datasets.py --promote-only --run-id=${VERSION} --version=${VERSION}\`" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| modal run modal_app/data_build.py \ | |
| --branch=${{ github.head_ref || github.ref_name }} \ | |
| ${STAGE_ARGS} |