[dsv4-umbp-hicache-amd-rocm720] #7255
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Test ROCm 7.2 (AMD) | |
| # Dynamic run-name for /rerun-stage commands to enable URL lookup | |
| # Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs | |
| run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }} | |
| on: | |
| schedule: | |
| - cron: '30 17 * * *' | |
| # Temporarily disabled: every-6-hours dsv4 accuracy run. Daily '30 17 * * *' | |
| # cron still runs the full suite (including dsv4). Re-enable to restore it. | |
| # - cron: '0 */6 * * *' # Every 6 hours (UTC): target dsv4 accuracy jobs only | |
| # push: | |
| # branches: [ main ] | |
| # paths: | |
| # - "python/**" | |
| # - "scripts/ci/**" | |
| # - "test/**" | |
| # - "python/sglang/kernels/aot/**" | |
| # - ".github/workflows/pr-test-amd-rocm720.yml" | |
| # - "docker/rocm.Dockerfile" | |
| # pull_request: | |
| # paths: | |
| # - "python/**" | |
| # - "scripts/ci/**" | |
| # - "test/**" | |
| # - "python/sglang/kernels/aot/**" | |
| # - ".github/workflows/pr-test-amd-rocm720.yml" | |
| # - "docker/rocm.Dockerfile" | |
| workflow_dispatch: | |
| inputs: | |
| target_stage_select: | |
| description: "Select a stage to run from dropdown (leave empty for auto-detect)" | |
| required: false | |
| type: choice | |
| default: '' | |
| options: | |
| - '' | |
| - sgl-kernel-unit-test-amd-rocm720 | |
| - sgl-kernel-unit-test-2-gpu-amd-rocm720 | |
| - stage-a-test-1-gpu-small-amd-rocm720 | |
| - jit-kernel-unit-test-amd-rocm720 | |
| - jit-kernel-benchmark-test-amd-rocm720 | |
| - stage-b-test-1-gpu-small-amd-rocm720 | |
| - stage-b-test-1-gpu-small-amd-nondeterministic-rocm720 | |
| - stage-b-test-1-gpu-small-amd-mi35x-rocm720 | |
| - stage-b-test-1-gpu-large-amd-rocm720 | |
| - stage-b-test-2-gpu-large-amd-rocm720 | |
| - multimodal-gen-test-1-gpu-amd-rocm720 | |
| - multimodal-gen-test-2-gpu-amd-rocm720 | |
| - multimodal-gen-unit-test-amd-rocm720 | |
| - stage-c-test-large-8-gpu-amd-rocm720 | |
| - stage-c-test-large-8-gpu-amd-mi35x-rocm720 | |
| - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 | |
| - dsv4-umbp-hicache-amd-rocm720 | |
| - stage-c-test-4-gpu-amd-rocm720 | |
| - dsv4-flash-fp4-fp8-amd-rocm720 | |
| - dsv4-pro-fp4-amd-rocm720 | |
| target_stage: | |
| description: "Or type comma-separated stage names (overrides dropdown if non-empty)" | |
| required: false | |
| type: string | |
| default: "" | |
| pr_head_sha: | |
| description: "PR head SHA to checkout (for /rerun-stage on fork PRs)" | |
| required: false | |
| type: string | |
| default: "" | |
| aiter_ref: | |
| description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' | |
| required: false | |
| type: string | |
| default: '' | |
| continue_on_error: | |
| description: 'Continue on error (do not fail the workflow on test failures)' | |
| required: false | |
| type: boolean | |
| default: true | |
| run_all_tests: | |
| description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.' | |
| required: false | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| inputs: | |
| ref: | |
| description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' | |
| required: false | |
| type: string | |
| default: '' | |
| run_all_tests: | |
| description: "Run all tests (for releasing or testing purpose)" | |
| required: false | |
| type: boolean | |
| default: false | |
| aiter_ref: | |
| description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' | |
| required: false | |
| type: string | |
| default: '' | |
| continue_on_error: | |
| description: 'Continue on error (do not fail the workflow on test failures)' | |
| required: false | |
| type: boolean | |
| default: true | |
| # Mirror pr-test.yml: the chained extra suite (call-pr-test-amd-extra-rocm720 | |
| # -> pr-test-amd-extra.yml) declares actions: write / issues: read / | |
| # pull-requests: read. A called reusable workflow can only use scopes the | |
| # caller already holds, so without this block the call fails workflow | |
| # validation ("requesting actions: write... but only allowed ...none"). | |
| permissions: | |
| actions: write | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| env: | |
| AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} | |
| DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} | |
| DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} | |
| concurrency: | |
| # Scheduled and manual full runs use unique groups so they never cancel each other. | |
| # PR cancellation is kept only for pull_request events if this trigger is re-enabled. | |
| group: pr-test-amd-rocm720-${{ github.event_name == 'schedule' && format('schedule-{0}', github.run_id) || (inputs.run_all_tests || github.event_name == 'workflow_dispatch') && format('full-{0}', github.run_id) || inputs.pr_head_sha || inputs.ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| call-gate: | |
| # Runs on the daily '30 17 * * *' cron, workflow_dispatch and workflow_call. | |
| # It is skipped only on the 6h ('0 */6 * * *') cron, which cascades a skip | |
| # to check-changes and every existing test job so that cron runs the two | |
| # dsv4 jobs only. | |
| if: github.event.schedule != '0 */6 * * *' | |
| uses: ./.github/workflows/pr-gate.yml | |
| secrets: inherit | |
| check-changes: | |
| needs: [call-gate] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} | |
| sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }} | |
| jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} | |
| multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} | |
| continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Determine run mode | |
| id: run-mode | |
| run: | | |
| # Run all tests for workflow_call (when ref input is provided) | |
| # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref | |
| # Any scheduled run that reaches check-changes is the daily '30 17 * * *' cron | |
| # (the 6h '0 */6 * * *' cron skips call-gate -> check-changes), so the daily run | |
| # executes the full suite (existing jobs + dsv4), like pr-test-amd.yml. | |
| if [[ "${{ inputs.run_all_tests }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then | |
| echo "run_all_tests=true" >> $GITHUB_OUTPUT | |
| echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }}, event=${{ github.event_name }})" | |
| else | |
| echo "run_all_tests=false" >> $GITHUB_OUTPUT | |
| echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" | |
| fi | |
| - name: Set continue-on-error for schedule/full runs | |
| id: set-continue-on-error | |
| run: | | |
| # Mirror pr-test-amd.yml: continue-on-error when any of | |
| # - run_all_tests was requested (workflow_dispatch checkbox, workflow_call input, | |
| # or a scheduled run, which run-mode above rolls into run_all_tests), | |
| # - inputs.continue_on_error was explicitly set, | |
| # - or this is a scheduled run (kept as an explicit belt-and-suspenders guard). | |
| if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then | |
| echo "continue_on_error=true" >> $GITHUB_OUTPUT | |
| echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})" | |
| else | |
| echo "continue_on_error=false" >> $GITHUB_OUTPUT | |
| echo "Continue-on-error: DISABLED" | |
| fi | |
| - name: Detect file changes | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| if: steps.run-mode.outputs.run_all_tests != 'true' | |
| with: | |
| filters: | | |
| main_package: | |
| - "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)" | |
| - "python/sglang/kernels/!(*.md)" | |
| - "python/sglang/kernels/!(aot)/**/!(*.md)" | |
| - "python/pyproject_rocm.toml" | |
| - "python/pyproject_other.toml" | |
| - "scripts/ci/amd/*" | |
| - "scripts/ci/utils/*" | |
| - "test/**/!(*.md)" | |
| - ".github/workflows/pr-test-amd-rocm720.yml" | |
| sgl_kernel: | |
| - "python/sglang/kernels/aot/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)" | |
| - ".github/workflows/pr-test-amd-rocm720.yml" | |
| jit_kernel: | |
| - "python/sglang/kernels/!(*.md)" | |
| - "python/sglang/kernels/!(aot)/**" | |
| - "test/registered/kernels/**" | |
| - ".github/workflows/pr-test-amd-rocm720.yml" | |
| multimodal_gen: | |
| - "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)" | |
| - "python/sglang/cli/**" | |
| - "python/sglang/srt/observability/**" | |
| - "python/sglang/kernels/ops/diffusion/**" | |
| - "test/registered/kernels/ops/diffusion/**" | |
| - "test/registered/kernels/benchmark/diffusion/**" | |
| - "python/pyproject_rocm.toml" | |
| - "python/pyproject_other.toml" | |
| # =============================================== extra (scheduled) ==================================================== | |
| # ROCm 7.2 mirror of pr-test-amd.yml's `call-pr-test-amd-extra`: chain the | |
| # label-gated AMD extra tier into this workflow's daily schedule, but in a | |
| # ROCm 7.2 container (`rocm_version: rocm720`). On `schedule` (and | |
| # run_all_tests dispatch) the extra suite runs on `main` without the | |
| # `run-ci-extra` label (pr-gate.yml only enforces labels on pull_request | |
| # events). Targeted dispatches (target_stage set) are excluded. Not wired | |
| # into any finish aggregator so the base rocm720 run never depends on it. | |
| call-pr-test-amd-extra-rocm720: | |
| if: | | |
| (github.event_name == 'schedule' || inputs.run_all_tests == true) && | |
| !(inputs.target_stage || inputs.target_stage_select) | |
| uses: ./.github/workflows/pr-test-amd-extra.yml | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} | |
| runner_arch: mi300 | |
| rocm_version: rocm720 | |
| aiter_ref: ${{ inputs.aiter_ref }} | |
| continue_on_error: true | |
| secrets: inherit | |
| # =============================================== sgl-kernel ==================================================== | |
| sgl-kernel-unit-test-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',sgl-kernel-unit-test-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| needs.check-changes.outputs.sgl_kernel == 'true' | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 30 | |
| run: | | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_align.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_topk_softmax.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests/speculative ci_sglang python3 -m pytest test_eagle_utils.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_apply_token_bitmask_inplace.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_activation.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_topk.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_kvcacheio.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_topk_sigmoid.py | |
| docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_torch_defaults_reset.py | |
| sgl-kernel-unit-test-2-gpu-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',sgl-kernel-unit-test-2-gpu-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| needs.check-changes.outputs.sgl_kernel == 'true' | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-2gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 30 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite sgl-kernel-unit-test-2-gpu-amd | |
| # =============================================== primary ==================================================== | |
| stage-a-test-1-gpu-small-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-1-gpu-small-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 30 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| jit-kernel-unit-test-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',jit-kernel-unit-test-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| needs.check-changes.outputs.jit_kernel == 'true' | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run JIT kernel unit tests | |
| timeout-minutes: 30 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} | |
| jit-kernel-benchmark-test-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',jit-kernel-benchmark-test-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| needs.check-changes.outputs.jit_kernel == 'true' | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run JIT kernel benchmarks | |
| timeout-minutes: 30 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-benchmark-test-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} | |
| stage-b-test-1-gpu-small-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 75 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| stage-b-test-1-gpu-small-amd-nondeterministic-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-nondeterministic-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 75 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| stage-b-test-1-gpu-small-amd-mi35x-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-mi35x-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi35x-gpu-1] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 30 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| stage-b-test-1-gpu-large-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-large-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| part: [0, 1, 2] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 60 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| stage-b-test-2-gpu-large-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-2-gpu-large-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| runner: [linux-mi300-2gpu-sglang] | |
| part: [0, 1] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 120 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| multimodal-gen-test-1-gpu-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-test-1-gpu-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| runner: [linux-mi300-1gpu-sglang] | |
| part: [0, 1, 2, 3] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Download artifacts | |
| if: needs.check-changes.outputs.sgl_kernel == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: python/sglang/kernels/aot/dist/ | |
| merge-multiple: true | |
| pattern: wheel-python3.10-cuda12.9 | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion | |
| docker exec ci_sglang pip install amdsmi | |
| - name: Setup kernel caches | |
| run: | | |
| docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub | |
| # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) | |
| # This tells the test cleanup code to NOT delete downloaded models | |
| if docker exec ci_sglang test -d /sgl-data && docker exec ci_sglang mountpoint -q /sgl-data 2>/dev/null; then | |
| docker exec ci_sglang touch /sgl-data/hf-cache/.persistent_cache | |
| echo "Created .persistent_cache marker - HF cache will persist" | |
| else | |
| echo "WARNING: /sgl-data is not a mount point - models will be cleaned up after each test" | |
| fi | |
| # Check MIOpen cache (VAE convolution kernels) | |
| miopen_files=$(docker exec ci_sglang find /sgl-data/miopen-cache -name "*.udb" 2>/dev/null | wc -l || echo "0") | |
| echo "Found ${miopen_files} MIOpen cache files" | |
| - name: Diagnose HF cache and system resources | |
| run: | | |
| echo "=== System Memory Status ===" | |
| free -h | |
| echo "" | |
| echo "=== Disk Space ===" | |
| df -h /home/runner/sglang-data 2>/dev/null || df -h | |
| echo "" | |
| echo "=== HF Cache Directory Structure ===" | |
| docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" | |
| docker exec ci_sglang ls -la /sgl-data/hf-cache/hub/ 2>/dev/null || echo "HF hub cache not found" | |
| echo "" | |
| echo "=== Checking for cached diffusion models (1-GPU tests) ===" | |
| # Models used in 1-GPU tests: Wan2.1-T2V-1.3B, HunyuanVideo, Qwen-Image, FLUX.1, FLUX.2 | |
| for model in "Wan-AI--Wan2.1-T2V-1.3B-Diffusers" "tencent--HunyuanVideo" "Qwen--Qwen-Image" "black-forest-labs--FLUX.1-dev" "black-forest-labs--FLUX.2-dev"; do | |
| cache_path="/sgl-data/hf-cache/hub/models--${model}" | |
| if docker exec ci_sglang test -d "$cache_path"; then | |
| size=$(docker exec ci_sglang du -sh "$cache_path" 2>/dev/null | cut -f1) | |
| echo "✓ CACHED: $model ($size)" | |
| else | |
| echo "✗ NOT CACHED: $model" | |
| fi | |
| done | |
| echo "" | |
| echo "=== GPU Memory Status ===" | |
| docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" | |
| - name: Run diffusion server tests (1-GPU) | |
| timeout-minutes: 90 | |
| run: | | |
| # AMD CI: All 1-GPU tests except FLUX.2 (FLUX.1 covers same code path) | |
| # Tests: T2V, T2I, I2V, LoRA | |
| # | |
| # HF download env vars: | |
| # - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available) | |
| # - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings | |
| docker exec \ | |
| -e SGLANG_E2E_TOLERANCE=0.3 \ | |
| -e SGLANG_STAGE_TIME_TOLERANCE=0.2 \ | |
| -e SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=0.6 \ | |
| -e SGLANG_DENOISE_STEP_TOLERANCE=0.6 \ | |
| -e SGLANG_DENOISE_AGG_TOLERANCE=0.3 \ | |
| -e SGLANG_SKIP_CONSISTENCY=1 \ | |
| -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ | |
| -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ | |
| -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ | |
| -e HF_HUB_ENABLE_HF_TRANSFER=1 \ | |
| -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ | |
| -w /sglang-checkout/python \ | |
| ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ | |
| --suite 1-gpu \ | |
| --partition-id ${{ matrix.part }} \ | |
| --total-partitions 4 \ | |
| -k "not flux_2" | |
| # Post-test diagnostics | |
| echo "=== Post-test System Memory Status ===" | |
| free -h | |
| - name: Upload diffusion failure artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: diffusion-failures-amd-rocm720-1gpu-${{ matrix.part }}-${{ github.run_attempt }} | |
| path: diffusion-failures/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| multimodal-gen-test-2-gpu-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-test-2-gpu-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| runner: [linux-mi300-2gpu-sglang] | |
| part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py) | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Download artifacts | |
| if: needs.check-changes.outputs.sgl_kernel == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: python/sglang/kernels/aot/dist/ | |
| merge-multiple: true | |
| pattern: wheel-python3.10-cuda12.9 | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion | |
| docker exec ci_sglang pip install amdsmi | |
| - name: Setup kernel caches | |
| run: | | |
| docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub | |
| # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) | |
| # This tells the test cleanup code to NOT delete downloaded models | |
| if docker exec ci_sglang test -d /sgl-data && docker exec ci_sglang mountpoint -q /sgl-data 2>/dev/null; then | |
| docker exec ci_sglang touch /sgl-data/hf-cache/.persistent_cache | |
| echo "Created .persistent_cache marker - HF cache will persist" | |
| else | |
| echo "WARNING: /sgl-data is not a mount point - models will be cleaned up after each test" | |
| fi | |
| # Check MIOpen cache (VAE convolution kernels) | |
| miopen_files=$(docker exec ci_sglang find /sgl-data/miopen-cache -name "*.udb" 2>/dev/null | wc -l || echo "0") | |
| echo "Found ${miopen_files} MIOpen cache files" | |
| - name: Diagnose HF cache and system resources | |
| run: | | |
| echo "=== System Memory Status ===" | |
| free -h | |
| echo "" | |
| echo "=== Disk Space ===" | |
| df -h /home/runner/sglang-data 2>/dev/null || df -h | |
| echo "" | |
| echo "=== HF Cache Directory Structure ===" | |
| docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" | |
| docker exec ci_sglang ls -la /sgl-data/hf-cache/hub/ 2>/dev/null || echo "HF hub cache not found" | |
| echo "" | |
| echo "=== Checking for cached diffusion models (2-GPU tests) ===" | |
| # Models used in 2-GPU tests: Wan2.2-T2V-A14B, Wan2.1-T2V-14B, Qwen-Image, FLUX.1 | |
| for model in "Wan-AI--Wan2.2-T2V-A14B-Diffusers" "Wan-AI--Wan2.1-T2V-14B-Diffusers" "Qwen--Qwen-Image" "black-forest-labs--FLUX.1-dev"; do | |
| cache_path="/sgl-data/hf-cache/hub/models--${model}" | |
| if docker exec ci_sglang test -d "$cache_path"; then | |
| size=$(docker exec ci_sglang du -sh "$cache_path" 2>/dev/null | cut -f1) | |
| echo "✓ CACHED: $model ($size)" | |
| else | |
| echo "✗ NOT CACHED: $model" | |
| fi | |
| done | |
| echo "" | |
| echo "=== GPU Memory Status ===" | |
| docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" | |
| - name: Run diffusion server tests (2-GPU) | |
| timeout-minutes: 180 | |
| run: | | |
| # AMD CI: All 2-GPU tests including LoRA | |
| # Tests: T2V, T2I, I2V, LoRA | |
| # | |
| # HF download env vars: | |
| # - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available) | |
| # - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings | |
| docker exec \ | |
| -e SGLANG_E2E_TOLERANCE=0.3 \ | |
| -e SGLANG_STAGE_TIME_TOLERANCE=0.2 \ | |
| -e SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=0.6 \ | |
| -e SGLANG_DENOISE_STEP_TOLERANCE=0.6 \ | |
| -e SGLANG_DENOISE_AGG_TOLERANCE=0.3 \ | |
| -e SGLANG_SKIP_CONSISTENCY=1 \ | |
| -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ | |
| -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ | |
| -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ | |
| -e HF_HUB_ENABLE_HF_TRANSFER=1 \ | |
| -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ | |
| -w /sglang-checkout/python \ | |
| ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ | |
| --suite 2-gpu \ | |
| --partition-id ${{ matrix.part }} \ | |
| --total-partitions 3 | |
| # Post-test diagnostics | |
| echo "=== Post-test System Memory Status ===" | |
| free -h | |
| - name: Upload diffusion failure artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: diffusion-failures-amd-rocm720-2gpu-${{ matrix.part }}-${{ github.run_attempt }} | |
| path: diffusion-failures/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| # ROCm 7.2.0 counterpart of `multimodal-gen-unit-test-amd`: run the portable | |
| # mm_gen `unit` suite on ROCm 7.2.0 too. Skips the CUDA-only | |
| # ltx2_vae_channels_last file (channels_last_3d assertions the ROCm conv path | |
| # doesn't reproduce). | |
| multimodal-gen-unit-test-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| runs-on: linux-mi300-1gpu-sglang | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Download artifacts | |
| if: needs.check-changes.outputs.sgl_kernel == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: python/sglang/kernels/aot/dist/ | |
| merge-multiple: true | |
| pattern: wheel-python3.10-cuda12.9 | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: | | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion | |
| docker exec ci_sglang pip install amdsmi | |
| - name: Run diffusion unit tests | |
| timeout-minutes: 60 | |
| run: | | |
| # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` | |
| # memory-format behavior that the ROCm conv path doesn't reproduce. | |
| docker exec \ | |
| -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ | |
| -w /sglang-checkout/python \ | |
| ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ | |
| --suite unit \ | |
| -k "not ltx2_vae_channels_last" | |
| stage-c-test-4-gpu-amd-rocm720: | |
| needs: [check-changes, stage-b-test-1-gpu-small-amd-rocm720, stage-b-test-2-gpu-large-amd-rocm720] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi300-4gpu-sglang] | |
| part: [0] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 90 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh \ | |
| -e NCCL_CUMEM_ENABLE=0 \ | |
| -e NCCL_NVLS_ENABLE=0 \ | |
| -e RCCL_MSCCL_ENABLE=0 \ | |
| -e SGLANG_USE_ROCM700A=1 \ | |
| -w "/sglang-checkout/test" \ | |
| python3 run_suite.py \ | |
| --hw amd \ | |
| --suite stage-c-test-4-gpu-amd \ | |
| --auto-partition-id ${{ matrix.part }} \ | |
| --auto-partition-size 1 \ | |
| --timeout-per-file 5400 \ | |
| --enable-retry \ | |
| --max-attempts 2 \ | |
| --retry-wait-seconds 120 \ | |
| --retry-timeout-increase 0 \ | |
| ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| stage-c-test-large-8-gpu-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| env: | |
| RUNNER_LABELS: linux-mi300-8gpu-sglang | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| runner: [linux-mi300-8gpu-sglang] | |
| part: [0, 1, 2, 3] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Test RCCL multi-GPU communication | |
| timeout-minutes: 5 | |
| run: | | |
| echo "Testing RCCL multi-GPU communication with debug info..." | |
| docker exec ci_sglang bash -c "cd /sglang-checkout && NCCL_DEBUG=INFO RCCL_DEBUG=INFO torchrun --nproc_per_node=8 scripts/ci/amd/test_rccl_multi_gpu.py" | |
| - name: Run test | |
| timeout-minutes: 120 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} | |
| stage-c-test-large-8-gpu-amd-mi35x-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd-mi35x-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi35x-gpu-8] | |
| part: [0, 1, 2] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Start CI container | |
| run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Run test | |
| timeout-minutes: 60 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| # =============================================== Disaggregation ==================================================== | |
| stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| (!failure() && !cancelled()) && | |
| ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) | |
| ) | |
| ) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux-mi35x-gpu-8.fabric] | |
| runs-on: ${{matrix.runner}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Check Host RDMA Environment | |
| id: rdma_detect | |
| run: | | |
| set +e | |
| echo "=== Checking Host RDMA Environment ===" | |
| echo "" | |
| echo "=== 1. Ionic driver library check ===" | |
| ls -l /usr/lib/x86_64-linux-gnu/libibverbs/libionic* 2>/dev/null || echo "libionic not found in standard path" | |
| echo "" | |
| echo "=== 2. Infiniband devices ===" | |
| ls -la /dev/infiniband/ 2>/dev/null || echo "/dev/infiniband not found" | |
| ls -la /sys/class/infiniband/ 2>/dev/null || echo "/sys/class/infiniband not found" | |
| echo "" | |
| echo "=== 3. ibv_devinfo ===" | |
| which ibv_devinfo 2>/dev/null && ibv_devinfo 2>&1 || echo "ibv_devinfo not available" | |
| echo "" | |
| echo "=== 4. Kernel modules ===" | |
| lsmod 2>/dev/null | grep -E "ib_|rdma|ionic" || echo "No RDMA kernel modules loaded" | |
| echo "" | |
| echo "=== 5. Detect RDMA Devices for test environment ===" | |
| if [ -d "/sys/class/infiniband" ]; then | |
| # RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) | |
| RDMA_DEVS=rdma0 | |
| echo "Detected RDMA Devices: $RDMA_DEVS" | |
| echo "SGLANG_TEST_RDMA_DEVICE=$RDMA_DEVS" >> $GITHUB_ENV | |
| else | |
| echo "No RDMA devices found in /sys/class/infiniband" | |
| echo "SGLANG_TEST_RDMA_DEVICE=" >> $GITHUB_ENV | |
| fi | |
| echo "" | |
| echo "=== Host RDMA Check Complete ===" | |
| - name: Start Special Container | |
| run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Verify NIXL in Container | |
| run: | | |
| docker exec ci_sglang bash -lc ' | |
| set -eux | |
| export LD_LIBRARY_PATH=/opt/ucx/lib:/opt/rocm/lib:${LD_LIBRARY_PATH:-} | |
| python -c "import nixl; print(\"nixl OK:\", nixl.__file__)" | |
| test -d /opt/ucx/lib | |
| UCX_INFO=$(command -v ucx_info || true) | |
| if [ -z "$UCX_INFO" ]; then | |
| UCX_INFO=/opt/ucx/bin/ucx_info | |
| fi | |
| test -x "$UCX_INFO" | |
| "$UCX_INFO" -v | |
| "$UCX_INFO" -d | grep -iq rocm | |
| python -c "from nixl._api import nixl_agent, nixl_agent_config; nixl_agent(\"ci-smoke\", nixl_agent_config(backends=[\"UCX\"])); print(\"nixl UCX backend init OK\")" | |
| ' | |
| - name: Verify RDMA in Container | |
| run: | | |
| docker exec -u root ci_sglang bash -c ' | |
| echo "=== Container RDMA Verification ===" | |
| echo "Device nodes:" | |
| ls -la /dev/infiniband/ | |
| echo "" | |
| echo "Provider libraries:" | |
| ls /usr/lib/x86_64-linux-gnu/libibverbs/ | grep -E "ionic|mlx" || echo "No Ionic/Mellanox providers" | |
| echo "" | |
| echo "HCA devices:" | |
| HCA_COUNT=$(ibv_devinfo -list 2>&1 | grep -oE "^[0-9]+ HCAs? found" | grep -oE "^[0-9]+" || echo "0") | |
| ibv_devinfo -list | |
| if [ "$HCA_COUNT" -gt 0 ]; then | |
| echo "" | |
| echo "=== SUCCESS: RDMA setup complete. Found $HCA_COUNT HCA(s) ===" | |
| else | |
| echo "" | |
| echo "=== WARNING: No HCAs detected. RDMA tests may fail ===" | |
| fi | |
| ' | |
| - name: Run Aiter Op Test (RMSNorm) | |
| timeout-minutes: 10 | |
| run: | | |
| echo "Running pre-check: test_rmsnorm2d.py" | |
| docker exec \ | |
| -e MAX_JOBS=192 \ | |
| ci_sglang \ | |
| python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py | |
| - name: Run test_disaggregation | |
| timeout-minutes: 60 | |
| run: | | |
| docker exec ci_sglang rocm-smi --showtopotype || true | |
| docker exec ci_sglang bash -c 'strings $(python3 -c "import mooncake,os;print(os.path.dirname(mooncake.__file__))")/engine*.so | grep -iE "rdma,hip|Unsupported multi-protocol"' || true | |
| bash scripts/ci/amd/amd_ci_exec.sh \ | |
| -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ | |
| -e MC_LOG_LEVEL=TRACE \ | |
| -e GLOG_logtostderr=1 \ | |
| -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} | |
| dsv4-umbp-hicache-amd-rocm720: | |
| needs: [call-gate] | |
| if: | | |
| always() && !cancelled() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-umbp-hicache-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| ( | |
| (github.event_name == 'schedule') || | |
| inputs.run_all_tests | |
| ) | |
| ) | |
| ) | |
| runs-on: linux-mi35x-gpu-8.fabric | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Check Host RDMA Environment | |
| run: | | |
| set -euo pipefail | |
| test -d /dev/infiniband | |
| HCA_COUNT=$(ibv_devinfo -list 2>&1 | awk '/HCAs? found/ {print $1; exit}') | |
| ibv_devinfo -list | |
| if [[ -z "${HCA_COUNT}" || "${HCA_COUNT}" -lt 1 ]]; then | |
| echo "No active RDMA HCA found" | |
| exit 1 | |
| fi | |
| echo "SGLANG_TEST_RDMA_DEVICE=rdma0" >> "$GITHUB_ENV" | |
| - name: Start Fresh RDMA Container | |
| run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| ENABLE_CACHE_HOST: "1" | |
| - name: Install dependencies | |
| run: bash scripts/ci/amd/amd_ci_install_dependency.sh | |
| - name: Verify RDMA and UMBP in Container | |
| run: | | |
| docker exec -u root ci_sglang bash -lc ' | |
| set -euo pipefail | |
| test -e /dev/infiniband/rdma_cm | |
| ibv_devinfo -list | |
| python3 -c "import mori.umbp; print(\"mori.umbp OK:\", mori.umbp.__file__)" | |
| ' | |
| - name: Run DeepSeek-V4 HiCache UMBP E2E | |
| timeout-minutes: 120 | |
| run: | | |
| bash scripts/ci/amd/amd_ci_exec.sh \ | |
| -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ | |
| -e MC_LOG_LEVEL=TRACE \ | |
| -e GLOG_logtostderr=1 \ | |
| -w "/sglang-checkout/test" \ | |
| python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-umbp --nightly --timeout-per-file 6000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} | |
| # =============================================== DeepSeek-V4 (MI35x, 8-GPU) ==================================================== | |
| # On the 6h ('0 */6 * * *') cron these are the only jobs that run; they also | |
| # run on the daily cron alongside the full suite. Scheduled runs pass | |
| # --continue-on-error. They stay selectable via workflow_dispatch / run on | |
| # workflow_call full runs. | |
| dsv4-flash-fp4-fp8-amd-rocm720: | |
| needs: [call-gate] | |
| if: | | |
| always() && !cancelled() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-flash-fp4-fp8-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| ( | |
| (github.event_name == 'schedule') || | |
| inputs.run_all_tests | |
| ) | |
| ) | |
| ) | |
| runs-on: linux-mi35x-gpu-8 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Setup docker (ROCm 7.2) | |
| run: | | |
| touch github_summary.md | |
| bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| ENABLE_CACHE_HOST: "1" | |
| - name: Install dependencies | |
| run: | | |
| # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps | |
| bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate | |
| - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP4 + FP8) | |
| timeout-minutes: 300 | |
| run: | | |
| > github_summary.md # Clear summary file | |
| # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test | |
| # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. | |
| bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ | |
| -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ | |
| -e SGLANG_DSV4_ACCURACY_ONLY=1 \ | |
| -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ | |
| python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? | |
| echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true | |
| exit ${TEST_EXIT_CODE:-0} | |
| dsv4-pro-fp4-amd-rocm720: | |
| needs: [call-gate] | |
| if: | | |
| always() && !cancelled() && | |
| ( | |
| (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-pro-fp4-amd-rocm720,')) || | |
| ( | |
| !(inputs.target_stage || inputs.target_stage_select) && | |
| ( | |
| (github.event_name == 'schedule') || | |
| inputs.run_all_tests | |
| ) | |
| ) | |
| ) | |
| runs-on: linux-mi35x-gpu-8 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} | |
| - name: Ensure VRAM is clear | |
| run: bash scripts/ci/amd/ensure_vram_clear.sh rocm | |
| - name: Setup docker (ROCm 7.2) | |
| run: | | |
| touch github_summary.md | |
| bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| ENABLE_CACHE_HOST: "1" | |
| - name: Install dependencies | |
| run: | | |
| # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. | |
| bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps | |
| bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate | |
| - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP4) | |
| timeout-minutes: 480 | |
| run: | | |
| > github_summary.md # Clear summary file | |
| # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test | |
| # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. | |
| bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ | |
| -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ | |
| -e SGLANG_DSV4_ACCURACY_ONLY=1 \ | |
| -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ | |
| python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$? | |
| echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true | |
| if [[ "${{ github.event_name == 'schedule' || inputs.continue_on_error }}" == "true" ]]; then | |
| exit 0 | |
| fi | |
| exit ${TEST_EXIT_CODE:-0} | |
| pr-test-amd-rocm720-finish: | |
| needs: | |
| [ | |
| call-gate, | |
| check-changes, | |
| sgl-kernel-unit-test-amd-rocm720, | |
| sgl-kernel-unit-test-2-gpu-amd-rocm720, | |
| multimodal-gen-test-1-gpu-amd-rocm720, | |
| multimodal-gen-test-2-gpu-amd-rocm720, | |
| multimodal-gen-unit-test-amd-rocm720, | |
| stage-a-test-1-gpu-small-amd-rocm720, | |
| jit-kernel-unit-test-amd-rocm720, | |
| jit-kernel-benchmark-test-amd-rocm720, | |
| stage-b-test-1-gpu-small-amd-rocm720, | |
| stage-b-test-1-gpu-small-amd-nondeterministic-rocm720, | |
| stage-b-test-1-gpu-small-amd-mi35x-rocm720, | |
| stage-b-test-1-gpu-large-amd-rocm720, | |
| stage-b-test-2-gpu-large-amd-rocm720, | |
| stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720, | |
| dsv4-umbp-hicache-amd-rocm720, | |
| stage-c-test-4-gpu-amd-rocm720, | |
| stage-c-test-large-8-gpu-amd-rocm720, | |
| stage-c-test-large-8-gpu-amd-mi35x-rocm720, | |
| dsv4-flash-fp4-fp8-amd-rocm720, | |
| dsv4-pro-fp4-amd-rocm720, | |
| ] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check all dependent job statuses | |
| run: | | |
| # Convert the 'needs' context to a JSON string | |
| json_needs='${{ toJson(needs) }}' | |
| # Get a list of all job names from the JSON keys | |
| job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') | |
| for job in $job_names; do | |
| # For each job, extract its result | |
| result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') | |
| # Print the job name and its result | |
| echo "$job: $result" | |
| # Check for failure or cancellation and exit if found | |
| if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then | |
| echo "The above jobs failed." | |
| exit 1 | |
| fi | |
| done | |
| # If the loop completes, all jobs were successful | |
| echo "All jobs completed successfully" | |
| exit 0 |