diff --git a/.github/actions/zombienet-sdk/action.yml b/.github/actions/zombienet-sdk/action.yml index 5fb77867ac85b..d3719908c7fc1 100644 --- a/.github/actions/zombienet-sdk/action.yml +++ b/.github/actions/zombienet-sdk/action.yml @@ -7,9 +7,12 @@ inputs: ref-slug: description: "Ref slug (e.g branch-name-short)" required: true - test: + test-filter: description: "test filter to pass to nextest (e.g: functional::spam_statement_distribution_requests::spam_statement_distribution_requests_test)" required: true + job-name: + description: "Job name to use for artifact uploading" + required: true prefix: description: "Archive prefix for tests files (e.g polkadot, cumulus or substrate)" required: true @@ -25,7 +28,7 @@ runs: - name: common_vars shell: bash env: - TEST_NAME: ${{ inputs.test }} + TEST_FILTER: ${{ inputs.test-filter }} PREFIX: ${{ inputs.prefix }} run: | echo "Vars" @@ -36,7 +39,7 @@ runs: echo "COL_IMAGE: $COL_IMAGE" echo "MALUS_IMAGE: $MALUS_IMAGE" echo "Inputs" - echo "test: $TEST_NAME" + echo "test: $TEST_FILTER" echo "prefix: $PREFIX" - name: Download binaries for zombienet native tests @@ -66,7 +69,7 @@ runs: env: # don't retry sdk tests NEXTEST_RETRIES: 0 - TEST_NAME: ${{ inputs.test }} + TEST_FILTER: ${{ inputs.test-filter }} PREFIX: ${{ inputs.prefix }} run: | # RUN_IN_CI=1 shall be set only for k8s provider @@ -87,7 +90,7 @@ runs: # We want to run tests sequentially, '--no-capture' ensures that. # If we want to get rid of '--no-capture' some day, please use '--test-threads 1' or NEXTEST_TEST_THREADS=1 # Both options cannot coexist for cargo-nextest below v0.9.94 - cargo nextest run --archive-file ./artifacts/${PREFIX}-zombienet-tests.tar.zst --no-capture -- ${TEST_NAME} + cargo nextest run --archive-file ./artifacts/${PREFIX}-zombienet-tests.tar.zst --no-capture -- ${TEST_FILTER} - name: process_logs if: ${{ ! cancelled() }} @@ -108,7 +111,7 @@ runs: uses: actions/upload-artifact@v4 if: ${{ ! cancelled() }} with: - name: zombienet-logs-${{ github.job }}-${{ github.sha }} + name: zombienet-logs-${{ inputs.job-name }}-${{ github.sha }} path: | /tmp/zombie*/logs/* diff --git a/.github/actions/zombienet/action.yml b/.github/actions/zombienet/action.yml index 60aee7b616409..53c5e64e23f5a 100644 --- a/.github/actions/zombienet/action.yml +++ b/.github/actions/zombienet/action.yml @@ -1,9 +1,12 @@ name: "Zombienet test v1" description: "Runs zombienet tests" inputs: - test: + test-definition: description: "test definition (zndsl file)" required: true + job-name: + description: "Job name to use for artifact uploading" + required: true local-dir: description: "Path to the directory tha contains the test file (.zndsl)" required: true @@ -27,7 +30,7 @@ runs: - name: common_vars shell: bash env: - TEST_NAME: ${{ inputs.test }} + TEST_DEFINITION: ${{ inputs.test-definition }} LOCAL_PATH: ${{ inputs.local-dir }} CONCURRENCY: ${{ inputs.concurrency }} run: | @@ -36,7 +39,8 @@ runs: echo "ZOMBIENET_PROVIDER: $ZOMBIENET_PROVIDER" echo "COL_IMAGE: $COL_IMAGE" echo "Inputs" - echo "test: $TEST_NAME" + echo "test-definition: $TEST_DEFINITION" + echo "job-name: ${{ inputs.job-name }}" echo "local-dir: $LOCAL_PATH" echo "concurrency: $CONCURRENCY" @@ -59,7 +63,7 @@ runs: - name: zombie_test shell: bash env: - TEST_NAME: ${{ inputs.test }} + TEST_DEFINITION: ${{ inputs.test-definition }} LOCAL_PATH: ${{ inputs.local-dir }} CONCURRENCY: ${{ inputs.concurrency }} run: | @@ -71,7 +75,7 @@ runs: ./.github/scripts/run-zombienet-test.sh \ "$(pwd)/$LOCAL_PATH" \ $CONCURRENCY \ - "$TEST_NAME" + "$TEST_DEFINITION" else # no need to check other runner variables. for k8s they shall store the same value if [[ $ZOMBIENET_DEFAULT_RUNNER == "parity-zombienet" ]]; then @@ -81,7 +85,7 @@ runs: /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh \ --local-dir="$(pwd)/$LOCAL_PATH" \ --concurrency=$CONCURRENCY \ - --test="$TEST_NAME" + --test="$TEST_DEFINITION" fi - name: process_logs @@ -102,7 +106,7 @@ runs: uses: actions/upload-artifact@v4 if: ${{ ! cancelled() }} with: - name: zombienet-logs-${{ github.job }}-${{ github.sha }} + name: zombienet-logs-${{ inputs.job-name }}-${{ github.sha }} path: | /tmp/zombie*/logs/* diff --git a/.github/scripts/dispatch-zombienet-workflow.sh b/.github/scripts/dispatch-zombienet-workflow.sh new file mode 100755 index 0000000000000..38bb42295dca5 --- /dev/null +++ b/.github/scripts/dispatch-zombienet-workflow.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Zombienet Workflow Dispatcher +# +# This script triggers GitHub Actions workflows for zombienet tests and monitors their execution. +# It can run workflows multiple times for reliability testing and optionally filter tests by pattern. +# Results are automatically saved to a timestamped CSV file for analysis. +# +# Features: +# - Trigger workflows on specific branches +# - Filter tests by pattern (useful for debugging specific tests) +# - Run workflows multiple times for flaky test detection +# - Monitor workflow completion and collect results +# - Export results to CSV with job details (ID, name, conclusion, timing, URLs) +# +# Requirements: +# - GitHub CLI (gh) must be installed and authenticated +# - Must be run from polkadot-sdk repository root +# - Target branch must have corresponding PR with CI enabled + +# Exit on error +# set -e + +function dbg { + local msg="$@" + + local tstamp=$(date "+%Y-%m-%d %T") + printf "%s - %s\n" "$tstamp" "$msg" +} + +function write_job_results_to_csv { + local run_id="$1" + local branch="$2" + local csv_file="$3" + + dbg "Writing job results for run $run_id to $csv_file" + + # Get job details for the completed run, filtering only jobs starting with 'zombienet-' and with success or failure conclusions + gh run view "$run_id" --json jobs --jq \ + '.jobs[] | select(.name | startswith("zombienet-")) | + select(.conclusion == "success" or .conclusion == "failure") | + [.databaseId, .name, .conclusion, .startedAt, "'"$branch"'", .url] | @csv' >> "$csv_file" +} + +# Parse command line arguments +WORKFLOW_FILE="" +BRANCH="" +MAX_RESULT_CNT=-1 +TEST_PATTERN="" + +while getopts "w:b:m:p:h" opt; do + case $opt in + w) WORKFLOW_FILE="$OPTARG" ;; + b) BRANCH="$OPTARG" ;; + m) MAX_RESULT_CNT="$OPTARG" ;; + p) TEST_PATTERN="$OPTARG" ;; + h) echo "Usage: $0 -w -b [-m max-triggers] [-p test-pattern]" + echo " -w: Workflow file (required)" + echo " -b: Branch name (required)" + echo " -m: Maximum number of triggers (optional, default: infinite)" + echo " -p: Test pattern for workflow input (optional)" + exit 0 ;; + \?) echo "Invalid option -$OPTARG" >&2 + echo "Use -h for help" + exit 1 ;; + esac +done + +if [[ -z "$WORKFLOW_FILE" || -z "$BRANCH" ]]; then + echo "Error: Both workflow file (-w) and branch (-b) are required" + echo "Usage: $0 -w -b [-m max-triggers] [-p test-pattern]" + echo "Use -h for help" + exit 1 +fi + +# Create CSV file with headers +CSV_FILE="workflow_results_$(date +%Y%m%d_%H%M%S).csv" +echo "job_id,job_name,conclusion,started_at,branch,job_url" > "$CSV_FILE" +dbg "Created CSV file: $CSV_FILE" + +dbg "Starting loop for workflow: $WORKFLOW_FILE on branch: $BRANCH" + +TRIGGER_CNT=0 +RESULT_CNT=0 + +while [[ $MAX_RESULT_CNT -eq -1 || $RESULT_CNT -lt $MAX_RESULT_CNT ]]; do + + dbg "Waiting until workflow $WORKFLOW_FILE (branch: $BRANCH) jobs are completed" + + while true ; do + echo "" + gh run list --workflow=$WORKFLOW_FILE -e workflow_dispatch -b $BRANCH -L 5 + sleep 2 + # if job is completed it should have non-empty conclusion field + ALL_JOBS_COMPLETED=$(gh run list --workflow=$WORKFLOW_FILE -e workflow_dispatch -b $BRANCH --json conclusion --jq 'all(.[]; .conclusion != "")') + if [[ "$ALL_JOBS_COMPLETED" == "true" ]]; then + break + fi + sleep 60 + done + dbg "Workflow $WORKFLOW_FILE (branch: $BRANCH) jobs completed" + + # Skip the first iteration - latest run id is not the one we triggered here + if [ $TRIGGER_CNT -gt 0 ]; then + # Get the most recent completed run ID and write job results to CSV + LATEST_RUN_ID=$(gh run list --workflow=$WORKFLOW_FILE -e workflow_dispatch -b $BRANCH -L 1 --json databaseId --jq '.[0].databaseId') + write_job_results_to_csv "$LATEST_RUN_ID" "$BRANCH" "$CSV_FILE" + RESULT_CNT=$(( RESULT_CNT + 1 )) + fi + + TRIGGER_CNT=$(( TRIGGER_CNT + 1 )) + dbg "Triggering #$TRIGGER_CNT workflow $WORKFLOW_FILE (branch: $BRANCH)" + + if [[ -n "$TEST_PATTERN" ]]; then + gh workflow run "$WORKFLOW_FILE" --ref "$BRANCH" -f test_pattern="$TEST_PATTERN" + else + gh workflow run "$WORKFLOW_FILE" --ref "$BRANCH" + fi + + dbg "Sleeping 60s" + sleep 60 +done + diff --git a/.github/scripts/parse-zombienet-tests.py b/.github/scripts/parse-zombienet-tests.py new file mode 100644 index 0000000000000..a0bda294a504f --- /dev/null +++ b/.github/scripts/parse-zombienet-tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +""" +Zombienet Test Matrix Parser + +This script parses YAML test definition files and converts them to JSON format +for use as GitHub Actions matrix jobs. It provides filtering capabilities to: + +1. Exclude flaky tests (unless a specific test pattern is provided) +2. Filter tests by name pattern for targeted execution +3. Convert YAML test definitions to JSON matrix format + +The script is used by GitHub Actions workflows to dynamically generate +test matrices based on YAML configuration files, enabling flexible +test execution and maintenance. + +Usage: + python parse-zombienet-tests.py --matrix tests.yml [--flaky-tests flaky.txt] [--test-pattern pattern] + +Output: + JSON array of test job objects suitable for GitHub Actions matrix strategy +""" + +import argparse +import yaml +import json +import fnmatch + +def parse_args(): + parser = argparse.ArgumentParser(description="Parse test matrix YAML file with optional filtering") + parser.add_argument("--matrix", required=True, help="Path to the YAML matrix file") + parser.add_argument("--flaky-tests", default="", help="Newline-separated list of flaky job names") + parser.add_argument("--test-pattern", default="", help="Pattern to match job_name (substring or glob)") + return parser.parse_args() + +def load_jobs(matrix_path): + with open(matrix_path, "r") as f: + return yaml.safe_load(f) + +def filter_jobs(jobs, flaky_tests, test_pattern): + flaky_set = set(name.strip() for name in flaky_tests.splitlines() if name.strip()) + filtered = [] + + for job in jobs: + name = job.get("job-name", "") + + # If test_pattern provided then don't care about flaky tests, just check test_pattern + if test_pattern and len(test_pattern) > 0: + if fnmatch.fnmatch(name, f"*{test_pattern}*"): + filtered.append(job) + elif name not in flaky_set: + filtered.append(job) + + return filtered + +def main(): + args = parse_args() + jobs = load_jobs(args.matrix) + result = filter_jobs(jobs, args.flaky_tests, args.test_pattern) + print(json.dumps(result)) + +if __name__ == "__main__": + main() diff --git a/.github/workflows/zombienet-reusable-preflight.yml b/.github/workflows/zombienet-reusable-preflight.yml index dd2caac5af269..151f2e4dea3b9 100644 --- a/.github/workflows/zombienet-reusable-preflight.yml +++ b/.github/workflows/zombienet-reusable-preflight.yml @@ -16,6 +16,13 @@ name: Zombienet Preflight on: workflow_call: + inputs: + tests_yaml: + required: true + type: string + test_pattern: + required: false + type: string # Map the workflow outputs to job outputs outputs: changes_substrate: @@ -133,6 +140,10 @@ on: description: | comma separated list of flaky tests to skip. + TEST_MATRIX: + value: ${{ jobs.preflight.outputs.TEST_MATRIX }} + description: | + JSON formatted test matrix parsed from test yaml jobs: # # @@ -166,6 +177,7 @@ jobs: KUBERNETES_MEMORY_REQUEST: ${{ steps.set_vars.outputs.KUBERNETES_MEMORY_REQUEST }} TEMP_IMAGES_BASE: ${{ steps.set_vars.outputs.TEMP_IMAGES_BASE }} FLAKY_TESTS: ${{ steps.set_vars.outputs.FLAKY_TESTS }} + TEST_MATRIX: ${{ steps.generate_test_matrix.outputs.TEST_MATRIX }} # zombienet-sdk vars RUST_LOG: ${{ steps.set_vars.outputs.RUST_LOG }} @@ -282,6 +294,20 @@ jobs: echo "ZOMBIENET_IMAGE: ${{ steps.set_vars.outputs.ZOMBIENET_IMAGE }}" echo "ZOMBIENET_SDK_IMAGE: ${{ steps.set_vars.outputs.ZOMBIENET_SDK_IMAGE }}" + - name: Generate test matrix + id: generate_test_matrix + shell: bash + env: + TESTS_YAML: ${{ inputs.tests_yaml }} + TEST_PATTERN: ${{ inputs.test_pattern || '' }} + run: | + python3 .github/scripts/parse-zombienet-tests.py \ + --matrix ${TESTS_YAML} \ + --flaky-tests "${{ steps.set_vars.outputs.FLAKY_TESTS }}" \ + --test-pattern "${TEST_PATTERN}" > matrix.json + echo "TEST_MATRIX=$(cat matrix.json)" >> $GITHUB_OUTPUT + echo "TEST_MATRIX:" + cat matrix.json | jq '.' # # # diff --git a/.github/workflows/zombienet_cumulus.yml b/.github/workflows/zombienet_cumulus.yml index bbbe215b57271..80bbea2a1980b 100644 --- a/.github/workflows/zombienet_cumulus.yml +++ b/.github/workflows/zombienet_cumulus.yml @@ -2,6 +2,12 @@ name: Zombienet Cumulus on: workflow_dispatch: + inputs: + test_pattern: + type: string + description: "Run tests which names match this pattern (also flaky)" + default: "" + required: false push: branches: - master @@ -12,6 +18,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: read-all + env: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 LOCAL_DIR: "./cumulus/zombienet/tests" @@ -30,118 +38,44 @@ jobs: preflight: needs: isdraft uses: ./.github/workflows/zombienet-reusable-preflight.yml + with: + tests_yaml: .github/zombienet-tests/zombienet_cumulus_tests.yml + test_pattern: ${{ inputs.test_pattern }} - zombienet-cumulus-0001-sync_blocks_from_tip_without_connected_collator: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0001-sync_blocks_from_tip_without_connected_collator') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::sync_blocks::sync_blocks_from_tip_without_connected_collator" - prefix: "cumulus" - # Disabled, occasionally fails - # Takes too much CPU when spawning all nodes at the same time - # Re-enable when https://github.com/paritytech/zombienet-sdk/issues/371 is implemented - zombienet-cumulus-0002-pov_recovery: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0002-pov_recovery') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} + zombienet-cumulus-tests: + name: ${{ matrix.test.job-name }} + runs-on: ${{ matrix.test.runner-type == 'large' && needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER || needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::pov_recovery::pov_recovery" - prefix: "cumulus" - - zombienet-cumulus-0003-full_node_catching_up: needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0003-full_node_catching_up') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 + if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) }} container: image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} options: -v /tmp/zombienet:/tmp/zombienet env: POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" + CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/${{ matrix.test.cumulus-image }}:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::full_node_catching_up::full_node_catching_up" - prefix: "cumulus" - - zombienet-cumulus-0004-runtime_upgrade: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0004-runtime_upgrade') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.preflight.outputs.TEST_MATRIX) }} steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/download-artifact@v4.1.8 + if: ${{ matrix.test.needs-wasm-binary }} with: name: build-test-parachain-${{ needs.preflight.outputs.SOURCE_REF_SLUG }} github-token: ${{ secrets.GITHUB_TOKEN }} run-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - # runtime_upgrade tests needs a dedicated WASM binary - name: provide_wasm_binary + if: ${{ matrix.test.needs-wasm-binary }} run: | tar -xvf artifacts.tar ls -ltr artifacts/* @@ -152,266 +86,9 @@ jobs: - name: zombienet_test uses: ./.github/actions/zombienet-sdk with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::runtime_upgrade::runtime_upgrade" - prefix: "cumulus" - - zombienet-cumulus-0005-migrate_solo_to_para: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0005-migrate_solo_to_para') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::migrate_solo::migrate_solo_to_para" - prefix: "cumulus" - - # Disabled, occasionally fails - # Takes too much CPU when spawning all nodes at the same time - # Re-enable when https://github.com/paritytech/zombienet-sdk/issues/371 is implemented - zombienet-cumulus-0006-rpc_collator_builds_blocks: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0006-rpc_collator_builds_blocks') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::rpc_collator_build_blocks::rpc_collator_builds_blocks" + test-filter: ${{ matrix.test.test-filter }} + job-name: ${{ matrix.test.job-name }} prefix: "cumulus" - - - zombienet-cumulus-0007-full_node_warp_sync: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0007-full_node_warp_sync') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: gh-token: ${{ secrets.GITHUB_TOKEN }} build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::full_node_warp_sync::full_node_warp_sync" - prefix: "cumulus" - - zombienet-cumulus-0008-elastic_authoring: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0008-elastic_authoring') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::elastic_scaling::slot_based_authoring::elastic_scaling_slot_based_authoring" - prefix: "cumulus" - - - # Disabled, occasionally fails - # See https://github.com/paritytech/polkadot-sdk/issues/8986 - zombienet-cumulus-0009-elastic_scaling_pov_recovery: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0009-elastic_scaling_pov_recovery') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::elastic_scaling::pov_recovery::elastic_scaling_pov_recovery" - prefix: "cumulus" - - # Disabled, occasionally fails. - # See https://github.com/paritytech/polkadot-sdk/issues/8999 - zombienet-cumulus-0010-elastic_scaling_multiple_block_per_slot: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0010-elastic_scaling_multiple_block_per_slot') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::elastic_scaling::multiple_blocks_per_slot::elastic_scaling_multiple_blocks_per_slot" - prefix: "cumulus" - - zombienet-cumulus-0011-dht-bootnodes: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0011-dht-bootnodes') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::bootnodes::dht_bootnodes_test" - prefix: "cumulus" - - zombienet-cumulus-0012-parachain_extrinsic_gets_finalized: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0012-parachain_extrinsic_gets_finalized') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::parachain_extrinsic_get_finalized::parachain_extrinsic_gets_finalized" - prefix: "cumulus" - - zombienet-cumulus-0013-elastic_scaling_slot_based_rp_offset: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-cumulus-0013-elastic_scaling_slot_based_rp_offset') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "zombie_ci::elastic_scaling::slot_based_rp_offset::elastic_scaling_slot_based_relay_parent_offset_test" - prefix: "cumulus" diff --git a/.github/workflows/zombienet_parachain-template.yml b/.github/workflows/zombienet_parachain-template.yml index 6487ff2908f5c..35e470e673d60 100644 --- a/.github/workflows/zombienet_parachain-template.yml +++ b/.github/workflows/zombienet_parachain-template.yml @@ -2,6 +2,12 @@ name: Zombienet Parachain Templates on: workflow_dispatch: + inputs: + test_pattern: + type: string + description: "Run tests which names match this pattern (also flaky)" + default: "" + required: false push: branches: - master @@ -12,6 +18,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: read-all + env: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 GHA_CLUSTER_SERVER_ADDR: "https://kubernetes.default:443" @@ -25,79 +33,31 @@ jobs: preflight: needs: isdraft uses: ./.github/workflows/zombienet-reusable-preflight.yml + with: + tests_yaml: .github/zombienet-tests/zombienet_parachain-template_tests.yml + test_pattern: ${{ inputs.test_pattern }} - zombienet-parachain-template-0001-minimal_template_block_production: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-parachain-template-smoke') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} + zombienet-parachain-template-tests: + name: ${{ matrix.test.job-name }} + runs-on: ${{ matrix.test.runner-type == 'large' && needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER || needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} timeout-minutes: 30 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "smoke::minimal_template_block_production_test" - prefix: "parachain-templates" - - zombienet-parachain-template-0002-parachain_template_block_production: needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-parachain-template-smoke') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 30 + if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_cumulus || needs.preflight.outputs.changes_polkadot) }} container: image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} options: -v /tmp/zombienet:/tmp/zombienet env: - # sdk tests are looking for POLKADOT_IMAGE POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} + CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/${{ matrix.test.cumulus-image }}:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 - steps: - - name: Checkout - uses: actions/checkout@v4 + ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "smoke::parachain_template_block_production_test" - prefix: "parachain-templates" + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.preflight.outputs.TEST_MATRIX) }} - zombienet-parachain-template-0003-solochain_template_block_production: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-parachain-template-smoke') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 30 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 steps: - name: Checkout uses: actions/checkout@v4 @@ -105,8 +65,9 @@ jobs: - name: zombienet_test uses: ./.github/actions/zombienet-sdk with: + test-filter: ${{ matrix.test.test-filter }} + job-name: ${{ matrix.test.job-name }} + prefix: "parachain-templates" gh-token: ${{ secrets.GITHUB_TOKEN }} build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "smoke::solochain_template_block_production_test" - prefix: "parachain-templates" diff --git a/.github/workflows/zombienet_polkadot.yml b/.github/workflows/zombienet_polkadot.yml index 58f5c0c63146a..f4947daf5f514 100644 --- a/.github/workflows/zombienet_polkadot.yml +++ b/.github/workflows/zombienet_polkadot.yml @@ -2,6 +2,12 @@ name: Zombienet Polkadot on: workflow_dispatch: + inputs: + test_pattern: + type: string + description: "Run tests which names match this pattern (also flaky)" + default: "" + required: false push: branches: - master @@ -12,6 +18,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: read-all + env: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 LOCAL_DIR: "./polkadot/zombienet_tests" @@ -30,1026 +38,68 @@ jobs: preflight: needs: isdraft uses: ./.github/workflows/zombienet-reusable-preflight.yml + with: + tests_yaml: .github/zombienet-tests/zombienet_polkadot_tests.yml + test_pattern: ${{ inputs.test_pattern }} - - # - # functional - # - zombienet-polkadot-functional-0001-parachains-pvf: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0001-parachains-pvf') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0001-parachains-pvf.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - concurrency: 1 - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - zombienet-polkadot-functional-0002-parachains-disputes: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0002-parachains-disputes') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0002-parachains-disputes.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - concurrency: 1 - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - zombienet-polkadot-functional-0003-beefy-and-mmr: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0003-beefy-and-mmr') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0003-beefy-and-mmr.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - # TODO: Disabled, occasionally (2 on ~50-70 runs) fails - zombienet-polkadot-functional-0004-parachains-disputes-garbage-candidate: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0004-parachains-disputes-garbage-candidate') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0004-parachains-garbage-candidate.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - zombienet-polkadot-functional-0006-parachains-max-tranche0: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0006-parachains-max-tranche0') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} + zombienet-polkadot-tests: + name: ${{ matrix.test.job-name }} + runs-on: ${{ matrix.test.runner-type == 'large' && (matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER || needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER) || (matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER || needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER) }} timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0006-parachains-max-tranche0.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - zombienet-polkadot-functional-0007-dispute-freshly-finalized: needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0007-dispute-freshly-finalized') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 + if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) }} container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} + image: ${{ matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_IMAGE || needs.preflight.outputs.ZOMBIENET_IMAGE }} options: -v /tmp/zombienet:/tmp/zombienet env: ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" + POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" + CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/${{ matrix.test.cumulus-image || 'polkadot-parachain-debug' }}:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" DEBUG: ${{ needs.preflight.outputs.DEBUG }} ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0007-dispute-freshly-finalized.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-functional-0013-systematic-chunk-recovery: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0013-systematic-chunk-recovery') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0013-systematic-chunk-recovery.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} + RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} + ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.preflight.outputs.TEST_MATRIX) }} - # - # - zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - # this test require an old version of polkadot - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} steps: - name: Checkout uses: actions/checkout@v4 - # We need to use an older version of polkadot/polkadot-parachain, particulary a version that doesn't includes - # https://github.com/paritytech/polkadot-sdk/pull/1644, and since this change is also used by the collators we need to - # keep this test to ensure that works with older versions. - - name: download_old_release_bins + - name: Set additional environment variables + if: ${{ matrix.test.additional-env }} shell: bash run: | - BIN_DIR="$(pwd)/bin_old" - mkdir -p $BIN_DIR - for bin in polkadot polkadot-parachain; do - OLD_NAME="$bin-old" - echo "downloading $bin as $OLD_NAME in $BIN_DIR"; - curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-v1.7.0/$bin - chmod 755 $BIN_DIR/$OLD_NAME; - done - for bin in polkadot-execute-worker polkadot-prepare-worker; do - OLD_NAME="$bin" - echo "downloading $bin as $OLD_NAME in $BIN_DIR"; - curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-v1.7.0/$bin - chmod 755 $BIN_DIR/$OLD_NAME; - done - - ls -ltr $BIN_DIR - export PATH=$BIN_DIR:$PATH - echo "PATH=$PATH" >> $GITHUB_ENV - echo $PATH - - - name: zombienet_test - uses: ./.github/actions/zombienet - env: - OLD_SUFFIX: "-old" - with: - test: "0014-chunk-fetching-network-compatibility.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-functional-0015-coretime-shared-core: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0015-coretime-shared-core') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: cp_script - run: | - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/functional - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0015-coretime-shared-core.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-functional-0019-coretime-collation-fetching-fairness: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-0019-coretime-collation-fetching-fairness') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: cp_script - run: | - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/functional - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0019-coretime-collation-fetching-fairness.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/functional" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - # smoke - # - zombienet-polkadot-smoke-0001-parachains-smoke-test: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-smoke-0001-parachains-smoke-test') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0001-parachains-smoke-test.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/smoke" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 + echo '${{ toJson(matrix.test.additional-env) }}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> $GITHUB_ENV - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0002-parachains-upgrade-smoke-test.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/smoke" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # TODO: Disabled - zombienet-polkadot-smoke-0003-deregister-register-validator: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-smoke-0003-deregister-register-validator') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Additional setup + if: ${{ matrix.test.additional-setup }} + shell: bash + run: ${{ matrix.test.additional-setup }} - - name: zombienet_test + - name: zombienet_test (v1) + if: ${{ !matrix.test.use-zombienet-sdk }} uses: ./.github/actions/zombienet with: - test: "0003-deregister-register-validator-smoke.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/smoke" + test-definition: ${{ matrix.test.test-definition }} + job-name: ${{ matrix.test.job-name }} + local-dir: ${{ matrix.test.local-dir }} + concurrency: ${{ matrix.test.concurrency || 1 }} gh-token: ${{ secrets.GITHUB_TOKEN }} build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-smoke-0004-coretime-smoke-test: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-smoke-0004-coretime-smoke-test') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: zombienet_test - uses: ./.github/actions/zombienet + - name: zombienet_test (sdk) + if: ${{ matrix.test.use-zombienet-sdk }} + uses: ./.github/actions/zombienet-sdk with: - test: "0004-coretime-smoke-test.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/smoke" + test-filter: ${{ matrix.test.test-filter }} + job-name: ${{ matrix.test.job-name }} + prefix: "polkadot" gh-token: ${{ secrets.GITHUB_TOKEN }} build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - # - # - zombienet-polkadot-smoke-0005-precompile-pvf-smoke: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-smoke-0005-precompile-pvf-smoke') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0005-precompile-pvf-smoke.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/smoke" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - # misc - # TODO: Disabled, occasionally (1 on ~50-70 runs) fails - zombienet-polkadot-misc-0001-parachains-paritydb: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-misc-0001-parachains-paritydb') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0001-paritydb.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/misc" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # TODO: needs to resolve how to pass the GH_TOKEN to pods - # - # zombienet-polkadot-misc-0002-upgrade-node: - # needs: [preflight] - # if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-misc-0002-upgrade-node') }} - # runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - # timeout-minutes: 60 - # container: - # image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - # env: - # ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - # COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - # DEBUG: ${{ needs.preflight.outputs.DEBUG }} - # ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - - # - name: custom_setup - # run: | - # export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:latest" - # echo "Overrided polkadot image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}" - # export POLKADOT_PR_ARTIFACTS_URL=${{ needs.preflight.outputs.POLKADOT_PR_ARTIFACTS_URL }} - # echo "POLKADOT_PR_ARTIFACTS_URL: ${POLKADOT_PR_ARTIFACTS_URL}" - - # - name: zombienet_test - # uses: ./.github/actions/zombienet - # with: - # test: "0002-upgrade-node.zndsl" - # local-dir: "${{ env.LOCAL_DIR }}/misc" - # gh-token: ${{ secrets.GITHUB_TOKEN }} - # build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - # ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - - # - # Malus - # - zombienet-polkadot-malus-0001-dispute-valid: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-malus-0001-dispute-valid') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: ${{ needs.preflight.outputs.DEBUG }} - LOCAL_DIR: "./polkadot/node/malus" - ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "0001-dispute-valid-block.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/integrationtests" - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - # - # sdk tests - # - - # TODO: Disabled - zombienet-polkadot-coretime-revenue: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-coretime-revenue') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "smoke::coretime_revenue::coretime_revenue_test" - prefix: "polkadot" - # - # - zombienet-polkadot-elastic-scaling-slot-based-3cores: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-elastic-scaling-slot-based-3cores') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "elastic_scaling::slot_based_3cores::slot_based_3cores_test" - prefix: "polkadot" - # - # TODO: Disabled, fails very often with zombienet native provider - zombienet-polkadot-elastic-scaling-slot-based-12cores: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-elastic-scaling-slot-based-12cores') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "elastic_scaling::slot_based_12cores::slot_based_12cores_test" - prefix: "polkadot" - # - # - zombienet-polkadot-elastic-scaling-doesnt-break-parachains: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-elastic-scaling-doesnt-break-parachains') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "elastic_scaling::doesnt_break_parachains::doesnt_break_parachains_test" - prefix: "polkadot" - # - # - zombienet-polkadot-elastic-scaling-basic-3cores: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-elastic-scaling-basic-3cores') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "elastic_scaling::basic_3cores::basic_3cores_test" - prefix: "polkadot" - # - # - zombienet-polkadot-functional-sync-backing: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-sync-backing') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/test-parachain:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::sync_backing::sync_backing_test" - prefix: "polkadot" - # - # - zombienet-polkadot-functional-async-backing-6-seconds-rate: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-async-backing-6-seconds-rate') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::async_backing_6_seconds_rate::async_backing_6_seconds_rate_test" - prefix: "polkadot" - # - # - # TODO: Disabled, occasionally (1 on ~50-100 runs) fails - zombienet-polkadot-functional-duplicate-collations: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-duplicate-collations') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::duplicate_collations::duplicate_collations_test" - prefix: "polkadot" - # - # TODO: Disabled, occasionally (2 on ~50-70 runs) fails - zombienet-polkadot-disputes-slashing: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-disputes-slashing') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "disabling::slashing" - prefix: "polkadot" - - # - # - # TODO: Disabled, occasionally (1 on ~50-100 runs) fails - zombienet-polkadot-functional-spam-statement-distribution-requests: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-spam-statement-distribution-requests') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::spam_statement_distribution_requests::spam_statement_distribution_requests_test" - prefix: "polkadot" - - zombienet-polkadot-approval-voting-coalescing: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-approval-voting-coalescing') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::approval_voting_coalescing::approval_voting_coalescing_test" - prefix: "polkadot" - - # - # - zombienet-polkadot-approved-peer-mixed-validators: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-approved-peer-mixed-validators') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" - OLD_POLKADOT_COMMAND: "polkadot-old" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # We need to use an older version of polkadot - - name: download_old_release_bins - shell: bash - run: | - BIN_DIR="$(pwd)/bin_old" - mkdir -p $BIN_DIR - for bin in polkadot polkadot-parachain; do - OLD_NAME="$bin-old" - echo "downloading $bin as $OLD_NAME in $BIN_DIR"; - curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2503/$bin - chmod 755 $BIN_DIR/$OLD_NAME; - done - for bin in polkadot-execute-worker polkadot-prepare-worker; do - OLD_NAME="$bin" - echo "downloading $bin as $OLD_NAME in $BIN_DIR"; - curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2503/$bin - chmod 755 $BIN_DIR/$OLD_NAME; - done - - ls -ltr $BIN_DIR - export PATH=$BIN_DIR:$PATH - echo "PATH=$PATH" >> $GITHUB_ENV - echo $PATH - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::approved_peer_mixed_validators::approved_peer_mixed_validators_test" - prefix: "polkadot" - - # - # - zombienet-polkadot-functional-validator-disabling: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-functional-validator-disabling') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::validator_disabling::validator_disabling_test" - prefix: "polkadot" - -# -# - zombienet-polkadot-dispute-old-finalized: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-dispute-old-finalized') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - env: - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - COL_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/colander:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - MALUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/malus:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::dispute_old_finalized::dispute_old_finalized" - prefix: "polkadot" - # - # - zombienet-polkadot-shared-core-idle-parachain: - needs: [preflight] - if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-polkadot-shared-core-idle-parachain') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_SDK_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - # sdk tests are looking for POLKADOT_IMAGE - POLKADOT_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - CUMULUS_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-parachain-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - RUST_LOG: ${{ needs.preflight.outputs.RUST_LOG }} - ZOMBIE_PROVIDER: ${{ needs.preflight.outputs.ZOMBIE_PROVIDER }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: zombienet_test - uses: ./.github/actions/zombienet-sdk - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - test: "functional::shared_core_idle_parachain::shared_core_idle_parachain_test" - prefix: "polkadot" diff --git a/.github/workflows/zombienet_substrate.yml b/.github/workflows/zombienet_substrate.yml index 19510823c98f5..db5ed3ccc7570 100644 --- a/.github/workflows/zombienet_substrate.yml +++ b/.github/workflows/zombienet_substrate.yml @@ -2,6 +2,12 @@ name: Zombienet Substrate on: workflow_dispatch: + inputs: + test_pattern: + type: string + description: "Run tests which names match this pattern (also flaky)" + default: "" + required: false push: branches: - master @@ -12,9 +18,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: read-all + env: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 - LOCAL_DIR: "./substrate/zombienet" GHA_CLUSTER_SERVER_ADDR: "https://kubernetes.default:443" # use spot by default X_INFRA_INSTANCE: "spot" @@ -23,14 +30,11 @@ env: KUBECONFIG: "/data/config" ZOMBIE_CLEANER_DISABLED: 1 - # DB generated from commit: https://github.com/paritytech/polkadot-sdk/commit/868788a5bff3ef94869bd36432726703fe3b4e96 # TODO: As a workaround for https://github.com/paritytech/polkadot-sdk/issues/2568 the DB was generated in archive mode. # After the issue is fixed, we should replace it with a pruned version of the DB. DB_SNAPSHOT: "https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-9677807d738b951e9f6c82e5fd15518eb0ae0419.tgz" DB_BLOCK_HEIGHT: 56687 - DEFAULT_CONCURRENCY: 4 - ZOMBIENET_PROVIDER: "native" jobs: isdraft: @@ -38,123 +42,45 @@ jobs: preflight: needs: isdraft uses: ./.github/workflows/zombienet-reusable-preflight.yml + with: + tests_yaml: .github/zombienet-tests/zombienet_substrate_tests.yml + test_pattern: ${{ inputs.test_pattern }} - zombienet-substrate-0000-block-building: - needs: [preflight] - # only run if we have changes in ./substrate directory and the build workflow already finish with success status. - if: ${{ needs.preflight.outputs.changes_substrate || github.event_name == 'workflow_dispatch' }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} + zombienet-substrate-tests: + name: ${{ matrix.test.job-name }} + runs-on: ${{ matrix.test.runner-type == 'large' && (matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_LARGE_RUNNER || needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER) || (matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_DEFAULT_RUNNER || needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER) }} timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/substrate:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: "${{ needs.preflight.outputs.DEBUG }}" - steps: - - name: Checkout - uses: actions/checkout@v4 - - # - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "block-building.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/0000-block-building" - concurrency: ${{ env.DEFAULT_CONCURRENCY }} - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - - zombienet-substrate-0001-basic-warp-sync: needs: [preflight] - # only run if we have changes in ./substrate directory and the build workflow already finish with success status. - if: ${{ needs.preflight.outputs.changes_substrate || github.event_name == 'workflow_dispatch' }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 + if: ${{ (needs.preflight.outputs.changes_substrate || needs.preflight.outputs.changes_polkadot) }} container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} + image: ${{ matrix.test.use-zombienet-sdk && needs.preflight.outputs.ZOMBIENET_SDK_IMAGE || needs.preflight.outputs.ZOMBIENET_IMAGE }} options: -v /tmp/zombienet:/tmp/zombienet env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/substrate:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: "${{ needs.preflight.outputs.DEBUG }}" - steps: - - name: Checkout - uses: actions/checkout@v4 + ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/polkadot-debug:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" + DEBUG: ${{ needs.preflight.outputs.DEBUG }} + ZOMBIENET_PROVIDER: ${{ needs.preflight.outputs.ZOMBIENET_PROVIDER }} - # - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "test-warp-sync.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/0001-basic-warp-sync" - concurrency: ${{ env.DEFAULT_CONCURRENCY }} - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.preflight.outputs.TEST_MATRIX) }} - - # TODO: Disabled, fails 1 in 50 runs - zombienet-substrate-0002-validators-warp-sync: - needs: [preflight] - # only run if we have changes in ./substrate directory and the build workflow already finish with success status. - if: ${{ (needs.preflight.outputs.changes_substrate || github.event_name == 'workflow_dispatch') && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-substrate-0002-validators-warp-sync') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_LARGE_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/substrate:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - DEBUG: "${{ needs.preflight.outputs.DEBUG }}" steps: - name: Checkout uses: actions/checkout@v4 - - name: cp_spec + - name: Additional setup + if: ${{ matrix.test.additional-setup }} shell: bash - run: | - cp --remove-destination ${LOCAL_DIR}/0001-basic-warp-sync/chain-spec.json ${LOCAL_DIR}/0002-validators-warp-sync + run: ${{ matrix.test.additional-setup }} - - name: zombienet_test + - name: zombienet_test (v1) uses: ./.github/actions/zombienet with: - test: "test-validators-warp-sync.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/0002-validators-warp-sync" - concurrency: ${{ env.DEFAULT_CONCURRENCY }} + test-definition: ${{ matrix.test.test-definition }} + job-name: ${{ matrix.test.job-name }} + local-dir: ${{ matrix.test.local-dir }} + concurrency: ${{ matrix.test.concurrency || 1 }} gh-token: ${{ secrets.GITHUB_TOKEN }} build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - - - zombienet-substrate-0003-block-building-warp-sync: - needs: [preflight] - # only run if we have changes in ./substrate directory and the build workflow already finish with success status. - if: ${{ (needs.preflight.outputs.changes_substrate || github.event_name == 'workflow_dispatch') && ! contains(needs.preflight.outputs.FLAKY_TESTS, 'zombienet-substrate-0003-block-building-warp-sync') }} - runs-on: ${{ needs.preflight.outputs.ZOMBIENET_DEFAULT_RUNNER }} - timeout-minutes: 60 - container: - image: ${{ needs.preflight.outputs.ZOMBIENET_IMAGE }} - options: -v /tmp/zombienet:/tmp/zombienet - env: - ZOMBIENET_INTEGRATION_TEST_IMAGE: "${{ needs.preflight.outputs.TEMP_IMAGES_BASE }}/substrate:${{ needs.preflight.outputs.DOCKER_IMAGES_VERSION }}" - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: cp_spec - shell: bash - run: | - cp --remove-destination ${LOCAL_DIR}/0001-basic-warp-sync/chain-spec.json ${LOCAL_DIR}/0003-block-building-warp-sync - - - name: zombienet_test - uses: ./.github/actions/zombienet - with: - test: "test-block-building-warp-sync.zndsl" - local-dir: "${{ env.LOCAL_DIR }}/0003-block-building-warp-sync" - concurrency: ${{ env.DEFAULT_CONCURRENCY }} - gh-token: ${{ secrets.GITHUB_TOKEN }} - build-id: ${{ needs.preflight.outputs.BUILD_RUN_ID }} - ref-slug: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }} - diff --git a/.github/zombienet-tests/ZOMBIENET_CI.md b/.github/zombienet-tests/ZOMBIENET_CI.md new file mode 100644 index 0000000000000..4a023773e0232 --- /dev/null +++ b/.github/zombienet-tests/ZOMBIENET_CI.md @@ -0,0 +1,70 @@ +# Zombienet Tests + +This folder contains zombienet test definitions for CI execution. + +## Structure + +- **Test definitions**: YAML files defining test matrices (e.g., `zombienet_substrate_tests.yml`) +- **Flaky tests**: Listed in `.github/zombienet-flaky-tests` - tests with non-deterministic behavior +- **Parser**: `.github/scripts/parse-zombienet-tests.py` converts YAML to GitHub Actions matrix + +## Benefits + +- Easy test maintenance (add/remove tests) +- Efficient flaky test handling +- Pattern-based test execution for debugging + +## Manual Workflow Triggering + +### Prerequisites + +Before using the dispatch script, you must: + +1. **Create a branch** with your changes +2. **Create a Pull Request** for that branch +3. **Ensure CI starts building images** - the PR triggers image builds that the `preflight / wait_build_images` step depends on +4. [OPTIONAL] **Wait for image builds to complete** - zombienet tests require these images. +But if we don't wait then the job triggered by the script will wait for images if their building is in progress. + +**Important**: When you push new changes to the PR, CI will rebuild the images. Any jobs triggered after the rebuild will use the updated images. + +**Image Retention**: CI images have a 1-day retention period by default. For long-term testing (e.g., over weekends) without pushing changes, temporarily extend the retention by updating the `retention-days` value in `.github/workflows/build-publish-images.yml` to the required number of days. + +### Usage + +The dispatch script triggers GitHub Actions workflows remotely and monitors their execution. + +The script should be executed on developer's machine. + +Use `.github/scripts/dispatch-zombienet-workflow.sh`: + +```bash +Usage: .github/scripts/dispatch-zombienet-workflow.sh -w -b [-m max-triggers] [-p test-pattern] + -w: Workflow file (required) + -b: Branch name (required) + -m: Max triggers (optional, default: infinite) + -p: Test pattern (optional) +``` + +The script automatically creates a CSV file (`workflow_results_YYYYMMDD_HHMMSS.csv`) containing job results with columns: job_id, job_name, conclusion, started_at, branch, job_url. + +### Examples + +**Run workflow 5 times (respects flaky test exclusions):** +```bash +.github/scripts/dispatch-zombienet-workflow.sh -w zombienet_substrate.yml -b "my-branch" -m 5 +``` + +**Run specific test infinitely (includes flaky tests):** +```bash +.github/scripts/dispatch-zombienet-workflow.sh -w zombienet_substrate.yml -b "my-branch" -p zombienet-substrate-0000-block-building +``` + +### Requirements + +- Run from `polkadot-sdk` repository root +- Requires `gh` CLI (will prompt for login on first use) + +## Flaky Tests + +Flaky tests should have corresponding issues in the [Zombienet CI reliability project](https://github.com/orgs/paritytech/projects/216/views/1). diff --git a/.github/zombienet-tests/zombienet_cumulus_tests.yml b/.github/zombienet-tests/zombienet_cumulus_tests.yml new file mode 100644 index 0000000000000..b25d0ce5e414a --- /dev/null +++ b/.github/zombienet-tests/zombienet_cumulus_tests.yml @@ -0,0 +1,87 @@ +- job-name: "zombienet-cumulus-0001-sync_blocks_from_tip_without_connected_collator" + test-filter: "zombie_ci::sync_blocks::sync_blocks_from_tip_without_connected_collator" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +# Disabled, occasionally fails +# Takes too much CPU when spawning all nodes at the same time +# Re-enable when https://github.com/paritytech/zombienet-sdk/issues/371 is implemented +- job-name: "zombienet-cumulus-0002-pov_recovery" + test-filter: "zombie_ci::pov_recovery::pov_recovery" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0003-full_node_catching_up" + test-filter: "zombie_ci::full_node_catching_up::full_node_catching_up" + runner-type: "large" + cumulus-image: "test-parachain" + +- job-name: "zombienet-cumulus-0004-runtime_upgrade" + test-filter: "zombie_ci::runtime_upgrade::runtime_upgrade" + runner-type: "default" + cumulus-image: "test-parachain" + needs-wasm-binary: true + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0005-migrate_solo_to_para" + test-filter: "zombie_ci::migrate_solo::migrate_solo_to_para" + runner-type: "default" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +# Disabled, occasionally fails +# Takes too much CPU when spawning all nodes at the same time +# Re-enable when https://github.com/paritytech/zombienet-sdk/issues/371 is implemented +- job-name: "zombienet-cumulus-0006-rpc_collator_builds_blocks" + test-filter: "zombie_ci::rpc_collator_build_blocks::rpc_collator_builds_blocks" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0007-full_node_warp_sync" + test-filter: "zombie_ci::full_node_warp_sync::full_node_warp_sync" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0008-elastic_authoring" + test-filter: "zombie_ci::elastic_scaling::slot_based_authoring::elastic_scaling_slot_based_authoring" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +# Disabled, occasionally fails +# See https://github.com/paritytech/polkadot-sdk/issues/8986 +- job-name: "zombienet-cumulus-0009-elastic_scaling_pov_recovery" + test-filter: "zombie_ci::elastic_scaling::pov_recovery::elastic_scaling_pov_recovery" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +# Disabled, occasionally fails. +# See https://github.com/paritytech/polkadot-sdk/issues/8999 +- job-name: "zombienet-cumulus-0010-elastic_scaling_multiple_block_per_slot" + test-filter: "zombie_ci::elastic_scaling::multiple_blocks_per_slot::elastic_scaling_multiple_blocks_per_slot" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0011-dht-bootnodes" + test-filter: "zombie_ci::bootnodes::dht_bootnodes_test" + runner-type: "large" + cumulus-image: "polkadot-parachain-debug" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0012-parachain_extrinsic_gets_finalized" + test-filter: "zombie_ci::parachain_extrinsic_get_finalized::parachain_extrinsic_gets_finalized" + runner-type: "default" + cumulus-image: "polkadot-parachain-debug" + use-zombienet-sdk: true + +- job-name: "zombienet-cumulus-0013-elastic_scaling_slot_based_rp_offset" + test-filter: "zombie_ci::elastic_scaling::slot_based_rp_offset::elastic_scaling_slot_based_relay_parent_offset_test" + runner-type: "large" + cumulus-image: "test-parachain" + use-zombienet-sdk: true diff --git a/.github/zombienet-tests/zombienet_parachain-template_tests.yml b/.github/zombienet-tests/zombienet_parachain-template_tests.yml new file mode 100644 index 0000000000000..5c99d5e4c5d20 --- /dev/null +++ b/.github/zombienet-tests/zombienet_parachain-template_tests.yml @@ -0,0 +1,17 @@ +- job-name: "zombienet-parachain-template-0001-minimal_template_block_production" + test-filter: "smoke::minimal_template_block_production_test" + runner-type: "default" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-parachain-template-0002-parachain_template_block_production" + test-filter: "smoke::parachain_template_block_production_test" + runner-type: "default" + cumulus-image: "test-parachain" + use-zombienet-sdk: true + +- job-name: "zombienet-parachain-template-0003-solochain_template_block_production" + test-filter: "smoke::solochain_template_block_production_test" + runner-type: "default" + cumulus-image: "test-parachain" + use-zombienet-sdk: true diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml new file mode 100644 index 0000000000000..bfae60cc46a71 --- /dev/null +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -0,0 +1,254 @@ +# Functional tests using traditional zombienet +- job-name: "zombienet-polkadot-functional-0001-parachains-pvf" + test-definition: "0001-parachains-pvf.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + concurrency: 1 + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0002-parachains-disputes" + test-definition: "0002-parachains-disputes.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + concurrency: 1 + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0003-beefy-and-mmr" + test-definition: "0003-beefy-and-mmr.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "default" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0004-parachains-disputes-garbage-candidate" + test-definition: "0004-parachains-garbage-candidate.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0006-parachains-max-tranche0" + test-definition: "0006-parachains-max-tranche0.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0007-dispute-freshly-finalized" + test-definition: "0007-dispute-freshly-finalized.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0013-systematic-chunk-recovery" + test-definition: "0013-systematic-chunk-recovery.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility" + test-definition: "0014-chunk-fetching-network-compatibility.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + additional-setup: | + BIN_DIR="$(pwd)/bin_old" + mkdir -p $BIN_DIR + for bin in polkadot polkadot-parachain; do + OLD_NAME="$bin-old" + echo "downloading $bin as $OLD_NAME in $BIN_DIR"; + curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-v1.7.0/$bin + chmod 755 $BIN_DIR/$OLD_NAME; + done + for bin in polkadot-execute-worker polkadot-prepare-worker; do + OLD_NAME="$bin" + echo "downloading $bin as $OLD_NAME in $BIN_DIR"; + curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-v1.7.0/$bin + chmod 755 $BIN_DIR/$OLD_NAME; + done + ls -ltr $BIN_DIR + export PATH=$BIN_DIR:$PATH + echo "PATH=$PATH" >> $GITHUB_ENV + echo "OLD_SUFFIX=-old" >> $GITHUB_ENV + additional-env: + OLD_SUFFIX: "-old" + +- job-name: "zombienet-polkadot-functional-0015-coretime-shared-core" + test-definition: "0015-coretime-shared-core.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + additional-setup: | + cp --remove-destination ./polkadot/zombienet_tests/assign-core.js ./polkadot/zombienet_tests/functional + +- job-name: "zombienet-polkadot-functional-0019-coretime-collation-fetching-fairness" + test-definition: "0019-coretime-collation-fetching-fairness.zndsl" + local-dir: "./polkadot/zombienet_tests/functional" + runner-type: "large" + use-zombienet-sdk: false + additional-setup: | + cp --remove-destination ./polkadot/zombienet_tests/assign-core.js ./polkadot/zombienet_tests/functional + +# Smoke tests using traditional zombienet +- job-name: "zombienet-polkadot-smoke-0001-parachains-smoke-test" + test-definition: "0001-parachains-smoke-test.zndsl" + local-dir: "./polkadot/zombienet_tests/smoke" + runner-type: "default" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke" + test-definition: "0002-parachains-upgrade-smoke-test.zndsl" + local-dir: "./polkadot/zombienet_tests/smoke" + runner-type: "default" + use-zombienet-sdk: false + +# TODO: Disabled +- job-name: "zombienet-polkadot-smoke-0003-deregister-register-validator" + test-definition: "0003-deregister-register-validator-smoke.zndsl" + local-dir: "./polkadot/zombienet_tests/smoke" + runner-type: "default" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-smoke-0004-coretime-smoke-test" + test-definition: "0004-coretime-smoke-test.zndsl" + local-dir: "./polkadot/zombienet_tests/smoke" + runner-type: "large" + use-zombienet-sdk: false + +- job-name: "zombienet-polkadot-smoke-0005-precompile-pvf-smoke" + test-definition: "0005-precompile-pvf-smoke.zndsl" + local-dir: "./polkadot/zombienet_tests/smoke" + runner-type: "large" + use-zombienet-sdk: false + +# Misc tests using traditional zombienet +# TODO: Disabled, occasionally (1 on ~50-70 runs) fails +- job-name: "zombienet-polkadot-misc-0001-parachains-paritydb" + test-definition: "0001-paritydb.zndsl" + local-dir: "./polkadot/zombienet_tests/misc" + runner-type: "large" + use-zombienet-sdk: false + +# TODO: needs to resolve how to pass the GH_TOKEN to pods +# - job-name: "zombienet-polkadot-misc-0002-upgrade-node" +# test-definition: "0002-upgrade-node.zndsl" +# local-dir: "./polkadot/zombienet_tests/misc" +# runner-type: "large" +# additional-env: +# ZOMBIENET_INTEGRATION_TEST_IMAGE: "docker.io/parity/polkadot:latest" +# POLKADOT_PR_ARTIFACTS_URL: ${{ needs.preflight.outputs.POLKADOT_PR_ARTIFACTS_URL }} +# use-zombienet-sdk: false + +# Malus tests using traditional zombienet +- job-name: "zombienet-polkadot-malus-0001-dispute-valid" + test-definition: "0001-dispute-valid-block.zndsl" + local-dir: "./polkadot/node/malus/integrationtests" + runner-type: "large" + use-zombienet-sdk: false + +# SDK tests using zombienet-sdk +# TODO: Disabled +- job-name: "zombienet-polkadot-coretime-revenue" + test-filter: "smoke::coretime_revenue::coretime_revenue_test" + runner-type: "default" + use-zombienet-sdk: true + cumulus-image: "colander" + +- job-name: "zombienet-polkadot-elastic-scaling-slot-based-3cores" + test-filter: "elastic_scaling::slot_based_3cores::slot_based_3cores_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "test-parachain" + +# TODO: Disabled, fails very often with zombienet native provider +- job-name: "zombienet-polkadot-elastic-scaling-slot-based-12cores" + test-filter: "elastic_scaling::slot_based_12cores::slot_based_12cores_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "test-parachain" + +- job-name: "zombienet-polkadot-elastic-scaling-doesnt-break-parachains" + test-filter: "elastic_scaling::doesnt_break_parachains::doesnt_break_parachains_test" + runner-type: "large" + use-zombienet-sdk: true + +- job-name: "zombienet-polkadot-elastic-scaling-basic-3cores" + test-filter: "elastic_scaling::basic_3cores::basic_3cores_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "colander" + +- job-name: "zombienet-polkadot-functional-sync-backing" + test-filter: "functional::sync_backing::sync_backing_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "test-parachain" + +- job-name: "zombienet-polkadot-functional-async-backing-6-seconds-rate" + test-filter: "functional::async_backing_6_seconds_rate::async_backing_6_seconds_rate_test" + runner-type: "large" + use-zombienet-sdk: true + +# TODO: Disabled, occasionally (1 on ~50-100 runs) fails +- job-name: "zombienet-polkadot-functional-duplicate-collations" + test-filter: "functional::duplicate_collations::duplicate_collations_test" + runner-type: "large" + use-zombienet-sdk: true + +# TODO: Disabled, occasionally (2 on ~50-70 runs) fails +- job-name: "zombienet-polkadot-disputes-slashing" + test-filter: "disabling::slashing" + runner-type: "default" + use-zombienet-sdk: true + +# TODO: Disabled, occasionally (1 on ~50-100 runs) fails +- job-name: "zombienet-polkadot-functional-spam-statement-distribution-requests" + test-filter: "functional::spam_statement_distribution_requests::spam_statement_distribution_requests_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "colander" + +- job-name: "zombienet-polkadot-approval-voting-coalescing" + test-filter: "functional::approval_voting_coalescing::approval_voting_coalescing_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "colander" + +- job-name: "zombienet-polkadot-approved-peer-mixed-validators" + test-filter: "functional::approved_peer_mixed_validators::approved_peer_mixed_validators_test" + runner-type: "large" + use-zombienet-sdk: true + cumulus-image: "colander" + additional-setup: | + BIN_DIR="$(pwd)/bin_old" + mkdir -p $BIN_DIR + for bin in polkadot polkadot-parachain; do + OLD_NAME="$bin-old" + echo "downloading $bin as $OLD_NAME in $BIN_DIR"; + curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2503/$bin + chmod 755 $BIN_DIR/$OLD_NAME; + done + for bin in polkadot-execute-worker polkadot-prepare-worker; do + OLD_NAME="$bin" + echo "downloading $bin as $OLD_NAME in $BIN_DIR"; + curl -L -o $BIN_DIR/$OLD_NAME https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2503/$bin + chmod 755 $BIN_DIR/$OLD_NAME; + done + ls -ltr $BIN_DIR + export PATH=$BIN_DIR:$PATH + echo "PATH=$PATH" >> $GITHUB_ENV + additional-env: + OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" + OLD_POLKADOT_COMMAND: "polkadot-old" + +- job-name: "zombienet-polkadot-functional-validator-disabling" + test-filter: "functional::validator_disabling::validator_disabling_test" + runner-type: "large" + use-zombienet-sdk: true + +- job-name: "zombienet-polkadot-dispute-old-finalized" + test-filter: "functional::dispute_old_finalized::dispute_old_finalized" + runner-type: "large" + use-zombienet-sdk: true + +- job-name: "zombienet-polkadot-shared-core-idle-parachain" + test-filter: "functional::shared_core_idle_parachain::shared_core_idle_parachain_test" + runner-type: "large" + use-zombienet-sdk: true diff --git a/.github/zombienet-tests/zombienet_substrate_tests.yml b/.github/zombienet-tests/zombienet_substrate_tests.yml new file mode 100644 index 0000000000000..288e490e6fdef --- /dev/null +++ b/.github/zombienet-tests/zombienet_substrate_tests.yml @@ -0,0 +1,32 @@ +- job-name: "zombienet-substrate-0000-block-building" + test-definition: "block-building.zndsl" + local-dir: "./substrate/zombienet/0000-block-building" + runner-type: "default" + concurrency: 4 + use-zombienet-sdk: false + +- job-name: "zombienet-substrate-0001-basic-warp-sync" + test-definition: "test-warp-sync.zndsl" + local-dir: "./substrate/zombienet/0001-basic-warp-sync" + runner-type: "default" + concurrency: 4 + use-zombienet-sdk: false + +# TODO: Disabled, fails 1 in 50 runs +- job-name: "zombienet-substrate-0002-validators-warp-sync" + test-definition: "test-validators-warp-sync.zndsl" + local-dir: "./substrate/zombienet/0002-validators-warp-sync" + runner-type: "default" + concurrency: 4 + use-zombienet-sdk: false + additional-setup: | + cp --remove-destination ./substrate/zombienet/0001-basic-warp-sync/chain-spec.json ./substrate/zombienet/0002-validators-warp-sync + +- job-name: "zombienet-substrate-0003-block-building-warp-sync" + test-definition: "test-block-building-warp-sync.zndsl" + local-dir: "./substrate/zombienet/0003-block-building-warp-sync" + runner-type: "default" + concurrency: 4 + use-zombienet-sdk: false + additional-setup: | + cp --remove-destination ./substrate/zombienet/0001-basic-warp-sync/chain-spec.json ./substrate/zombienet/0003-block-building-warp-sync