Automated Cascade of Benchmarks #405
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automated Cascade of Benchmarks | |
| #description: | | |
| # This workflow runs a series of benchmarks in a cascade fashion. | |
| # It is triggered manually. The benchmarks are defined in `benchmark_cascade/benchmark-configs.json`. | |
| # Benchmarks are run in parallel. Instructions can be found in `benchmark_cascade/README.md`. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| benchmark_set: | |
| description: "Benchmark set to run (from benchmark_cascade/benchmark-configs.json). Default is 'single': ghcr/dev, dbpedia-openai-1M-1536-angular, qdrant-rps-m-16-ef-128, feature_flags=true." | |
| default: "single" | |
| params_override: | |
| description: "Override parameters for the benchmark set (JSON format, e.g. '{\"params\": {\"qdrant_version\": [\"ghcr/dev\", \"docker/master\"], \"dataset\": [\"glove-100-angular\"]}}')" | |
| default: "{}" | |
| build_vector_db_image: | |
| description: "Build vector DB image from source branch and use it (false by default)" | |
| default: "false" | |
| machines_per_bench: | |
| type: boolean | |
| description: "Create/destroy machines for each benchmark run (true) or reuse machines across batches (false)" | |
| default: false | |
| process_results: | |
| type: boolean | |
| description: "Skip all the jobs except for processBenchmarks (true) or run a normal cascade (false)" | |
| default: false | |
| workflow_run_ids: | |
| description: "Internal use, comma-separated list of workflow run IDs to process (used when process_results=true or for cascade tracking)" | |
| default: "" | |
| current_batch: | |
| description: "Internal use, current batch index" | |
| default: "0" | |
| machines_info: | |
| description: "Internal use, JSON array of machine pairs to use when machines_per_bench is false, e.g. '[{\"server_name\":\"server-0\",\"client_name\":\"client-0\"},{\"server_name\":\"server-1\",\"client_name\":\"client-1\"}]'" | |
| default: "[]" | |
| region: | |
| description: "Hetzner region to run the benchmarks in" | |
| default: "fsn1" | |
| server_machine_type: | |
| description: "Hetzner server machine type to run the benchmarks in" | |
| default: "cpx42" | |
| concurrency: | |
| group: hetzner-machines | |
| env: | |
| # Common environment variables | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }} | |
| POSTGRES_TABLE: benchmark_manual | |
| jobs: | |
| prepareBenchmarks: | |
| name: Prepare Benchmarks | |
| runs-on: ubuntu-latest | |
| if: inputs.process_results == false | |
| outputs: | |
| matrix: ${{ steps.prepare.outputs.matrix }} | |
| has_next_batch: ${{ steps.prepare.outputs.has_next_batch }} | |
| next_batch: ${{ steps.prepare.outputs.next_batch }} | |
| benchmark_set: ${{ steps.prepare.outputs.benchmark_set }} | |
| workflow_run_ids: ${{ steps.prepare.outputs.workflow_run_ids }} | |
| machines_info: ${{ steps.prepare.outputs.machines_info }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 | |
| with: | |
| python-version: '3.10.12' | |
| - name: Prepare benchmark matrix | |
| id: prepare | |
| run: | | |
| CONFIG_FILE="benchmark_cascade/benchmark-configs.json" | |
| BENCHMARK_SET="${{ inputs.benchmark_set }}" | |
| CURRENT_BATCH=${{ inputs.current_batch || 0 }} | |
| BATCH_SIZE=12 | |
| echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT | |
| # Accumulate workflow run IDs | |
| WORKFLOW_RUN_IDS="${{ inputs.workflow_run_ids }}" | |
| if [ -n "$WORKFLOW_RUN_IDS" ]; then | |
| WORKFLOW_RUN_IDS="${WORKFLOW_RUN_IDS},${{ github.run_id }}" | |
| else | |
| WORKFLOW_RUN_IDS="${{ github.run_id }}" | |
| fi | |
| echo "workflow_run_ids=$WORKFLOW_RUN_IDS" >> "$GITHUB_OUTPUT" | |
| # Benchmark set - read from config file | |
| if [ ! -f "$CONFIG_FILE" ]; then | |
| echo "Configuration file $CONFIG_FILE not found!" | |
| exit 1 | |
| fi | |
| BENCHMARK_CONFIGS=$(jq -r ".benchmark_sets.\"$BENCHMARK_SET\"" "$CONFIG_FILE") | |
| if [ "$BENCHMARK_CONFIGS" = "null" ]; then | |
| echo "Benchmark set '$BENCHMARK_SET' not found in configuration file" | |
| exit 1 | |
| fi | |
| # Expand per-item "repeat" field: each item with "repeat": N becomes N copies (default 1) | |
| BENCHMARK_CONFIGS=$(jq '[.[] | . as $c | range($c.repeat // 1) | $c | del(.repeat)]' <<< "$BENCHMARK_CONFIGS") | |
| # Apply params_override if provided | |
| PARAMS_OVERRIDE='${{ inputs.params_override }}' | |
| if [ "$PARAMS_OVERRIDE" != "{}" ]; then | |
| echo "Applying params_override: $PARAMS_OVERRIDE" | |
| # Parse the override params | |
| OVERRIDE_PARAMS=$(echo "$PARAMS_OVERRIDE" | jq -r '.params // {}') | |
| if [ "$OVERRIDE_PARAMS" != "{}" ]; then | |
| # Generate all parameter combinations using separate Python script | |
| if ! BENCHMARK_CONFIGS=$(python3 benchmark_cascade/generate_configs.py "$OVERRIDE_PARAMS" "$BENCHMARK_CONFIGS"); then | |
| echo "Error: Failed to generate benchmark configurations" | |
| exit 1 | |
| fi | |
| echo "Generated $(echo "$BENCHMARK_CONFIGS" | jq length) configurations after applying params_override" | |
| fi | |
| fi | |
| TOTAL_CONFIGS=$(echo "$BENCHMARK_CONFIGS" | jq length) | |
| START_INDEX=$((CURRENT_BATCH * BATCH_SIZE)) | |
| END_INDEX=$((START_INDEX + BATCH_SIZE - 1)) | |
| echo "Batch info: Current batch=$CURRENT_BATCH, Start=$START_INDEX, End=$END_INDEX, Total=$TOTAL_CONFIGS" | |
| # Check if we need more batches | |
| NEXT_START=$((END_INDEX + 1)) | |
| if [ $NEXT_START -lt $TOTAL_CONFIGS ]; then | |
| echo "has_next_batch=true" >> $GITHUB_OUTPUT | |
| echo "next_batch=$((CURRENT_BATCH + 1))" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_next_batch=false" >> $GITHUB_OUTPUT | |
| echo "next_batch=0" >> $GITHUB_OUTPUT | |
| fi | |
| # Create matrix for current batch | |
| MATRIX_JSON="[" | |
| FIRST=true | |
| for i in $(seq $START_INDEX $END_INDEX); do | |
| if [ $i -ge $TOTAL_CONFIGS ]; then | |
| break | |
| fi | |
| if [ "$FIRST" = false ]; then | |
| MATRIX_JSON="${MATRIX_JSON}," | |
| fi | |
| FIRST=false | |
| CONFIG=$(echo "$BENCHMARK_CONFIGS" | jq -r ".[$i]") | |
| QDRANT_VERSION=$(echo "$CONFIG" | jq -r '.qdrant_version') | |
| DATASET=$(echo "$CONFIG" | jq -r '.dataset') | |
| ENGINE_CONFIG=$(echo "$CONFIG" | jq -r '.engine_config') | |
| FEATURE_FLAGS=$(echo "$CONFIG" | jq -r '.feature_flags_all') | |
| QDRANT_VERSION_SANITIZED=$(echo "$QDRANT_VERSION" | sed 's|/|-|g') | |
| MATRIX_JSON="${MATRIX_JSON}{\"index\": $i, \"qdrant_version\": \"$QDRANT_VERSION\", \"qdrant_version_sanitized\": \"$QDRANT_VERSION_SANITIZED\", \"dataset\": \"$DATASET\", \"engine_config\": \"$ENGINE_CONFIG\", \"feature_flags_all\": $FEATURE_FLAGS}" | |
| done | |
| MATRIX_JSON="${MATRIX_JSON}]" | |
| echo "Current batch: $CURRENT_BATCH (indices $START_INDEX-$END_INDEX of $TOTAL_CONFIGS total)" | |
| echo "Matrix JSON: $MATRIX_JSON" | |
| echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT | |
| # Generate machines_info array for the current batch | |
| MACHINES_INFO="[" | |
| MACHINES_FIRST=true | |
| for i in $(seq $START_INDEX $END_INDEX); do | |
| if [ $i -ge $TOTAL_CONFIGS ]; then | |
| break | |
| fi | |
| if [ "$MACHINES_FIRST" = false ]; then | |
| MACHINES_INFO="${MACHINES_INFO}," | |
| fi | |
| MACHINES_FIRST=false | |
| SERVER_NAME="benchmark-cascade-server-${i}" | |
| CLIENT_NAME="benchmark-cascade-server-${i}" | |
| MACHINES_INFO="${MACHINES_INFO}{\"server_name\":\"$SERVER_NAME\",\"client_name\":\"$CLIENT_NAME\"}" | |
| done | |
| MACHINES_INFO="${MACHINES_INFO}]" | |
| echo "Generated machines_info: $MACHINES_INFO" | |
| echo "machines_info=$MACHINES_INFO" >> $GITHUB_OUTPUT | |
| buildImage: | |
| name: Build Vector DB Image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| if: ${{ inputs.build_vector_db_image == 'true' && inputs.process_results == false && (inputs.current_batch == '' || inputs.current_batch == '0') }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: prepare image tag | |
| id: prepare-tag | |
| shell: bash | |
| run: | | |
| branch_tmp="${{ github.ref_name }}" | |
| branch=${branch_tmp//\//-} # replace all / with - | |
| tag="ghcr.io/${{ github.repository_owner }}/vector-db-benchmark:${branch}" | |
| echo "Use tag ${tag}" | |
| echo "tag=${tag}" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 | |
| - name: Login to ghcr.io | |
| uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build Vector DB image | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.prepare-tag.outputs.tag }} | |
| provenance: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| setupMachines: | |
| name: Setup Machines ${{ matrix.config.index }} | |
| needs: prepareBenchmarks | |
| if: inputs.process_results == false | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} | |
| outputs: | |
| server_name: ${{ steps.extract_names.outputs.server_name }} | |
| client_name: ${{ steps.extract_names.outputs.client_name }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Extract Machine Names | |
| id: extract_names | |
| uses: ./.github/workflows/actions/extract-machine-names | |
| with: | |
| machines_per_bench: ${{ inputs.machines_per_bench }} | |
| current_batch: ${{ inputs.current_batch || 0 }} | |
| current_index: ${{ matrix.config.index }} | |
| job_index: ${{ strategy.job-index }} | |
| machines_info: ${{ (inputs.current_batch != 0 && inputs.machines_info) || needs.prepareBenchmarks.outputs.machines_info }} | |
| - name: Setup CI | |
| if: ${{ inputs.machines_per_bench == true || inputs.current_batch == 0 }} | |
| run: bash -x tools/setup_ci.sh | |
| - name: Create Server | |
| if: ${{ inputs.machines_per_bench == true || inputs.current_batch == 0 }} | |
| uses: ./.github/workflows/actions/create-server-with-retry | |
| with: | |
| server_name: ${{ steps.extract_names.outputs.server_name }} | |
| server_type: ${{ inputs.server_machine_type }} | |
| region: ${{ inputs.region }} | |
| max_retries: 5 | |
| - name: Create Client | |
| if: ${{ (inputs.machines_per_bench == true || inputs.current_batch == 0) && steps.extract_names.outputs.client_name != steps.extract_names.outputs.server_name }} | |
| uses: ./.github/workflows/actions/create-server-with-retry | |
| with: | |
| server_name: ${{ steps.extract_names.outputs.client_name }} | |
| server_type: "cpx32" | |
| region: ${{ inputs.region }} | |
| max_retries: 5 | |
| runBenchmarks: | |
| name: Run Benchmark ${{ matrix.config.index }} | |
| needs: [prepareBenchmarks, setupMachines, buildImage] | |
| if: ${{ !cancelled() && inputs.process_results == false && needs.prepareBenchmarks.result == 'success' && needs.setupMachines.result == 'success' && (needs.buildImage.result == 'success' || needs.buildImage.result == 'skipped') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Extract Machine Names | |
| id: extract_names | |
| uses: ./.github/workflows/actions/extract-machine-names | |
| with: | |
| machines_per_bench: ${{ inputs.machines_per_bench }} | |
| current_batch: ${{ inputs.current_batch || 0 }} | |
| current_index: ${{ matrix.config.index }} | |
| job_index: ${{ strategy.job-index }} | |
| machines_info: ${{ (inputs.current_batch != 0 && inputs.machines_info) || needs.prepareBenchmarks.outputs.machines_info }} | |
| - name: Run benchmark | |
| run: | | |
| export QDRANT_VERSION=${{ matrix.config.qdrant_version }} | |
| export DATASETS=${{ matrix.config.dataset }} | |
| export ENGINE_NAME=${{ matrix.config.engine_config }} | |
| export QDRANT__FEATURE_FLAGS__ALL=${{ matrix.config.feature_flags_all }} | |
| export SERVER_NAME=${{ steps.extract_names.outputs.server_name }} | |
| export CLIENT_NAME=${{ steps.extract_names.outputs.client_name }} | |
| export FETCH_ALL_RESULTS="true" | |
| if [ "${{ inputs.build_vector_db_image }}" = "true" ]; then | |
| branch_tmp="${{ github.ref_name }}" | |
| branch=${branch_tmp//\//-} | |
| export VECTOR_DB_BENCHMARK_IMAGE="ghcr.io/${{ github.repository_owner }}/vector-db-benchmark:${branch}" | |
| export GHCR_USERNAME=${{ github.repository_owner }} | |
| export GHCR_PASSWORD=${{ secrets.GITHUB_TOKEN }} | |
| fi | |
| bash -x tools/setup_ci.sh | |
| # Clear previous results on remote client to avoid mixing data from different runs | |
| source tools/ssh.sh | |
| CLOUD_NAME=${CLOUD_NAME:-"hetzner"} | |
| IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME") | |
| ssh_with_retry -o StrictHostKeyChecking=no "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" || true | |
| bash -x tools/run_ci.sh | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: results-${{ matrix.config.qdrant_version_sanitized }}-bench-${{ matrix.config.dataset }}-${{ matrix.config.engine_config }}-${{ matrix.config.index }} | |
| path: results/ | |
| retention-days: 7 | |
| - name: Teardown machines | |
| if: ${{ always() && (inputs.machines_per_bench == true || needs.prepareBenchmarks.outputs.has_next_batch == 'false') }} | |
| continue-on-error: true | |
| uses: ./.github/workflows/actions/extract-machine-names | |
| with: | |
| machines_per_bench: ${{ inputs.machines_per_bench }} | |
| current_batch: ${{ inputs.current_batch || 0 }} | |
| current_index: ${{ matrix.config.index }} | |
| job_index: ${{ strategy.job-index }} | |
| machines_info: ${{ (inputs.current_batch != 0 && inputs.machines_info) || needs.prepareBenchmarks.outputs.machines_info }} | |
| id: teardown_names | |
| - name: Perform teardown | |
| if: ${{ always() && (inputs.machines_per_bench == true || needs.prepareBenchmarks.outputs.has_next_batch == 'false') }} | |
| continue-on-error: true | |
| run: | | |
| export SERVER_NAME=${{ steps.teardown_names.outputs.server_name }} | |
| export CLIENT_NAME=${{ steps.teardown_names.outputs.client_name }} | |
| bash -x tools/setup_ci.sh | |
| bash -x tools/tear_down.sh | |
| processBenchmarks: | |
| name: Process All Benchmark Results | |
| needs: [prepareBenchmarks, runBenchmarks] | |
| if: always() && (needs.prepareBenchmarks.outputs.has_next_batch == 'false' || inputs.process_results == true) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:3.11-slim | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Install dependencies | |
| run: pip install pandas jupyter nbconvert | |
| - name: Download artifacts from specific workflow runs | |
| shell: bash | |
| run: | | |
| # Install GitHub CLI | |
| apt-get update && apt-get install -y curl unzip jq | |
| curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null | |
| apt-get update && apt-get install -y gh | |
| # Download artifacts from specific workflow runs | |
| mkdir -p artifacts | |
| # Get workflow run IDs from the accumulated list | |
| # If process_results is true, use the input workflow_run_ids instead | |
| if [ "${{ inputs.process_results }}" = "true" ]; then | |
| WORKFLOW_RUN_IDS="${{ inputs.workflow_run_ids }}" | |
| echo "Using workflow run IDs from input (process_results=true): $WORKFLOW_RUN_IDS" | |
| else | |
| WORKFLOW_RUN_IDS="${{ needs.prepareBenchmarks.outputs.workflow_run_ids }}" | |
| echo "Using workflow run IDs from prepareBenchmarks: $WORKFLOW_RUN_IDS" | |
| fi | |
| # Convert comma-separated list to array and download artifacts from each run | |
| echo "$WORKFLOW_RUN_IDS" | tr ',' '\n' | while read run_id; do | |
| if [ -n "$run_id" ]; then | |
| echo "Downloading artifacts from run $run_id" | |
| # List artifacts for this run | |
| gh api repos/${{ github.repository }}/actions/runs/$run_id/artifacts \ | |
| --jq '.artifacts[] | select(.name | startswith("results-")) | {name: .name, url: .archive_download_url}' \ | |
| | while IFS= read -r line; do | |
| artifact_name=$(echo "$line" | jq -r '.name') | |
| artifact_url=$(echo "$line" | jq -r '.url') | |
| if [ -n "$artifact_url" ] && [ "$artifact_url" != "null" ]; then | |
| echo "Downloading $artifact_name from run $run_id" | |
| gh api "$artifact_url" > "artifacts/${run_id}-${artifact_name}.zip" || true | |
| fi | |
| done | |
| fi | |
| done | |
| # Keep zips for version parsing, don't extract yet | |
| ls -la artifacts/ | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload all artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: all-unprocessed-results | |
| path: artifacts/ | |
| retention-days: 7 | |
| - name: Process results by Qdrant version | |
| run: | | |
| ROOT_DIR=$(pwd) | |
| mkdir -p final_results | |
| cd artifacts | |
| # Check if there are any zip files | |
| if ! ls *.zip >/dev/null 2>&1; then | |
| echo "No artifacts found to process" | |
| exit 0 | |
| fi | |
| # Extract version from artifact names (between "results-" and "-bench-") | |
| VERSIONS=$(ls *.zip | sed -n 's/^[0-9]*-results-\(.*\)-bench-.*/\1/p' | sort -u) | |
| echo "Found the following Qdrant versions:" | |
| echo "$VERSIONS" | |
| # Process each version separately | |
| for VERSION in $VERSIONS; do | |
| echo "Processing version: $VERSION" | |
| # Create results directory for this version | |
| rm -rf "$ROOT_DIR/results" | |
| mkdir -p "$ROOT_DIR/results" | |
| # Extract only artifacts for this version | |
| for zip_file in *results-${VERSION}-bench-*.zip; do | |
| if [ -f "$zip_file" ]; then | |
| echo "Extracting $zip_file" | |
| unzip -o "$zip_file" -d "$ROOT_DIR/results/" | |
| fi | |
| done | |
| # Run Jupyter notebook | |
| cd "$ROOT_DIR/scripts" | |
| jupyter nbconvert --to notebook --execute process-benchmarks.ipynb --output process-benchmarks-executed.ipynb | |
| # Copy results to final_results with version suffix | |
| if [ -f "results.json" ]; then | |
| cp results.json "$ROOT_DIR/final_results/results-${VERSION}.json" | |
| echo "Created final_results/results-${VERSION}.json" | |
| else | |
| echo "Warning: results.json not found for version $VERSION" | |
| fi | |
| # Go back to artifacts directory for next iteration | |
| cd "$ROOT_DIR/artifacts" | |
| # Clean up results directory for next iteration | |
| rm -rf "$ROOT_DIR/results" | |
| done | |
| cd "$ROOT_DIR" | |
| echo "Final results:" | |
| ls -la final_results/ | |
| - name: Upload processed results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: final-processed-results | |
| path: final_results/ | |
| retention-days: 7 | |
| triggerNextBatch: | |
| name: Trigger Next Batch | |
| needs: [prepareBenchmarks, setupMachines, runBenchmarks, buildImage] | |
| if: always() && (needs.prepareBenchmarks.outputs.has_next_batch == 'true' && inputs.process_results == false && (needs.buildImage.result == 'success' || needs.buildImage.result == 'skipped')) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check setupMachines job status | |
| id: check_setup | |
| run: | | |
| SETUP_RESULT="${{ needs.setupMachines.result }}" | |
| MACHINES_PER_BENCH="${{ inputs.machines_per_bench }}" | |
| echo "setupMachines job result: $SETUP_RESULT" | |
| echo "machines_per_bench: $MACHINES_PER_BENCH" | |
| if [ "$MACHINES_PER_BENCH" = "false" ] && [ "$SETUP_RESULT" != "success" ]; then | |
| echo "machines_per_bench is false and setupMachines job did not succeed (result: $SETUP_RESULT)" | |
| echo "Skipping trigger of next batch" | |
| echo "should_trigger=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "Either machines_per_bench is true or setupMachines succeeded" | |
| echo "Proceeding with next batch trigger" | |
| echo "should_trigger=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Trigger next batch | |
| if: steps.check_setup.outputs.should_trigger == 'true' | |
| run: | | |
| echo "Workflows ids ran so far: ${{ needs.prepareBenchmarks.outputs.workflow_run_ids }}" | |
| echo "🚀 Triggering next batch of benchmarks..." | |
| echo "Next batch index: ${{ needs.prepareBenchmarks.outputs.next_batch }}" | |
| # Use machines_info - from prepareBenchmarks for batch 0, from inputs for batch 1+ | |
| CURRENT_BATCH=${{ inputs.current_batch || 0 }} | |
| if [ "$CURRENT_BATCH" = "0" ]; then | |
| echo "Initial workflow (batch 0), using machines_info from prepareBenchmarks" | |
| MACHINES_INFO='${{ needs.prepareBenchmarks.outputs.machines_info }}' | |
| else | |
| echo "Subsequent workflow (batch $CURRENT_BATCH), using machines_info from inputs" | |
| MACHINES_INFO='${{ inputs.machines_info }}' | |
| fi | |
| echo "Using machines_info: $MACHINES_INFO" | |
| PARAMS_OVERRIDE='${{ inputs.params_override }}' | |
| # Escape the JSON for embedding in the curl command | |
| ESCAPED_PARAMS=$(echo "$PARAMS_OVERRIDE" | jq -Rs .) | |
| ESCAPED_MACHINES_INFO=$(echo "$MACHINES_INFO" | jq -Rs .) | |
| # Trigger next batch via workflow dispatch | |
| curl -X POST \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/workflows/manual-benchmarks-cascade.yaml/dispatches" \ | |
| -d "{ | |
| \"inputs\": { | |
| \"benchmark_set\": \"${{ needs.prepareBenchmarks.outputs.benchmark_set }}\", | |
| \"current_batch\": \"${{ needs.prepareBenchmarks.outputs.next_batch }}\", | |
| \"workflow_run_ids\": \"${{ needs.prepareBenchmarks.outputs.workflow_run_ids }}\", | |
| \"params_override\": $ESCAPED_PARAMS, | |
| \"machines_per_bench\": \"${{ inputs.machines_per_bench }}\", | |
| \"machines_info\": $ESCAPED_MACHINES_INFO, | |
| \"build_vector_db_image\": \"${{ inputs.build_vector_db_image }}\" | |
| }, | |
| \"ref\": \"${{ github.ref }}\" | |
| }" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| cleanupMachines: | |
| name: Cleanup All Machines | |
| needs: [prepareBenchmarks, setupMachines, runBenchmarks] | |
| if: | | |
| always() && inputs.process_results == false && ( | |
| (inputs.machines_per_bench == false && needs.prepareBenchmarks.outputs.has_next_batch == 'false') || | |
| (inputs.machines_per_bench == false && inputs.current_batch == 0 && needs.setupMachines.result == 'failure') | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ github.ref }} | |
| - uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Cleanup all machines | |
| continue-on-error: true | |
| run: | | |
| # Get machines_info array - from inputs for batch 1+, from prepareBenchmarks for batch 0 | |
| CURRENT_BATCH=${{ inputs.current_batch || 0 }} | |
| if [ "$CURRENT_BATCH" != "0" ]; then | |
| echo "Using machines_info from inputs for batch $CURRENT_BATCH" | |
| MACHINES_INFO='${{ inputs.machines_info }}' | |
| else | |
| echo "Using machines_info from prepareBenchmarks for batch $CURRENT_BATCH" | |
| MACHINES_INFO='${{ needs.prepareBenchmarks.outputs.machines_info }}' | |
| fi | |
| echo "Machines to cleanup: $MACHINES_INFO" | |
| # Setup tools | |
| bash -x tools/setup_ci.sh | |
| # Parse machines_info and store in arrays to avoid subshell issues | |
| MACHINE_COUNT=$(echo "$MACHINES_INFO" | jq 'length') | |
| echo "Found $MACHINE_COUNT machine pairs to cleanup" | |
| # Arrays to store server and client names | |
| declare -a SERVER_NAMES | |
| declare -a CLIENT_NAMES | |
| declare -a PIDS | |
| # Extract all machine pairs first | |
| for i in $(seq 0 $((MACHINE_COUNT - 1))); do | |
| SERVER_NAMES[$i]=$(echo "$MACHINES_INFO" | jq -r ".[$i].server_name") | |
| CLIENT_NAMES[$i]=$(echo "$MACHINES_INFO" | jq -r ".[$i].client_name") | |
| done | |
| # Launch all teardown processes in parallel | |
| for i in $(seq 0 $((MACHINE_COUNT - 1))); do | |
| SERVER_NAME="${SERVER_NAMES[$i]}" | |
| CLIENT_NAME="${CLIENT_NAMES[$i]}" | |
| echo "Launching teardown for: SERVER_NAME=$SERVER_NAME, CLIENT_NAME=$CLIENT_NAME" | |
| # Run teardown in background and capture PID | |
| ( | |
| export SERVER_NAME | |
| export CLIENT_NAME | |
| if bash -x tools/tear_down.sh; then | |
| echo "Successfully cleaned up $SERVER_NAME and $CLIENT_NAME" | |
| else | |
| echo "Warning: Failed to cleanup $SERVER_NAME and $CLIENT_NAME (may already be deleted)" | |
| fi | |
| ) & | |
| # Store the PID | |
| PIDS[$i]=$! | |
| done | |
| # Wait for all background teardown processes to complete | |
| echo "Waiting for all $MACHINE_COUNT teardown processes to complete..." | |
| JOBS_FAILED=0 | |
| for i in $(seq 0 $((MACHINE_COUNT - 1))); do | |
| PID=${PIDS[$i]} | |
| SERVER_NAME="${SERVER_NAMES[$i]}" | |
| CLIENT_NAME="${CLIENT_NAMES[$i]}" | |
| echo "Waiting for teardown of $SERVER_NAME and $CLIENT_NAME (PID: $PID)..." | |
| if wait $PID; then | |
| echo "Teardown completed for $SERVER_NAME and $CLIENT_NAME" | |
| else | |
| echo "Teardown failed for $SERVER_NAME and $CLIENT_NAME" | |
| JOBS_FAILED=$((JOBS_FAILED + 1)) | |
| fi | |
| done | |
| if [ $JOBS_FAILED -gt 0 ]; then | |
| echo "Warning: $JOBS_FAILED teardown operations failed" | |
| else | |
| echo "All teardown operations completed successfully" | |
| fi | |
| echo "Cleanup process completed" | |
| cleanupImage: | |
| name: Cleanup Vector DB Image | |
| needs: [prepareBenchmarks, runBenchmarks] | |
| if: | | |
| always() && inputs.build_vector_db_image == 'true' && inputs.process_results == false && | |
| needs.prepareBenchmarks.outputs.has_next_batch == 'false' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| steps: | |
| - name: Delete image from ghcr.io | |
| continue-on-error: true | |
| run: | | |
| branch_tmp="${{ github.ref_name }}" | |
| branch=${branch_tmp//\//-} | |
| PACKAGE_NAME="vector-db-benchmark" | |
| TAG="${branch}" | |
| if [ "$TAG" = "master" ]; then | |
| echo "Refusing to delete master image, skipping cleanup" | |
| exit 0 | |
| fi | |
| echo "Cleaning up image ghcr.io/${{ github.repository_owner }}/${PACKAGE_NAME}:${TAG}" | |
| # Find the package version ID by tag | |
| VERSION_ID=$(gh api \ | |
| "/orgs/${{ github.repository_owner }}/packages/container/${PACKAGE_NAME}/versions?per_page=100" \ | |
| --jq ".[] | select(.metadata.container.tags[] == \"${TAG}\") | .id") || true | |
| if [ -n "$VERSION_ID" ]; then | |
| echo "Found version ID: $VERSION_ID, deleting..." | |
| gh api --method DELETE \ | |
| "/orgs/${{ github.repository_owner }}/packages/container/${PACKAGE_NAME}/versions/${VERSION_ID}" \ | |
| && echo "Image deleted successfully" \ | |
| || echo "Warning: Failed to delete image (may require manual cleanup)" | |
| else | |
| echo "Warning: Could not find image version with tag ${TAG}" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |