From 847ba6d61aa8bd8973a3b7ff4fa0e887da36e75e Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Wed, 28 May 2025 10:10:04 +0100 Subject: [PATCH 1/3] Benchmarks workflow prints diff when changes found Motivation: The current method of parsing the ASCII table output is brittle and because the text output and the stored thresholds in JSON have different precisions doesn't always work. Modifications: * In the event that a deviation is found re-run the benchmarks in update mode and print the git diff to the logs. * The threshold update script scrapes this from the logs and applys it locally so that it can then be committed. Result: More reliable threshold updates. --- .github/workflows/benchmarks.yml | 5 +- dev/thresholds-from-benchmark-output.sh | 224 +++++++++--------------- scripts/check_benchmark_thresholds.sh | 46 +++++ 3 files changed, 130 insertions(+), 145 deletions(-) create mode 100755 scripts/check_benchmark_thresholds.sh diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 22319c310a9..0d79a3d6d72 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -74,8 +74,9 @@ jobs: - id: generate-matrix run: echo "benchmarks-matrix=$(curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT" env: - MATRIX_LINUX_COMMAND: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/" - MATRIX_LINUX_SETUP_COMMAND: "swift --version && apt-get update -y -q && apt-get install -y -q libjemalloc-dev" + MATRIX_LINUX_COMMAND_: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/" + MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check_benchmark_thresholds.sh | BENCHMARK_PACKAGE_PATH=${{ inputs.benchmark_package_path }} bash" + MATRIX_LINUX_SETUP_COMMAND: "swift --version && apt-get update -y -q && apt-get install -y -q curl libjemalloc-dev && git config --global --add safe.directory /$(basename ${{ github.workspace }})" MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }} MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }} MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }} diff --git a/dev/thresholds-from-benchmark-output.sh b/dev/thresholds-from-benchmark-output.sh index 5e91ece96a1..c14642c9562 100755 --- a/dev/thresholds-from-benchmark-output.sh +++ b/dev/thresholds-from-benchmark-output.sh @@ -22,14 +22,16 @@ log() { printf -- "** %s\n" "$*" >&2; } error() { printf -- "** ERROR: %s\n" "$*" >&2; } fatal() { error "$@"; exit 1; } -pr_url="${FETCH_PR_URL:=""}" +pr_url="${PR_URL:=""}" +run_url="${RUN_URL:=""}" + output_dir="${OUTPUT_DIRECTORY:=""}" -if [ -z "$pr_url" ]; then - fatal "Pull request URL must be specified." +if [ -z "$pr_url" ] && [ -z "$run_url" ]; then + fatal "Pull request URL or workflow run URL must be specified." fi -if [ -z "$output_dir" ]; then +if [ -n "$pr_url" ] && [ -z "$output_dir" ]; then fatal "Output directory must be specified." fi @@ -38,27 +40,13 @@ GH_BIN="${GH_BIN:-$(which gh)}" || fatal "GH_BIN unset and no gh on PATH" JQ_BIN="${JQ_BIN:-$(which jq)}" || fatal "JQ_BIN unset and no jq on PATH" YQ_BIN="${YQ_BIN:-$(which yq)}" || fatal "YQ_BIN unset and no yq on PATH" -# Parsing constants -readonly benchmark_title_name_prefix="than threshold for " -readonly table_pipe="│" -readonly table_current_run_heading="Current_run" - fetch_checks_for_pr() { pr_url=$1 "$GH_BIN" pr checks "$pr_url" | grep Benchmarks | grep -v Construct } -fetch_check_logs() { - repo=$1 - job=$2 - - # We use `gh api` rather than `gh run view --log` because of https://github.com/cli/cli/issues/5011. - # Look for the table outputted by the benchmarks tool if there is a discrepancy - "$GH_BIN" api "/repos/${repo}/actions/jobs/${job}/logs" | grep -e "$benchmark_title_name_prefix" -e "$table_pipe" -} - -parse_check() { +parse_check_for_pr() { check_line=$1 # Something like: @@ -66,152 +54,102 @@ parse_check() { echo "$check_line" | sed -E 's/.*\(([^\)]+)\).*github\.com\/(.*)\/actions\/runs\/[0-9]+\/job\/([0-9]+)/\1 \2 \3/g' } -parse_benchmark_header() { - line=$1 +parse_workflow_url() { + workflow_url=$1 + # https://github.com/apple/swift-nio/actions/runs/15269806473 + echo "$workflow_url" | awk -F'/' '{print $4, $5, $8}' +} - echo "$line" | grep "$table_pipe" | grep "$table_current_run_heading" > /dev/null || fatal "Unexpected line format when expecting benchmark table header: $line" +fetch_checks_for_workflow() { + repo=$1 + run=$2 - # Something like: - # │ Malloc (total) (#, %) │ p90 threshold │ Current_run │ Difference % │ Threshold % │ - benchmark_header_name="$(echo "$line" | awk '{split($0,a,"│"); print a[2]}' | xargs)" - - case "$benchmark_header_name" in - "Malloc (total) (#, Δ)") - benchmark_metric_name="mallocCountTotal" - scale=1 - ;; - - "Malloc (total) (K, Δ)") - benchmark_metric_name="mallocCountTotal" - scale=1000 - ;; - - "Malloc / free Δ (#, Δ)") - benchmark_metric_name="memoryLeaked" - scale=1 - ;; - esac - echo "$benchmark_metric_name" "$scale" + "$GH_BIN" --repo "$repo" run view "$run" | grep Benchmarks | grep ID | grep -v Construct } -parse_benchmark_value() { - line=$1 - echo "$line" | grep "$table_pipe" | grep -v "$table_current_run_heading" > /dev/null || fatal "Unexpected line format when expecting benchmark table values: $line" - +parse_check_for_workflow() { + check_line=$1 + # Something like: - # │ p90 │ 8000 │ 6000 │ -2000 │ 0 │ - echo "$line" | awk '{split($0,a,"│"); print a[4]}' + # ✓ Benchmarks / Benchmarks / Linux (5.10) in 5m10s (ID 42942543009) + echo "$check_line" | sed -En 's/.*ID ([0-9][0-9]*).*/\1/p' } -parse_benchmark_title() { - line=$1 - echo "$line" | grep -q "$benchmark_title_name_prefix" || fatal "Unexpected line format when expecting threshold title: $line" - - # Something like: - # Deviations better than threshold for NIOCoreBenchmarks:WaitOnPromise - benchmark_title="$(echo "$line" | sed -E 's/.*than threshold for ([^:]*)\:(.*)$/\1.\2/g')" - threshold_file="${benchmark_title}.p90.json" +fetch_check_logs() { + repo=$1 + job=$2 - echo "$threshold_file" + # We use `gh api` rather than `gh run view --log` because of https://github.com/cli/cli/issues/5011. + # Look for the table outputted by the benchmarks tool if there is a discrepancy + "$GH_BIN" api "/repos/${repo}/actions/jobs/${job}/logs" } -# States for the output parsing state machine -readonly STATE_EXPECTING_BENCHMARK_TITLE=0 -readonly STATE_EXPECTING_BENCHMARK_HEADER_ROW=1 -readonly STATE_EXPECTING_BENCHMARK_VALUE_ROW=2 -readonly STATE_PROCESSED_VALUE_ROW=3 - -parse_benchmarks_output() { +apply_benchmarks_output_diff() { lines=$1 job=$2 - swift_version=$3 - - # We can ignore the percentage difference rows (with '#, %' in the title) - lines="$(echo "$lines" | sed -e '/#, %/,+1d')" - - state=$STATE_EXPECTING_BENCHMARK_TITLE - while read -r line; do - case "$state" in - "$STATE_EXPECTING_BENCHMARK_TITLE") - output_file="$(parse_benchmark_title "$line")" - output_path="${output_dir}/${swift_version}/${output_file}" - output="" - state=$STATE_EXPECTING_BENCHMARK_HEADER_ROW - ;; - - "$STATE_EXPECTING_BENCHMARK_HEADER_ROW") - read -r benchmark_metric_name scale <<< "$(parse_benchmark_header "$line")" - output="${output}\"${benchmark_metric_name}\":" - - state=$STATE_EXPECTING_BENCHMARK_VALUE_ROW - ;; - - "$STATE_EXPECTING_BENCHMARK_VALUE_ROW") - benchmark_metric_value=$(parse_benchmark_value "$line") - benchmark_metric_scaled_value="$(( benchmark_metric_value * scale ))" - - output="${output}${benchmark_metric_scaled_value}," - - state=$STATE_PROCESSED_VALUE_ROW - ;; - - "$STATE_PROCESSED_VALUE_ROW") - # next up is either another metric in the same benchmark or a new benchmark - if [[ "$line" =~ $table_current_run_heading ]]; then - # this is a BENCHMARK_HEADER_ROW - output="${output}\"$(parse_benchmark_header "$line")\":" - state=$STATE_EXPECTING_BENCHMARK_VALUE_ROW - else - # this is a new BENCHMARK_TITLE, finish-up the old benchmarK - write_output "$output" "$output_path" "$job" - - # reset state and output JSON buffer - output="" - state=$STATE_EXPECTING_BENCHMARK_HEADER_ROW - - # new benchmark means a new output file - output_file="$(parse_benchmark_title "$line")" - output_path="${output_dir}/${swift_version}/${output_file}" - fi - ;; - - *) - fatal "Unexpected state: $state" - ;; - esac - done <<< "$lines" - - write_output "$output" "$output_path" "$job" -} -write_output() { - output=$1 - output_path=$2 - job=$3 + # Trim out everything but the diff + git_diff="$(echo "$lines" | sed '1,/=== BEGIN DIFF ===/d' | sed '/Post job cleanup/,$d' | sed 's/^[0-9][0-9][0-9][0-9]-.*Z //')" + if [ -z "$git_diff" ]; then + log "No git diff found to apply for job $job" + return + fi + + log "Applying diff: + ${git_diff}" - log "Updating: $output_path job:$job" - # go via `yq` to clean up the trailing comma - echo "{$output}" | "$YQ_BIN" . | "$JQ_BIN" . > "$output_path" + echo "$git_diff" | git apply } #### -check_lines="$(fetch_checks_for_pr "$pr_url")" +if [ -n "$pr_url" ]; then + log "Fetching checks for $pr_url" + check_lines="$(fetch_checks_for_pr "$pr_url")" -if [ -z "$check_lines" ]; then - fatal "Could not locate benchmark checks on PR: $pr_url" -fi + if [ -z "$check_lines" ]; then + fatal "Could not locate benchmark checks on PR: $pr_url" + fi + + while read -r check_line; do + read -r swift_version repo job <<< "$(parse_check_for_pr "$check_line")" -while read -r check_line; do - read -r swift_version repo job <<< "$(parse_check "$check_line")" + lines=$(fetch_check_logs "$repo" "$job") - lines=$(fetch_check_logs "$repo" "$job") + if [ -z "$lines" ]; then + log "Nothing to update: $repo $swift_version job: $job" + continue + fi - if [ -z "$lines" ]; then - log "Nothing to update: $repo $swift_version job:$job" - continue + parse_benchmarks_output "$lines" "$job" "$swift_version" + + done <<< "$check_lines" + +elif [ -n "$run_url" ]; then + read -r repo_org repo_name run <<< "$(parse_workflow_url "$run_url")" + repo="$repo_org/$repo_name" + + log "Fetching checks for $repo run $run" + check_lines="$(fetch_checks_for_workflow "$repo" "$run")" + + if [ -z "$check_lines" ]; then + fatal "Could not locate benchmark checks on workflow run: $run_url" fi - parse_benchmarks_output "$lines" "$job" "$swift_version" + while read -r check_line; do + job="$(parse_check_for_workflow "$check_line")" + + lines=$(fetch_check_logs "$repo" "$job") -done <<< "$check_lines" + if [ -z "$lines" ]; then + log "Nothing to update: $repo $swift_version job: $job" + continue + fi + + apply_benchmarks_output_diff "$lines" "$job" + + done <<< "$check_lines" + +else + fatal "Either pull request or workflow run URL must be specified." +fi diff --git a/scripts/check_benchmark_thresholds.sh b/scripts/check_benchmark_thresholds.sh new file mode 100755 index 00000000000..5fb57360598 --- /dev/null +++ b/scripts/check_benchmark_thresholds.sh @@ -0,0 +1,46 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2025 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -uo pipefail + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +# Parameter environment variables +if [ -z "$SWIFT_VERSION" ]; then + fatal "SWIFT_VERSION must be specified." +fi + +benchmark_package_path="${BENCHMARK_PACKAGE_PATH:-"."}" +swift_version="${SWIFT_VERSION:-""}" + +# Any parameters to the script are passed along to SwiftPM +swift_package_arguments=("$@") + +#"swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/" +swift package --package-path "$benchmark_package_path" "${swift_package_arguments[@]}" benchmark thresholds check --format metricP90AbsoluteThresholds --path "${benchmark_package_path}/Thresholds/${swift_version}/" +rc="$?" + +# Benchmarks are unchanged, nothing to recalculate +if [[ "$rc" == 0 ]]; then + exit 0 +fi + +log "Recalculating thresholds..." + +swift package --package-path "$benchmark_package_path" "${swift_package_arguments[@]}" benchmark thresholds update --format metricP90AbsoluteThresholds --path "${benchmark_package_path}/Thresholds/${swift_version}/" +echo "=== BEGIN DIFF ===" # use echo, not log for clean output to be scraped +git diff --exit-code HEAD From ceada75fc07eefd5ad472780d22183bc05c105c8 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Wed, 28 May 2025 14:59:35 +0100 Subject: [PATCH 2/3] update thresholds based on diff in logs --- .github/workflows/benchmarks.yml | 1 - dev/thresholds-from-benchmark-output.sh | 159 +++++++++++++++--------- 2 files changed, 99 insertions(+), 61 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0d79a3d6d72..b47fa437d90 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -74,7 +74,6 @@ jobs: - id: generate-matrix run: echo "benchmarks-matrix=$(curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT" env: - MATRIX_LINUX_COMMAND_: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/" MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check_benchmark_thresholds.sh | BENCHMARK_PACKAGE_PATH=${{ inputs.benchmark_package_path }} bash" MATRIX_LINUX_SETUP_COMMAND: "swift --version && apt-get update -y -q && apt-get install -y -q curl libjemalloc-dev && git config --global --add safe.directory /$(basename ${{ github.workspace }})" MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }} diff --git a/dev/thresholds-from-benchmark-output.sh b/dev/thresholds-from-benchmark-output.sh index c14642c9562..c9d795714e0 100755 --- a/dev/thresholds-from-benchmark-output.sh +++ b/dev/thresholds-from-benchmark-output.sh @@ -13,7 +13,7 @@ ## ##===----------------------------------------------------------------------===## -# This script allows you to consume swift package benchmark output and +# This script allows you to consume swift package benchmark output and # update JSON threshold files set -uo pipefail @@ -22,17 +22,10 @@ log() { printf -- "** %s\n" "$*" >&2; } error() { printf -- "** ERROR: %s\n" "$*" >&2; } fatal() { error "$@"; exit 1; } -pr_url="${PR_URL:=""}" -run_url="${RUN_URL:=""}" +url="${URL:=""}" -output_dir="${OUTPUT_DIRECTORY:=""}" - -if [ -z "$pr_url" ] && [ -z "$run_url" ]; then - fatal "Pull request URL or workflow run URL must be specified." -fi - -if [ -n "$pr_url" ] && [ -z "$output_dir" ]; then - fatal "Output directory must be specified." +if [ -z "$url" ]; then + fatal "Pull request or workflow run URL must be specified." fi # Check for required tools @@ -46,18 +39,60 @@ fetch_checks_for_pr() { "$GH_BIN" pr checks "$pr_url" | grep Benchmarks | grep -v Construct } +parse_url() { + workflow_url=$1 + # https://github.com/apple/swift-nio/actions/runs/15269806473 + # https://github.com/apple/swift-nio/pull/3257 + if echo "$url" | grep -q "pull"; then + type="PR" + elif echo "$url" | grep -q "actions/runs"; then + type="run" + else + fatal "Cannot parse URL: $url" + fi + echo "$url" | awk -v type="$type" -F '/' '{print $4, $5, type}' +} + +parse_check() { + type=$1 + check_line=$2 + + case "$type" in + "PR") + parse_check_for_pr "$check_line" + ;; + + "run") + parse_check_for_workflow "$check_line" + ;; + + *) + fatal "Unknown type '$type'" + # Add error handling commands here + ;; + esac +} + +parse_check_for_workflow() { + check_line=$1 + + # Something like: + # ✓ Benchmarks / Benchmarks / Linux (5.10) in 5m10s (ID 42942543009) + echo "$check_line" | sed -En 's/.*ID ([0-9][0-9]*).*/\1/p' +} + parse_check_for_pr() { check_line=$1 # Something like: # Benchmarks / Benchmarks / Linux (5.10) pass 4m21s https://github.com/apple/swift-nio-ssl/actions/runs/13793783082/job/38580234681 - echo "$check_line" | sed -E 's/.*\(([^\)]+)\).*github\.com\/(.*)\/actions\/runs\/[0-9]+\/job\/([0-9]+)/\1 \2 \3/g' + echo "$check_line" | sed -E 's/.*\(([^\)]+)\).*github\.com\/(.*)\/actions\/runs\/[0-9]+\/job\/([0-9]+)/\3/g' } parse_workflow_url() { workflow_url=$1 # https://github.com/apple/swift-nio/actions/runs/15269806473 - echo "$workflow_url" | awk -F'/' '{print $4, $5, $8}' + echo "$workflow_url" | awk -F '/' '{print $8}' } fetch_checks_for_workflow() { @@ -67,89 +102,93 @@ fetch_checks_for_workflow() { "$GH_BIN" --repo "$repo" run view "$run" | grep Benchmarks | grep ID | grep -v Construct } -parse_check_for_workflow() { - check_line=$1 - - # Something like: - # ✓ Benchmarks / Benchmarks / Linux (5.10) in 5m10s (ID 42942543009) - echo "$check_line" | sed -En 's/.*ID ([0-9][0-9]*).*/\1/p' -} - fetch_check_logs() { repo=$1 job=$2 + log "Pulling logs for $repo job $job" # We use `gh api` rather than `gh run view --log` because of https://github.com/cli/cli/issues/5011. # Look for the table outputted by the benchmarks tool if there is a discrepancy "$GH_BIN" api "/repos/${repo}/actions/jobs/${job}/logs" } -apply_benchmarks_output_diff() { +scrape_benchmarks_output_diff() { lines=$1 job=$2 + log "Scraping diff from log" # Trim out everything but the diff git_diff="$(echo "$lines" | sed '1,/=== BEGIN DIFF ===/d' | sed '/Post job cleanup/,$d' | sed 's/^[0-9][0-9][0-9][0-9]-.*Z //')" + + echo "$git_diff" +} + +apply_benchmarks_output_diff() { + git_diff=$1 + if [ -z "$git_diff" ]; then - log "No git diff found to apply for job $job" + log "No git diff found to apply" return fi - log "Applying diff: - ${git_diff}" - + log "Applying git diff" echo "$git_diff" | git apply } -#### - -if [ -n "$pr_url" ]; then - log "Fetching checks for $pr_url" - check_lines="$(fetch_checks_for_pr "$pr_url")" - - if [ -z "$check_lines" ]; then - fatal "Could not locate benchmark checks on PR: $pr_url" - fi +filter_thresholds_to_allowlist() { + git_diff=$1 - while read -r check_line; do - read -r swift_version repo job <<< "$(parse_check_for_pr "$check_line")" + log "Filtering thresholds" + for thresholds_file in $(echo "$git_diff" | grep "+++" | sed -E 's/^\+\+\+ b\/(.*)$/\1/g'); do + jq 'with_entries(select(.key | in({mallocCountTotal:1, memoryLeaked:1})))' "$thresholds_file" > temp.json && mv -f temp.json "$thresholds_file" + done +} - lines=$(fetch_check_logs "$repo" "$job") +#### - if [ -z "$lines" ]; then - log "Nothing to update: $repo $swift_version job: $job" - continue - fi +read -r repo_org repo_name type <<< "$(parse_url "$url")" +repo="$repo_org/$repo_name" +log "URL is of type $type in $repo" - parse_benchmarks_output "$lines" "$job" "$swift_version" +case "$type" in +"PR") + log "Fetching checks for $url" + check_lines="$(fetch_checks_for_pr "$url")" - done <<< "$check_lines" + if [ -z "$check_lines" ]; then + fatal "Could not locate benchmark checks on PR: $url" + fi + ;; -elif [ -n "$run_url" ]; then - read -r repo_org repo_name run <<< "$(parse_workflow_url "$run_url")" - repo="$repo_org/$repo_name" +"run") + run="$(parse_workflow_url "$url")" log "Fetching checks for $repo run $run" check_lines="$(fetch_checks_for_workflow "$repo" "$run")" if [ -z "$check_lines" ]; then - fatal "Could not locate benchmark checks on workflow run: $run_url" + fatal "Could not locate benchmark checks on workflow run: $url" fi + ;; - while read -r check_line; do - job="$(parse_check_for_workflow "$check_line")" +*) + fatal "Unknown type '$type'" + ;; +esac - lines=$(fetch_check_logs "$repo" "$job") +while read -r check_line; do + job="$(parse_check "$type" "$check_line")" - if [ -z "$lines" ]; then - log "Nothing to update: $repo $swift_version job: $job" - continue - fi + lines=$(fetch_check_logs "$repo" "$job") - apply_benchmarks_output_diff "$lines" "$job" + if [ -z "$lines" ]; then + log "Nothing to update: $repo job: $job" + continue + fi - done <<< "$check_lines" + git_diff="$(scrape_benchmarks_output_diff "$lines" "$job")" + apply_benchmarks_output_diff "$git_diff" -else - fatal "Either pull request or workflow run URL must be specified." -fi + filter_thresholds_to_allowlist "$git_diff" + +done <<< "$check_lines" From 12ad2017e4a88ba41acefa41a6d33087032b1e5f Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 29 May 2025 15:04:59 +0100 Subject: [PATCH 3/3] Update dev/thresholds-from-benchmark-output.sh Co-authored-by: George Barnett --- dev/thresholds-from-benchmark-output.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/thresholds-from-benchmark-output.sh b/dev/thresholds-from-benchmark-output.sh index c9d795714e0..729358f1017 100755 --- a/dev/thresholds-from-benchmark-output.sh +++ b/dev/thresholds-from-benchmark-output.sh @@ -43,9 +43,9 @@ parse_url() { workflow_url=$1 # https://github.com/apple/swift-nio/actions/runs/15269806473 # https://github.com/apple/swift-nio/pull/3257 - if echo "$url" | grep -q "pull"; then + if [[ "$url" =~ pull ]]; then type="PR" - elif echo "$url" | grep -q "actions/runs"; then + elif [[ "$url" =~ actions/runs ]]; then type="run" else fatal "Cannot parse URL: $url"