intro bench all

README.md

@@ -4,11 +4,12 @@
 For now just copy existing command and modify. Later there will be more ways to do this easier from the bot itself or here via CLI. 
 
 - Run `yarn --immutable` to install `command-bot` dependency, which includes the actual supported schema for commands validation.
+  If you develop in parallel with `command-bot` then use something like `yarn link`
 - Copy any existing command which looks the most similar.
 - Change the name of command. The structure should be `/commands/<command_name>/<command_name>.cmd.json` and `/commands/<command_name>/<command_name>.sh`, because the `command_name` will be used to build a path from command to script.
 - Test it in your PR:
   - when running a new command - add specific flag (-v PIPELINE_SCRIPTS_REF=your-branch) to test it before merge.  
-  Example: `/cmd queue -v PIPELINE_SCRIPTS_REF=your-branch -c new-command $ some arguments`
+  Example: `bot new-command -v PIPELINE_SCRIPTS_REF=your-branch $ some arguments`
 - In PR with your new command - add test evidence links to your PR with the result of your command and merge after approval.
 - That's it
 
@@ -17,4 +18,4 @@ For now just copy existing command and modify. Later there will be more ways to
 To make a context of companion you need to specify it with special env variable `-v PATCH_<repo_name>=<repo_PR_number>`, for example: `-v PATCH_substrate=11649`
 
 Example:
-`/cmd queue -v PATCH_substrate=11649 -c try-runtime $ polkadot`
+`bot try-runtime -v PATCH_substrate=11649 $ polkadot`

commands/bench-all/bench-all.cmd.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "../../node_modules/command-bot/src/schema/schema.cmd.json",
+  "command": {
+    "description": "This is a wrapper to run `bench` for all pallets within BM4,5,6 runners",
+    "configuration": {
+      "gitlab": {
+        "job": {
+          "tags": [
+            "weights"
+          ],
+          "variables": {}
+        }
+      }
+    },
+    "presets": {
+      "substrate-all": {
+        "description": "Pallet + Overhead + Machine Benchmark for Substrate for all pallets",
+        "repos": ["substrate"]
+      },
+      "polkadot-all": {
+        "description": "Pallet + Overhead Benchmark for Polkadot",
+        "repos": ["polkadot"],
+        "args": {
+          "runtime":    { "label": "Runtime", "type_one_of": ["kusama", "polkadot", "rococo", "westend"] }
+        }
+      },
+      "cumulus-all": {
+        "description": "Pallet Benchmark for Cumulus",
+        "repos": ["cumulus"]
+      }
+    }
+  }
+}

commands/bench-all/bench-all.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+shopt -s inherit_errexit
+
+# this is a fallback for `bench` command running on BM4,5,6 machines
+. "$(dirname "${BASH_SOURCE[0]}")/../bench/bench.sh" all "$@"

commands/bench-vm/bench-vm.cmd.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "../../node_modules/command-bot/src/schema/schema.cmd.json",
+  "command": {
+    "description": "This is a testing for `bench` command running on VM machine",
+    "configuration": {
+      "gitlab": {
+        "job": {
+          "tags": [
+            "weights-vm"
+          ],
+          "variables": {}
+        }
+      }
+    }
+  }
+}

commands/bench-vm/bench-vm.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+shopt -s inherit_errexit
+
+# this is a fallback for `bench` command running on BM3 machine
+. "$(dirname "${BASH_SOURCE[0]}")/../bench/bench.sh" "$@"

commands/bench/bench.cmd.json

@@ -5,102 +5,113 @@
     "configuration": {
       "gitlab": {
         "job": {
-          "tags": ["weights-vm"],
+          "tags": ["bench-bot"],
           "variables": {}
         }
       }
     },
     "presets": {
-      "substrate": {
-        "description": "Pallet Benchmark for Substrate",
+      "substrate-pallet": {
+        "description": "Pallet Benchmark for Substrate for specific pallet",
         "repos": ["substrate"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": [ "pallet" ] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["dev"] },
-          "pallet":   { "label": "pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": [ "pallet" ] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["dev"] },
+          "pallet":     { "label": "pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
-      "polkadot": {
-        "description": "Pallet Benchmark for Polkadot",
+      "polkadot-pallet": {
+        "description": "Pallet Benchmark for Polkadot for specific pallet",
         "repos": ["polkadot"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["runtime", "xcm"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["dev", "kusama-dev", "polkadot-dev", "rococo-dev", "westend-dev"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["runtime", "xcm"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["dev", "kusama", "polkadot", "rococo", "westend"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
       "cumulus-assets": {
         "description": "Pallet Benchmark for Cumulus [assets]",
         "repos": ["cumulus"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["xcm", "pallet"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["statemine", "statemint", "test-utils", "westmint"] },
-          "kind":     { "label": "Kind", "type_one_of": ["asset"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["statemine", "statemint", "test-utils", "westmint"] },
+          "kind":       { "label": "Kind", "type_one_of": ["asset"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
       "cumulus-collectives": {
         "description": "Pallet Benchmark for Cumulus [collectives]",
         "repos": ["cumulus"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["xcm", "pallet"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["collectives-polkadot"] },
-          "kind":     { "label": "Kind", "type_one_of": ["collectives"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["collectives-polkadot"] },
+          "kind":       { "label": "Kind", "type_one_of": ["collectives"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
+        }
+      },
+      "cumulus-bridge-hubs": {
+        "description": "Pallet Benchmark for Cumulus [bridge-hubs]",
+        "repos": ["cumulus"],
+        "args": {
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["bridge-hub-polkadot", "bridge-hub-kusama", "bridge-hub-rococo"] },
+          "kind":       { "label": "Kind", "type_one_of": ["bridge-hubs"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
       "cumulus-contracts": {
         "description": "Pallet Benchmark for Cumulus [contracts]",
         "repos": ["cumulus"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["xcm", "pallet"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["contracts-rococo"] },
-          "kind":     { "label": "Kind", "type_one_of": ["contracts"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["contracts-rococo"] },
+          "kind":       { "label": "Kind", "type_one_of": ["contracts"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
       "cumulus-starters": {
         "description": "Pallet Benchmark for Cumulus [starters]",
         "repos": ["cumulus"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["xcm", "pallet"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["seedling", "shell"] },
-          "kind":     { "label": "Kind", "type_one_of": ["starters"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["seedling", "shell"] },
+          "kind":       { "label": "Kind", "type_one_of": ["starters"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name" }
         }
       },
       "cumulus-testing": {
         "description": "Pallet Benchmark for Cumulus [testing]",
         "repos": ["cumulus"],
         "args": {
-          "type":     { "label": "Type of bench", "type_one_of": ["xcm", "pallet"] },
-          "runtime":  { "label": "Runtime", "type_one_of": ["penpal", "rococo-parachain"] },
-          "kind":     { "label": "Kind", "type_one_of": ["testing"] },
-          "pallet":   { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/" }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["xcm", "pallet"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["penpal", "rococo-parachain"] },
+          "kind":       { "label": "Kind", "type_one_of": ["testing"] },
+          "pallet":     { "label": "Pallet", "type_rule": "/^([a-z_]+)([:]{2}[a-z_]+)?$/", "example": "pallet_name"
+          }
         }
       },
       "substrate-overhead": {
         "description": "Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files",
         "repos": ["substrate"],
         "args": {
-          "type": { "label": "Type of bench", "type_one_of": ["overhead"] }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["overhead"] }
         }
       },
       "polkadot-overhead": {
         "description": "Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files",
         "repos": ["polkadot"],
         "args": {
-          "type":   { "label": "Type of bench", "type_one_of": ["overhead"] },
-          "runtime":{ "label": "Runtime", "type_one_of": ["kusama-dev", "polkadot-dev", "rococo-dev", "westend-dev"] }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["overhead"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["kusama-dev", "polkadot-dev", "rococo-dev", "westend-dev"] }
         }
       },
       "cumulus-overhead": {
         "description": "Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files",
         "repos": ["cumulus"],
         "args": {
-          "type":   { "label": "Type of bench", "type_one_of": ["overhead"] },
-          "kind":   { "label": "Kind", "type_one_of": ["asset"] },
-          "runtime":{ "label": "Runtime", "type_one_of": ["statemine", "statemint", "test-utils", "westmint"] }
+          "subcommand": { "label": "Subcommand", "type_one_of": ["overhead"] },
+          "kind":       { "label": "Kind", "type_one_of": ["asset"] },
+          "runtime":    { "label": "Runtime", "type_one_of": ["statemine", "statemint", "test-utils", "westmint"] }
         }
       }
     }

commands/bench/bench.sh

@@ -11,7 +11,31 @@
 set -eu -o pipefail
 shopt -s inherit_errexit
 
-. "$(dirname "${BASH_SOURCE[0]}")/../cmd_runner.sh"
+# realpath allows to reuse the current
+BENCH_ROOT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+
+. "$BENCH_ROOT_DIR/../utils.sh"
+. "$BENCH_ROOT_DIR/../cmd_runner.sh"
+
+cargo_run_benchmarks="cargo run --quiet --profile=production"
+current_folder="$(basename "$PWD")"
+
+get_arg optional --repo "$@"
+repository="${out:=$current_folder}"
+
+echo "Repo: $repository"
+
+cargo_run() {
+  echo "Running $cargo_run_benchmarks" "${args[@]}"
+
+  # if not patched with PATCH_something=123 then use --locked
+  if [[ -z "${BENCH_PATCHED:-}" ]]; then
+    cargo_run_benchmarks+=" --locked"
+  fi
+
+  $cargo_run_benchmarks "${args[@]}"
+}
+
 
 main() {
   cmd_runner_setup
@@ -49,8 +73,28 @@ main() {
   cmd_runner_apply_patches
 
   set -x
-  # Runs the command to generate the weights
-  . "$(dirname "${BASH_SOURCE[0]}")/build-bench-args.sh" "$@"
+
+  local subcommand="$1"
+  shift
+
+  case "$subcommand" in
+    runtime|pallet|xcm)
+      echo 'Running bench_pallet'
+      . "$BENCH_ROOT_DIR/lib/bench-pallet.sh" "$subcommand" "$@"
+    ;;
+    overhead)
+      echo 'Running bench_overhead'
+      . "$BENCH_ROOT_DIR/lib/bench-overhead.sh" "$subcommand" "$@"
+    ;;
+    all)
+      echo "Running all-$repository"
+      . "$BENCH_ROOT_DIR/lib/bench-all-${repository}.sh" "$@"
+    ;;
+    *)
+      die "Invalid subcommand $subcommand to process_args"
+    ;;
+  esac
+
   set +x
 
   # in case we used diener to patch some dependency during benchmark execution,