chore: dump dmesg/net/cpu/mem usage at end of ci run

bootstrap.sh

@@ -156,6 +156,7 @@ export -f start_txes
 
 function test {
   echo_header "test all"
+  export NOIR_HASH=$(./noir/bootstrap.sh hash)
 
   start_txes
 

ci3/bootstrap_ec2

@@ -98,20 +98,21 @@ if [ -t 1 ]; then
   ssh_args="-t"
   docker_args+=" -t"
   if [ "$NO_TERMINATE" -eq 0 ]; then
-    cmd+=" || exec zsh"
+    run_cmd="run || exec zsh"
   else
-    cmd+="; exec zsh"
+    run_cmd="run; exec zsh"
   fi
 else
   # LOG_ID can optionally be set externally to be e.g. the GA run id.
-  cmd="ci3/aws_handle_evict '$cmd' 2>&1 | ci3/add_timestamps | DUP=1 ci3/cache_log 'CI run' \$ci_log_id"
+  run_cmd="ci3/aws_handle_evict run 2>&1 | ci3/add_timestamps | DUP=1 ci3/cache_log 'CI run' \$ci_log_id"
 fi
 
 container_script=$(cat <<EOF
   set -euo pipefail
   # When restarting the container, just hang around.
-  while [ -f started ]; do sleep 999; done
-  touch started
+  # Note we use the "ci-started" file to determine if we're running on a CI machine in some cases (e.g. npm cache).
+  while [ -f ci-started ]; do sleep 999; done
+  touch ci-started
   sudo chown aztec-dev:aztec-dev aztec-packages
   # Set up preferred commit attribution (used during releases).
   git config --global user.email "tech@aztecprotocol.com"
@@ -135,10 +136,23 @@ container_script=$(cat <<EOF
   if semver check "\$REF_NAME"; then
     echo "Performing a release because \$REF_NAME is a semver."
   fi
-  set +e
-  set -x
-  $cmd
+
+  function run {
+    set +e
+    set -x
+    $cmd
+    local code=\${PIPESTATUS[0]}
+    sudo dmesg 2>&1 | ci3/cache_log 'dmesg'
+    sudo cat /tmp/netfile | ci3/cache_log 'netfile'
+    sudo cat /tmp/cpufile | ci3/cache_log 'cpufile'
+    sudo cat /tmp/memfile | ci3/cache_log 'memfile'
+    exit \$code
+  }
+  export -f run
+
+  $run_cmd
   code=\${PIPESTATUS[0]}
+
   case \$code in
     155) ;;
     0) log_ci_run PASSED \$ci_log_id ;;
@@ -173,6 +187,19 @@ ssh ${ssh_args:-} -F $ci3/aws/build_instance_ssh_config ubuntu@$ip "
   sudo cp -r \$HOME/.bb-crs/* /mnt/bb-crs
   echo Done in \$SECONDS seconds.
 
+  echo Installing and starting sysdig...
+  sudo apt-get update &>/dev/null
+  sudo apt-get install -y sysdig &>/dev/null
+  sudo sysdig -p '%evt.time %proc.cmdline %evt.dir %evt.type %evt.args' 'evt.type=bind or evt.type=listen' > /tmp/netfile &
+  netfile_pid=\$!
+  # Capture cpu load.
+  mpstat 2 &> /tmp/cpufile &
+  cpufile_pid=\$!
+  # Capture mem load.
+  vmstat -w -S M 2 &> /tmp/memfile &
+  memfile_pid=\$!
+  trap 'sudo kill \$netfile_pid \$cpufile_pid \$memfile_pid' EXIT
+
   echo Starting devbox...
   docker run --privileged ${docker_args:-} \
     --name aztec_build \
@@ -182,6 +209,7 @@ ssh ${ssh_args:-} -F $ci3/aws/build_instance_ssh_config ubuntu@$ip "
     -v \$HOME/.aws:/home/aztec-dev/.aws:ro \
     -v /mnt/bb-crs:/home/aztec-dev/.bb-crs:ro \
     -v /tmp:/tmp \
+    -v /dev/kmsg:/dev/kmsg \
     -e USE_TEST_CACHE=1 \
     -e CI_REDIS='ci-redis.lzka0i.0001.use2.cache.amazonaws.com' \
     -e SSH_CONNECTION=' ' \

ci3/npm_install_deps

@@ -17,7 +17,9 @@ fi
 yarn_lock_path=$(realpath yarn.lock --relative-to=$REPO_PATH)
 package_json_path=$(realpath package.json --relative-to=$REPO_PATH)
 
-if [ "$CI" -eq 1 ]; then
+# We only use the node module cache in CI.
+# The checking for ci-started makes sure we really *are* on a CI machine (it's created in bootstrap_ec2).
+if [ "$CI" -eq 1 ] && [ -f $HOME/ci-started ]; then
   nm_hash=$(cache_content_hash "^$yarn_lock_path" "^$package_json_path")
   if ! cache_download node-modules-$nm_hash.zst; then
     denoise "retry 'yarn install --immutable'"

release-image/bootstrap.sh

@@ -5,6 +5,17 @@ cmd=${1:-}
 
 hash=$(cache_content_hash ^release-image/Dockerfile ^build-images/src/Dockerfile ^yarn-project/yarn.lock)
 
+function build_image {
+  set -euo pipefail
+  cd ..
+  docker build -f release-image/Dockerfile -t aztecprotocol/aztec:$(git rev-parse HEAD) .
+  docker tag aztecprotocol/aztec:$(git rev-parse HEAD) aztecprotocol/aztec:latest
+
+  # Remove all but the most recent image.
+  docker images aztecprotocol/aztec --format "{{.ID}}" | uniq | tail -n +2 | xargs -r docker rmi -f
+}
+export -f build_image
+
 function build {
   echo_header "release-image build"
 
@@ -16,8 +27,7 @@ function build {
     docker load < release-image-base
   fi
 
-  denoise "cd .. && docker build -f release-image/Dockerfile -t aztecprotocol/aztec:$(git rev-parse HEAD) ."
-  docker tag aztecprotocol/aztec:$(git rev-parse HEAD) aztecprotocol/aztec:latest
+  denoise "build_image"
 }
 
 case "$cmd" in