From ac5aa37758459ad026b07a0ea97b0a10a2eef34a Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:05:53 +0000 Subject: [PATCH 1/7] chore: calculate available memory for noir-projects/bootstrap.sh memsuspend --- noir-projects/noir-contracts/bootstrap.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 57a6900e2c53..2f6af1ea557d 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -38,8 +38,23 @@ function on_exit() { trap on_exit EXIT mkdir -p $tmp_dir +function get_hardware_memory { + local os=$(uname -s) + local total_mem_gb=64 # Default to 64GB + + if [[ "$os" == "Darwin" ]]; then + total_mem_bytes=$(sysctl -n hw.memsize) + total_mem_gb=$((total_mem_bytes / 1024 / 1024 / 1024)) + elif [[ "$os" == "Linux" ]]; then + total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') + fi + + echo $(( total_mem_gb < 64 ? total_mem_gb : 64 )) +} + +hardware_memory=$(get_hardware_memory) # Set common flags for parallel. -export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend ${MEMSUSPEND:-64G}" +export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend ${MEMSUSPEND:-${hardware_memory}G}" # This computes a vk and adds it to the input function json if it's private, else returns same input. # stdin has the function json. From b78361deb0969d708ce9bc852cf0c353284c0cc1 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:11:11 +0000 Subject: [PATCH 2/7] fix: move to memsuspend_limit file --- ci3/memsuspend_limit | 20 ++++++++++++++++++++ noir-projects/noir-contracts/bootstrap.sh | 17 +---------------- 2 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 ci3/memsuspend_limit diff --git a/ci3/memsuspend_limit b/ci3/memsuspend_limit new file mode 100644 index 000000000000..0215efaba5cb --- /dev/null +++ b/ci3/memsuspend_limit @@ -0,0 +1,20 @@ +#!/bin/bash + +# Get the hardware memory in GB, useful for setting the memsuspend limit. +function get_hardware_memory { + local os=$(uname -s) + local total_mem_gb=64 # Default to 64GB + + if [[ "$os" == "Darwin" ]]; then + total_mem_bytes=$(sysctl -n hw.memsize) + total_mem_gb=$((total_mem_bytes / 1024 / 1024 / 1024)) + elif [[ "$os" == "Linux" ]]; then + total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') + fi + + echo $(( total_mem_gb < 64 ? total_mem_gb : 64 )) +} + +hardware_memory=$(get_hardware_memory) + +export MEMSUSPEND=${MEMSUSPEND:-${hardware_memory}G} diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 2f6af1ea557d..57a6900e2c53 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -38,23 +38,8 @@ function on_exit() { trap on_exit EXIT mkdir -p $tmp_dir -function get_hardware_memory { - local os=$(uname -s) - local total_mem_gb=64 # Default to 64GB - - if [[ "$os" == "Darwin" ]]; then - total_mem_bytes=$(sysctl -n hw.memsize) - total_mem_gb=$((total_mem_bytes / 1024 / 1024 / 1024)) - elif [[ "$os" == "Linux" ]]; then - total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') - fi - - echo $(( total_mem_gb < 64 ? total_mem_gb : 64 )) -} - -hardware_memory=$(get_hardware_memory) # Set common flags for parallel. -export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend ${MEMSUSPEND:-${hardware_memory}G}" +export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend ${MEMSUSPEND:-64G}" # This computes a vk and adds it to the input function json if it's private, else returns same input. # stdin has the function json. From 565eeea5b7934a804d0746f43d20557e31ea92cd Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:19:27 +0000 Subject: [PATCH 3/7] Update --- ci3/memsuspend_limit | 8 ++++++++ 1 file changed, 8 insertions(+) mode change 100644 => 100755 ci3/memsuspend_limit diff --git a/ci3/memsuspend_limit b/ci3/memsuspend_limit old mode 100644 new mode 100755 index 0215efaba5cb..872b253d2b13 --- a/ci3/memsuspend_limit +++ b/ci3/memsuspend_limit @@ -1,6 +1,10 @@ #!/bin/bash +set -x # Get the hardware memory in GB, useful for setting the memsuspend limit. +# General rul of thumb: +# - Allocate 1/4 of total hardware memory for the build. +# - Max out at 64GB function get_hardware_memory { local os=$(uname -s) local total_mem_gb=64 # Default to 64GB @@ -12,6 +16,10 @@ function get_hardware_memory { total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') fi + # Allocate 1/4 of total hardware memory for the build. + total_mem_gb=$((total_mem_gb / 4)) + + # Max out at 64GB echo $(( total_mem_gb < 64 ? total_mem_gb : 64 )) } From 4a5647eee33da55676e72ddddb1a0b151a3de6de Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:23:15 +0000 Subject: [PATCH 4/7] fix: remove set -x --- ci3/memsuspend_limit | 1 - 1 file changed, 1 deletion(-) diff --git a/ci3/memsuspend_limit b/ci3/memsuspend_limit index 872b253d2b13..240f4d0a8f95 100755 --- a/ci3/memsuspend_limit +++ b/ci3/memsuspend_limit @@ -1,5 +1,4 @@ #!/bin/bash -set -x # Get the hardware memory in GB, useful for setting the memsuspend limit. # General rul of thumb: From 3d6f86bfed1adc00baf1d445ec3d689eecd0b0b4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 3 Mar 2025 17:06:02 +0000 Subject: [PATCH 5/7] call throughout --- ci3/memsuspend_limit | 35 +++++++++---------- ci3/parallelise | 2 +- noir-projects/noir-contracts/bootstrap.sh | 2 +- .../noir-protocol-circuits/bootstrap.sh | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/ci3/memsuspend_limit b/ci3/memsuspend_limit index 240f4d0a8f95..30350e53d709 100755 --- a/ci3/memsuspend_limit +++ b/ci3/memsuspend_limit @@ -1,27 +1,26 @@ #!/bin/bash +set -eu # Get the hardware memory in GB, useful for setting the memsuspend limit. # General rul of thumb: # - Allocate 1/4 of total hardware memory for the build. # - Max out at 64GB -function get_hardware_memory { - local os=$(uname -s) - local total_mem_gb=64 # Default to 64GB +if [ -n "${MEMSUSPEND:-}" ]; then + echo $MEMSUSPEND + exit +fi +os=$(uname -s) +total_mem_gb=64 # Default to 64GB - if [[ "$os" == "Darwin" ]]; then - total_mem_bytes=$(sysctl -n hw.memsize) - total_mem_gb=$((total_mem_bytes / 1024 / 1024 / 1024)) - elif [[ "$os" == "Linux" ]]; then - total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') - fi +if [[ "$os" == "Darwin" ]]; then + total_mem_bytes=$(sysctl -n hw.memsize) + total_mem_gb=$((total_mem_bytes / 1024 / 1024 / 1024)) +elif [[ "$os" == "Linux" ]]; then + total_mem_gb=$(free -g | awk '/^Mem:/ {print $2}') +fi - # Allocate 1/4 of total hardware memory for the build. - total_mem_gb=$((total_mem_gb / 4)) +# Allocate 1/4 of total hardware memory for the build. +total_mem_gb=$((total_mem_gb / 4)) - # Max out at 64GB - echo $(( total_mem_gb < 64 ? total_mem_gb : 64 )) -} - -hardware_memory=$(get_hardware_memory) - -export MEMSUSPEND=${MEMSUSPEND:-${hardware_memory}G} +# Max out at 64GB +echo $(( total_mem_gb < 64 ? total_mem_gb : 64 ))G diff --git a/ci3/parallelise b/ci3/parallelise index 5fde397e88c9..3d6299640a50 100755 --- a/ci3/parallelise +++ b/ci3/parallelise @@ -5,7 +5,7 @@ NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source cd $root jobs=$(get_num_cpus_max ${1:-}) -parallel_args="-j$jobs --memsuspend ${MEMSUSPEND:-64G} --line-buffer --joblog joblog.txt" +parallel_args="-j$jobs --memsuspend $(memsupend_limit) --line-buffer --joblog joblog.txt" # If not in CI, fail fast. if [ "$CI" -eq 0 ]; then diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 57a6900e2c53..790043591cdf 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -39,7 +39,7 @@ trap on_exit EXIT mkdir -p $tmp_dir # Set common flags for parallel. -export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend ${MEMSUSPEND:-64G}" +export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend $(memsuspend_limit)" # This computes a vk and adds it to the input function json if it's private, else returns same input. # stdin has the function json. diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 668dcde7b7d3..7bc5f830fae1 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -157,7 +157,7 @@ function build { echo "$(basename $dir)" fi done | \ - parallel -v --line-buffer --tag --halt now,fail=1 --memsuspend ${MEMSUSPEND:-64G} \ + parallel -v --line-buffer --tag --halt now,fail=1 --memsuspend $(memsuspend_limit) \ --joblog joblog.txt compile {} code=$? cat joblog.txt From 87052debb10d9f3126d26480fa6035a148e5a9d7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 3 Mar 2025 12:06:44 -0500 Subject: [PATCH 6/7] Update memsuspend_limit --- ci3/memsuspend_limit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/memsuspend_limit b/ci3/memsuspend_limit index 30350e53d709..12e16e70c55f 100755 --- a/ci3/memsuspend_limit +++ b/ci3/memsuspend_limit @@ -2,7 +2,7 @@ set -eu # Get the hardware memory in GB, useful for setting the memsuspend limit. -# General rul of thumb: +# General rule of thumb: # - Allocate 1/4 of total hardware memory for the build. # - Max out at 64GB if [ -n "${MEMSUSPEND:-}" ]; then From b02555d748c1b39119a54688953a23ba3aef10e9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 3 Mar 2025 12:23:17 -0500 Subject: [PATCH 7/7] Update parallelise --- ci3/parallelise | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/parallelise b/ci3/parallelise index 3d6299640a50..9c27140d7b7f 100755 --- a/ci3/parallelise +++ b/ci3/parallelise @@ -5,7 +5,7 @@ NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source cd $root jobs=$(get_num_cpus_max ${1:-}) -parallel_args="-j$jobs --memsuspend $(memsupend_limit) --line-buffer --joblog joblog.txt" +parallel_args="-j$jobs --memsuspend $(memsuspend_limit) --line-buffer --joblog joblog.txt" # If not in CI, fail fast. if [ "$CI" -eq 0 ]; then