diff --git a/bot/check-build.sh b/bot/check-build.sh index a3c3a3a1..507d962b 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -502,7 +502,9 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) + reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) @@ -531,6 +533,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')" fi comment_artifacts_list="${comment_artifacts_list}" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" + comment_artifacts_list="${comment_artifacts_list}
"
+ if [[ ! -z ${reprod_shortened} ]]; then
+ while IFS= read -r reprod ; do
+ comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${reprod})"
+ done <<< "${reprod_shortened}"
+ else
+ comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
+ fi
+ comment_artifacts_list="${comment_artifacts_list}"
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
comment_artifacts_list="${comment_artifacts_list}"
if [[ ! -z ${other_shortened} ]]; then
diff --git a/create_tarball.sh b/create_tarball.sh
index 84956d14..d104e210 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -90,8 +90,13 @@ for subdir in ${sw_subdirs}; do
for package_version in $(cat ${module_files_list}); do
echo "handling ${package_version}"
find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
+ # if there is a directory for this installation in the stack's reprod directory, include that too
+ if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then
+ find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
+ fi
done
fi
+
done
# add a bit debug output
diff --git a/eb_hooks.py b/eb_hooks.py
index e20ef2e4..baa7958d 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -1,5 +1,6 @@
# Hooks to customize how EasyBuild installs software in EESSI
# see https://docs.easybuild.io/en/latest/Hooks.html
+import datetime
import glob
import os
import re
@@ -7,9 +8,10 @@
import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import obtain_config_guess
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
+from easybuild.tools import config
from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import build_option, update_build_option
-from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which
+from easybuild.tools.config import build_option, install_path, update_build_option
+from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
@@ -46,6 +48,9 @@
# Make sure a single environment variable name is used for this throughout the hooks
EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220"
+STACK_REPROD_SUBDIR = 'reprod'
+
+
def is_gcccore_1220_based(**kwargs):
# ecname, ecversion, tcname, tcversion):
"""
@@ -516,6 +521,20 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs):
del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR]
+def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs):
+ """
+ Post easyblock hook that copies the easybuild subdirectory of every installed application
+ to a central and timestamped location in the root of the software stack, e.g.:
+ /path/to/stack/reprod/MyApp/1.2-foss-2025a/20250102T12:34:56Z
+ """
+
+ stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR)
+ now_utc_timestamp = datetime.datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S%Z')
+ app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg))
+ app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild')
+ copy_dir(app_easybuild_dir, app_reprod_dir)
+
+
# Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set
# so that the modules can be succesfully loaded without printing the error (so that we can create a module
# _with_ the warning for the current software being installed)
@@ -1297,6 +1316,24 @@ def post_module_hook(self, *args, **kwargs):
post_module_hook_zen4_gcccore1220(self, *args, **kwargs)
+# The post_easyblock_hook was introduced in EasyBuild 5.1.1.
+# Older versions would fail if the function is defined anyway, as EasyBuild performs some checks on function names in hooks files.
+if EASYBUILD_VERSION >= '5.1.1':
+ def post_easyblock_hook(self, *args, **kwargs):
+ """Main post easyblock hook: trigger custom functions based on software name."""
+ if self.name in POST_EASYBLOCK_HOOKS:
+ POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs)
+
+ # Always trigger this one for EESSI CVMFS/site installations and version 2025.06 or newer, regardless of self.name
+ if os.getenv('EESSI_CVMFS_INSTALL') or os.getenv('EESSI_SITE_INSTALL'):
+ if os.getenv('EESSI_VERSION') and LooseVersion(os.getenv('EESSI_VERSION')) >= '2025.06':
+ post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs)
+ else:
+ self.log.debug("No CVMFS/site installation requested, not running post_easyblock_hook_copy_easybuild_subdir.")
+else:
+ print_msg(f"Not enabling the post_easybuild_hook, as it requires EasyBuild 5.1.1 or newer.")
+
+
PARSE_HOOKS = {
'casacore': parse_hook_casacore_disable_vectorize,
'CGAL': parse_hook_cgal_toolchainopts_precise,
@@ -1365,6 +1402,8 @@ def post_module_hook(self, *args, **kwargs):
POST_MODULE_HOOKS = {}
+POST_EASYBLOCK_HOOKS = {}
+
# Define parallelism limit operations
def divide_by_factor(parallel, factor):
"""Divide parallelism by given factor"""