diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
index 540fafcea1..4521186580 100644
--- a/.github/workflows/container_tests.yml
+++ b/.github/workflows/container_tests.yml
@@ -17,10 +17,10 @@ jobs:
python: [3.7]
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: ${{matrix.python}}
architecture: x64
@@ -28,7 +28,7 @@ jobs:
- name: install OS & Python packages
run: |
# for modules tool
- APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+ APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
# for building Singularity images
APT_PKGS+=" rpm dnf"
@@ -74,7 +74,7 @@ jobs:
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
- pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+ pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
- name: run test
run: |
@@ -95,7 +95,7 @@ jobs:
echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
# build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
# see https://docs.easybuild.io/en/latest/Containers.html
- curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+ curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
export EASYBUILD_CONTAINERPATH=$PWD
export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
index f7040000f9..1f8417ea7d 100644
--- a/.github/workflows/container_tests_apptainer.yml
+++ b/.github/workflows/container_tests_apptainer.yml
@@ -15,13 +15,13 @@ jobs:
strategy:
matrix:
python: [3.7]
- apptainer: [1.0.0, 1.1.7]
+ apptainer: [1.0.0, 1.1.7, 1.3.6]
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: set up Python
- uses: actions/setup-python@v3
+ uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: ${{matrix.python}}
architecture: x64
@@ -31,7 +31,7 @@ jobs:
# for building CentOS 7 container images
APT_PKGS="rpm dnf"
# for modules tool
- APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+ APT_PKGS+=" lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
# Avoid apt-get update, as we don't really need it,
# and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
@@ -41,12 +41,6 @@ jobs:
sudo apt-get install $APT_PKGS
fi
- # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
- # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
- if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
- sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
- fi
-
- name: install Lmod
run: |
# avoid downloading modules tool sources into easybuild-framework dir
@@ -74,7 +68,7 @@ jobs:
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
- pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+ pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
- name: run test
run: |
@@ -95,7 +89,7 @@ jobs:
echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
# build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
# see https://docs.easybuild.io/en/latest/Containers.html
- curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+ curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
export EASYBUILD_CONTAINERPATH=$PWD
export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
export EASYBUILD_CONTAINER_TYPE='apptainer'
diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml
index 220258cd9f..a2ef7c6c1d 100644
--- a/.github/workflows/eb_command.yml
+++ b/.github/workflows/eb_command.yml
@@ -11,16 +11,19 @@ concurrency:
jobs:
test-eb:
- runs-on: ubuntu-20.04
strategy:
matrix:
- python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+ python: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
+ include:
+ - python: 3.7
+ os: ubuntu-22.04
fail-fast: false
+ runs-on: ${{matrix.os || 'ubuntu-24.04'}}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: ${{matrix.python}}
architecture: x64
@@ -32,9 +35,13 @@ jobs:
# update to latest pip, check version
pip install --upgrade pip
pip --version
+ if ! python -c "import distutils" 2> /dev/null; then
+ # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+ pip install --upgrade setuptools
+ fi
# for modules tool
- APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+ APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
# Avoid apt-get update, as we don't really need it,
# and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
@@ -44,12 +51,6 @@ jobs:
sudo apt-get install $APT_PKGS
fi
- # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
- # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
- if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
- sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
- fi
-
- name: install modules tool
run: |
# avoid downloading modules tool sources into easybuild-framework dir
@@ -91,7 +92,7 @@ jobs:
pymajver=$(python -c 'import sys; print(sys.version_info[0])')
pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
# check patterns in verbose output
- for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+ for pattern in "^>> Considering .python3.\.\.\." "^>> .python3. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python3' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python3 \(.*/bin/python3\)" "^This is EasyBuild 5\.[0-9.]\+"; do
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
grep "$pattern" eb_version.out
done
@@ -103,7 +104,7 @@ jobs:
for eb_python in "python${pymajver}" "python${pymajminver}"; do
export EB_PYTHON="${eb_python}"
eb --version | tee eb_version.out 2>&1
- for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+ for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 5\.[0-9.]\+"; do
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
grep "$pattern" eb_version.out
done
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
index e74b65cc79..b2bf320d94 100644
--- a/.github/workflows/end2end.yml
+++ b/.github/workflows/end2end.yml
@@ -7,29 +7,28 @@ jobs:
strategy:
matrix:
container:
- - centos-7.9
- centos-8.5
- - fedora-36
+ - fedora-41
- opensuse-15.4
- - rockylinux-8.8
- - rockylinux-9.2
+ - rockylinux-8.10
+ - rockylinux-9.5
- ubuntu-20.04
- ubuntu-22.04
+ - ubuntu-24.04
fail-fast: false
container:
image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
- env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
steps:
- name: Check out the repo
- uses: actions/checkout@v3
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: download and unpack easyblocks and easyconfigs repositories
run: |
cd $HOME
for pkg in easyblocks easyconfigs; do
- curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
- tar xfz develop.tar.gz
- rm -f develop.tar.gz
+ curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/5.0.x.tar.gz
+ tar xfz 5.0.x.tar.gz
+ rm -f 5.0.x.tar.gz
done
- name: Set up environment
@@ -37,7 +36,7 @@ jobs:
run: |
# collect environment variables to be set in subsequent steps in script that can be sourced
echo "export PATH=$PWD:$PATH" > /tmp/eb_env
- echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+ echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-5.0.x:$HOME/easybuild-easyconfigs-5.0.x" >> /tmp/eb_env
- name: Run commands to check test environment
shell: bash
@@ -50,6 +49,7 @@ jobs:
"eb --show-system-info"
"eb --check-eb-deps"
"eb --show-config"
+ "eb -x bzip2-1.0.8.eb"
)
for cmd in "${cmds[@]}"; do
echo ">>> $cmd"
@@ -59,4 +59,8 @@ jobs:
- name: End-to-end test of installing bzip2 with EasyBuild
shell: bash
run: |
- sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"
+ EB_ARGS=''
+ if [[ "${{ matrix.container }}" == "fedora-41" ]] || [[ "${{ matrix.container }}" == "ubuntu-24.04" ]]; then
+ EB_ARGS='--filter-deps=binutils'
+ fi
+ sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}"
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index e7c25b22d7..d5a7163f70 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -10,16 +10,19 @@ concurrency:
jobs:
python-linting:
- runs-on: ubuntu-20.04
strategy:
matrix:
- python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-
+ python: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
+ include:
+ - python: 3.7
+ os: ubuntu-22.04
+ fail-fast: false
+ runs-on: ${{matrix.os || 'ubuntu-24.04'}}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: ${{ matrix.python-version }}
@@ -30,10 +33,4 @@ jobs:
- name: Run flake8 to verify PEP8-compliance of Python code
run: |
- # don't check py2vs3/py3.py when testing with Python 2, and vice versa
- if [[ "${{ matrix.python-version }}" =~ "2." ]]; then
- py_excl=py3
- else
- py_excl=py2
- fi
- flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py
+ flake8
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 8a8b13af96..45f32bd866 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -11,53 +11,47 @@ concurrency:
jobs:
setup:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-latest
outputs:
- lmod7: Lmod-7.8.22
- lmod8: Lmod-8.7.6
- modulesTcl: modules-tcl-1.147
- modules3: modules-3.2.10
- modules4: modules-4.1.4
+ lmod8: Lmod-8.7.58
+ modules4: modules-4.5.3
+ modules5: modules-5.3.1
steps:
- run: "true"
build:
needs: setup
- runs-on: ubuntu-20.04
+ runs-on: ${{matrix.os || 'ubuntu-24.04'}}
strategy:
matrix:
- python: [3.6]
+ # Python 3.10 is default in Ubuntu 22.04
+ python: ['3.10']
modules_tool:
# use variables defined by 'setup' job above, see also
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
- - ${{needs.setup.outputs.lmod7}}
- ${{needs.setup.outputs.lmod8}}
- - ${{needs.setup.outputs.modulesTcl}}
- - ${{needs.setup.outputs.modules3}}
- ${{needs.setup.outputs.modules4}}
- lc_all: [""]
+ - ${{needs.setup.outputs.modules5}}
include:
- # Test different Python 3 versions with Lmod 8.x
- - python: 3.7
+ # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+ - python: '3.7'
modules_tool: ${{needs.setup.outputs.lmod8}}
- - python: 3.8
+ os: ubuntu-22.04
+ - python: '3.8'
modules_tool: ${{needs.setup.outputs.lmod8}}
- - python: 3.9
- modules_tool: ${{needs.setup.outputs.lmod8}}
- - python: '3.10'
+ - python: '3.9'
modules_tool: ${{needs.setup.outputs.lmod8}}
- python: '3.11'
modules_tool: ${{needs.setup.outputs.lmod8}}
- # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
- # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
- - python: 3.6
+ - python: '3.12'
+ modules_tool: ${{needs.setup.outputs.lmod8}}
+ - python: '3.13'
modules_tool: ${{needs.setup.outputs.lmod8}}
- lc_all: C
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: ${{matrix.python}}
architecture: x64
@@ -65,7 +59,7 @@ jobs:
- name: install OS & Python packages
run: |
# for modules tool
- APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+ APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
# for GitPython, python-hglib
APT_PKGS+=" git mercurial"
# dep for GC3Pie
@@ -79,19 +73,18 @@ jobs:
sudo apt-get install $APT_PKGS
fi
- # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
- # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
- if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
- sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
- fi
# Python packages
pip --version
pip install --upgrade pip
pip --version
pip install -r requirements.txt
+ if ! python -c "import distutils" 2> /dev/null; then
+ # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+ pip install --upgrade setuptools
+ fi
# git config is required to make actual git commits (cfr. tests for GitRepository)
- git config --global user.name "Travis CI"
- git config --global user.email "travis@travis-ci.org"
+ git config --global user.name "Github Actions"
+ git config --global user.email "actions@github.com"
git config --get-regexp 'user.*'
- name: install GitHub token (if available)
@@ -101,9 +94,9 @@ jobs:
# and are only run after the PR gets merged
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
run: |
- # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
+ # only install GitHub token when testing with Lmod 8.x + Python 3.9, to avoid hitting GitHub rate limit
# tests that require a GitHub token are skipped automatically when no GitHub token is available
- if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
+ if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.9 ]]; then
if [ ! -z $GITHUB_TOKEN ]; then
SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
@@ -129,7 +122,7 @@ jobs:
run: |
# make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements,
# since EasyBuild should not have a runtime requirement on setuptools
- SETUPTOOLS_IMPORTS=$(egrep -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
+ SETUPTOOLS_IMPORTS=$(egrep --exclude setup.py -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1)
- name: install sources
@@ -143,14 +136,16 @@ jobs:
- name: run test suite
env:
EB_VERBOSE: 1
- LC_ALL: ${{matrix.lc_all}}
+ LC_ALL: ""
run: |
# run tests *outside* of checked out easybuild-framework directory,
# to ensure we're testing installed version (see previous step)
cd $HOME
# initialize environment for modules tool
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
- source $(cat $HOME/mod_init); type module
+ source $(cat $HOME/mod_init)
+ type module
+ module --version
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
# also pick up changes to $PATH set by sourcing $MOD_INIT
export PREFIX=/tmp/$USER/$GITHUB_SHA
@@ -158,11 +153,7 @@ jobs:
export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
eb --version
# tell EasyBuild which modules tool is available
- if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
- export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
- elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
- export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
- elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+ if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModules
else
export EASYBUILD_MODULES_TOOL=Lmod
@@ -195,9 +186,8 @@ jobs:
IGNORE_PATTERNS+="|skipping SvnRepository test"
IGNORE_PATTERNS+="|requires Lmod as modules tool"
IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
- IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7"
IGNORE_PATTERNS+="|from cryptography.* import "
- IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
IGNORE_PATTERNS+="|Blowfish"
IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml
deleted file mode 100644
index aa00f9a2fd..0000000000
--- a/.github/workflows/unit_tests_python2.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
-name: EasyBuild framework unit tests (python2)
-on: [push, pull_request]
-
-permissions:
- contents: read # to fetch code (actions/checkout)
-
-concurrency:
- group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
- cancel-in-progress: true
-
-jobs:
- test_python2:
- runs-on: ubuntu-20.04
- container:
- # CentOS 7.9 container that already includes Lmod & co,
- # see https://github.com/easybuilders/easybuild-containers
- image: ghcr.io/easybuilders/centos-7.9-amd64
- env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
- steps:
- - uses: actions/checkout@v3
-
- - name: install Python packages
- run: |
- # Python packages
- python2 -V
- python2 -m pip --version
- python2 -m pip install --upgrade pip
- python2 -m pip --version
- # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
- sed -i '/GC3Pie/d' requirements.txt
- python2 -m pip install -r requirements.txt
- # git config is required to make actual git commits (cfr. tests for GitRepository)
- sudo -u easybuild git config --global user.name "GitHub Actions"
- sudo -u easybuild git config --global user.email "actions@github.com"
- sudo -u easybuild git config --get-regexp 'user.*'
-
- - name: install GitHub token (if available)
- env:
- # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
- # this token is not available in pull requests, so tests that require it are skipped in PRs,
- # and are only run after the PR gets merged
- GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
- run: |
- # tests that require a GitHub token are skipped automatically when no GitHub token is available
- if [ ! -z $GITHUB_TOKEN ]; then
- sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
- echo "GitHub token installed!"
- else
- echo "Installation of GitHub token skipped!"
- fi
-
- - name: install sources
- run: |
- # install from source distribution tarball, to test release as published on PyPI
- python2 setup.py sdist
- ls dist
- export PREFIX=/tmp/$USER/$GITHUB_SHA
- python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
-
- - name: run test suite
- run: |
- # run tests *outside* of checked out easybuild-framework directory,
- # to ensure we're testing installed version (see previous step)
- cd $HOME
- # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
- export PREFIX=/tmp/$USER/$GITHUB_SHA
- ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
- ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
- # run EasyBuild command via (non-root) easybuild user + login shell
- sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
- # show active EasyBuild configuration
- sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
- # gather some useful info on test system
- sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
- # check GitHub configuration
- sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
- # create file owned by root but writable by anyone (used by test_copy_file)
- sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
- sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
- # run test suite (via easybuild user + login shell)
- sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 4eed2c680c..9375667477 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -1,10 +1,150 @@
This file contains a description of the major changes to the easybuild-framework EasyBuild package.
For more detailed information, please see the git log.
-These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+These release notes can also be consulted at https://docs.easybuild.io/release-notes .
-v4.9.4 (22 september 2024)
+v5.0.0 (18 March 2025)
+----------------------
+
+- remove support for Python 2.7 and 3.5 (#4229, #4270, #4306, #4473, #4477, #4476, #4478, #4524, #4607, #4756, #4810, #4811)
+ - also run unit test suite with Python 3.12 + 3.13 (#4484, #4674)
+- changed defaults in EasyBuild configuration:
+ - enable --trace by default (#4250, #4310, #4311, #4491)
+ - enable RPATH linking by default (#4448, #4515, #4779)
+ - disable strict RPATH sanity check by default, allow re-enabling it via `--strict-rpath-sanity-check` configuration option (#4475)
+ - enable `module-depends-on` by default (#4500)
+ - enable `module-extensions` by default (#4501)
+ - set default maximum level of parallellism to `16` via `max-parallel` configuration option (#4606, #4777, #4816)
+ - use Slurm as the default job backend (#4659)
+ - enable keeping of symbolic links by default when copying (set `keepsymlinks` easyconfig parameter to `True`) (#4773)
+- changed behaviour:
+ - change default for `change_into_dir` option in `extract_file` function to to `False` (#4246)
+ - enforce correct `.patch(.*)` extension for patch files (#4247)
+ - create `lib` -> `lib64` symlink (or vice versa) *before* running `postinstallcmds` (#4435)
+ - don't allow unresolved templates in easyconfig parameters by default + add support for `--allow-unresolved-templates` configuration option (#4516, #4725, #4726, #4727)
+ - switch default checksum to `sha256` (#4523)
+ - only allow use of `rpath` toolchain option when `system` toolchain is used (#4585, #4808)
+ - use default value `$XDG_CONFIG_DIRS` from XDG basedir spec: `/etc/xdg` (instead of `/etc`) (#4591)
+ - reverse order for parsing files in `XDG_CONFIG_DIRS` (#4630)
+ - move verifying of checksums from `source` to `fetch` step, to include it with `--fetch` (#4624, #4729)
+ - drop support for pep8 package (was used for `--check-contrib` + `--check-style`) (#4634)
+ - specify changes that should be made by generated module files via `module_load_environment` attribute of `EasyBlock` class (#4653, #4754, #4761, #4774, #4799, #4784, #4801, #4802, #4803, #4809)
+ - let jobs retweak easyconfigs themselves by passing down `--try-*` options (#4669)
+ - refactor `make_extension_string` method in `EasyBlock` class (#4690)
+ - change `Toolchain.get_flag` so it doesn't automatically prepend a dash (`-`) to compiler flags (#4698)
+ - rename `Compiler.COMPILER*_FLAGS` to `Compiler.COMPILER*_OPTIONS` (#4698)
+ - change semantics of `--dry-run`, so it doesn't imply `--robot` (#4704)
+ - run sanity checks commands from an empty temporary directory (rather than the software install directory) (#4723)
+ - add context manager for allowing unresolved templates and make the state members private (#4735)
+ - also remove support for directly setting `enable_templating` and `expect_resolved_template_values`
+- various enhancements:
+ - `run_shell_cmd` function to run shell commands which replaces both the (now deprecated) `run_cmd` and `run_cmd_qa` functions (#4284, #4314, #4321, #4322, #4327, #4334, #4335, #4336, #4351, #4356, #4378, #4380, #4383, #4390, #4422, #4423, #4427, #4428, #4430, #4431, #4432, #4441, #4443, #4444, #4453, #4454, #4471, #4504, #4509, #4544, #4612, #4617, #4664, #4721, #4728, #4731, #4755, #4757)
+ - detect Fortran `.mod` files for installations using `GCCcore` toolchain (#4389)
+ - create `env.sh` and `cmd.sh` helper scripts in `run_shell_cmd` to allow starting interactive shell to debug failing shell commands (#4486, #4611, #4662, #4666, #4685, #4792)
+ - add support for alternate easyconfig parameters/templates/constants (#4511, #4514, #4549, #4555)
+ - create reproducible tarballs for sources created via `git_config` (requires `.tar.xz` + Python 3.9+) (#4248, #4517, #4522, #4660, #4733, #4797, #4798, #4813)
+ - use more granular exit codes when `EasyBuildError` is raised (#4534)
+ - prepend to `$PYTHONPATH` or `$EBPYTHONPREFIXES` in generated module files by automatically scanning for Python site package directories (as configured via `prefer-python-search-path`) (#4539, #4686)
+ - copy build directory and/or log file(s) if installation failed to path specified via `--failed-install-build-dirs-path` or `--failed-install-logs-path` (#4601)
+ - add `--search-path-cpp-headers` configuration option to control how EasyBuild sets paths to headers at build time (#4645)
+ - add `module-search-path-headers` configuration option to control how modules set search paths to header files (#4655)
+ - add `--keep-debug-symbols` configuration option to set default value of '`debug`' toolchain option (#4688, #4764)
+ - add `--search-path-linker` option to control linker options at build time (#4697)
+ - don't raise error when required extensions are not found when installing extensions in parallel (#4671)
+ - mark support for installing extensions in parallel as being mature, since it's no longer experimental (#4672)
+ - mark easystack support as being mature, since it's no longer experimental (#4673)
+ - and several additional small enhancements:
+ - enhance download instructions by mentioning active source path (#4459)
+ - add CUDA compute capability integer format templates (`cuda_int_*_sep`) (#4463)
+ - add new `get_cwd` function to `tools.filetools` to retrieve current working directory (#4525)
+ - use `dict.items()` instead of repeatedly getting the value (#4533)
+ - set `usedforsecurity` to `False` when calling `hashlib.md5` with Python >= 3.9 (#4550)
+ - add `GNU_FTP_SOURCE` template constant (#4598)
+ - allow using `amend/try-amend` multiple times in an easystack file entry (#4667)
+ - add support for `%(rpath_enabled)s` template value (#4670)
+ - add `resolve_template` method to `EasyConfig` class (#4677)
+ - allow templates in `custom_paths` & `custom_commands` sanity-check arguments (#4679)
+ - updates and fixes for `findPythonDeps` script (#4682, #4740)
+ - allow use of custom delimiter for paths in module generator (#4687)
+ - enhance `get_software_libdir` to return full paths if requested (#4699)
+ - enhance `EasyBlock` class to allow passing in `logfile` (#4707)
+ - avoid checking loaded module twice in `findUpdatedEcs.sh` script (#4710)
+ - allow nesting values in checksum dicts (#4711) Flamefire:non-dict MERGED 2024-12-02T12:14:12Z
+ - use `enumerate` where applicable + fix for `ModuleGenerator._generate_multi_deps_list` (#4720)
+ - faster `nub` function (#4737)
+ - enhance `apply_regex_substitutions` to support use of multi-line patterns, requiring matching all patterns in each file, and use pre-compiled regular expressions (#4758)
+ - ignore other classes if software specific easyblock class was found (#4769)
+ - also allow trailing whitespaces for `examples` and `citing` easyconfig parameters (#4796)
+ - enhance `get_gpu_info` to also use `amd-smi` for AMD GPUs if possible (#4805)
+- various changes, improvemnents, and fixes for the supported modules tools (Environment Modules + Lmod)
+ - drop load storm safe guard for Environment Modules v4.2.4+ (#4373)
+ - run unit tests on an updated versions of Environment Modules: v4.5.3 + v5.3.1 (#4415)
+ - add `check_group` support for module files in Tcl syntax (#4418)
+ - bump minimum required Lmod to 8.0.0 (#4424)
+ - bump minimum required Tmod (4.x) to 4.3.0 (#4425)
+ - use `getenv` modulefile command with `EnvironmentModules` >= 4.2.0 (#4614)
+ - add module cache build support in `EnvironmentModules` class (#4615)
+ - derive `EnvironmentModules` class directly from `ModulesTool` rather than from to be deprecated `EnvironmentModulesTcl` (#4625)
+ - adapt `module show` command run to cope with non-zero exit code for non-existing module (required for Environment Modules v5.5+ and Lmod 8.7.56+) (#4739)
+- various improvements for the framework test suite:
+ - allow test case filtering by class name (#4788)
+ - improve test failure output for `assertEqual` (#4807)
+ - fix `test_modulerc` by taking into account that wrapper module may also be shown as being loaded (#4778)
+ - (temporarily) use 5.0.x branch for easyblocks + easyconfigs in CI workflows (#4358)
+ - fix GitHub Actions for CentOS 7.9 container (#4712)
+ - stop using Ubuntu 20.4 in GitHub Actions workflows, use Ubuntu 22.04 instead (#4783)
+ - update GitHub actions workflows to use `ubuntu-24.04` where possible (#4795)
+- various small bug fixes:
+ - silence output of included easyblocks for `--terse` (#4765)
+ - avoid processing the same EasyConfig multiple times (#4767)
+ - fix easyconfig parameter deprecation (#4479, #4480)
+ - switch from `ls` to `bash` in tests that are expecting this to be a binary (#4492)
+ - fix the checksum type check (#4578)
+ - fix `to_checksums` with `None` values in dicts and recursion (#4579)
+ - fix `test_toy_lock_cleanup_signals` (#4600)
+ - resolve symlink when making log dir writable (#4658)
+ - fix typo in `veryloose` toolchain option for RISC-V (#4668)
+ - fix dry-run output when using `multi_deps` (#4678)
+ - make `LooseVersion('1.0') == LooseVersion('1')` (#4691)
+ - fix FFT entry in `--list-toolchains` output for Cray toolchains (#4719)
+ - avoid making build directory read-only (#4736)
+- deprecated functionality:
+ - `easybuild.tools.py2vs3` module (#4229)
+ - rename unclear `*run*` methods to `*install_extension*` + rename `install_extensions` to `install_all_extensions` (#4400)
+ - deprecate `run_cmd` and `run_cmd_qa` & co, move them to `easybuild._deprecated` module (#4433)
+ - deprecate support for `EnvironmentModulesC` and `EnvironmentModulesTcl` module tools (#4439)
+ - deprecate old checksum types (incl. md5) (#4526, #4545)
+ - deprecate use of `parallel` easyconfig parameter and fix updating the template value (#4580)
+ - convert template constant lists to dicts and export the constants by name (#4595)
+ - rename '`source`' step to '`extract`' (affects `skipsteps` easyconfig parameter + `--stop` option) (#4629)
+ - deprecate `make_module_req_guess` method in `EasyBlock` class (#4653, #4763)
+ - deprecate support for GC3Pie as job backend (#4659)
+ - add deprecation warning for `optarch` value without leading dash (#4698)
+ - deprecate `post_install_step` method in `EasyBlock`, was renamed to `post_processing_step` (#4715)
+- remove functionality that was deprecated in EasyBuild v4.x, including:
+ - EasyBuild bootstrap script (#4233)
+ - support for YAML-based easyconfig format (.yeb) (#4237)
+ - `wait-on-lock` configuration setting (#4239)
+ - `dummy` toolchain (#4240)
+ - `accept-eula` configuration setting (#4242)
+ - `is_generic_easyblock` function from `easybuild.framework.easyconfig.easyconfig` (#4243)
+ - `use_git_am` option for `apply_patch` function (#4244)
+ - `fetch_extension_sources` method in `EasyBlock` class (#4245)
+ - support for 32-bit targets (#4272)
+ - `descr` option for `simple_option` function (#4273)
+ - `Toolchain.add_dependencies` method (#4274)
+ - `copytree`, `rmtree2` functions from `easybuild.filetools` (#4275)
+ - `skip_symlinks` option for `adjust_permissions` function (#4275)
+ - `log_error` option from `which` function (#4276)
+ - `skip_lower` option from `template_constant_dict` (#4277)
+ - `disable_templating` + `default_fallback` options in `get_easyblock_class` (#4278)
+ - `mod_exists_regex_template` options in `ModulesTool.exist` (#4279)
+- other changes
+ - take into account that `VERBOSE_VERSION` imported from `easybuild.easyblocks` is now a string value (#4357)
+
+
+v4.9.4 (22 September 2024)
--------------------------
update/bugfix release
diff --git a/easybuild/__init__.py b/easybuild/__init__.py
index 3637c9f395..fa668dfd4b 100644
--- a/easybuild/__init__.py
+++ b/easybuild/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2024 Ghent University
+# Copyright 2011-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/_deprecated.py b/easybuild/_deprecated.py
new file mode 100644
index 0000000000..2beebd8174
--- /dev/null
+++ b/easybuild/_deprecated.py
@@ -0,0 +1,854 @@
+# #
+# Copyright 2023-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+# #
+"""
+Deprecated functionality, which will be removed with next major EasyBuild version
+
+Authors:
+
+* Kenneth Hoste (Ghent University)
+"""
+import contextlib
+import functools
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import datetime
+
+import easybuild.tools.asyncprocess as asyncprocess
+from easybuild.base import fancylogger
+from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since
+from easybuild.tools.config import ERROR, IGNORE, WARN, build_option
+from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook
+from easybuild.tools.utilities import nub, trace_msg
+
+
+_log = fancylogger.getLogger('_deprecated', fname=False)
+
+
+errors_found_in_log = 0
+
+# default strictness level
+strictness = WARN
+
+
+CACHED_COMMANDS = [
+ "sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X)
+ "sysctl -n hw.memsize", # used in get_total_memory (OS X)
+ "sysctl -n hw.ncpu", # used in get_avail_core_count (OS X)
+ "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X)
+ "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X)
+ "type module", # used in ModulesTool.check_module_function
+ "type _module_raw", # used in EnvironmentModules.check_module_function
+ "ulimit -u", # used in det_parallelism
+]
+
+
+def run_cmd_cache(func):
+ """Function decorator to cache (and retrieve cached) results of running commands."""
+ cache = {}
+
+ @functools.wraps(func)
+ def cache_aware_func(cmd, *args, **kwargs):
+ """Retrieve cached result of selected commands, or run specified and collect & cache result."""
+
+ # cache key is combination of command and input provided via stdin ('inp' named option)
+ key = (cmd, kwargs.get('inp', None))
+ # fetch from cache if available, cache it if it's not, but only on cmd strings
+ if isinstance(cmd, str) and key in cache:
+ _log.debug("Using cached value for command '%s': %s", cmd, cache[key])
+ return cache[key]
+ else:
+ res = func(cmd, *args, **kwargs)
+ if cmd in CACHED_COMMANDS:
+ cache[key] = res
+ return res
+
+ # expose clear/update methods of cache to wrapped function
+ cache_aware_func.clear_cache = cache.clear
+ cache_aware_func.update_cache = cache.update
+
+ return cache_aware_func
+
+
+def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True):
+ """
+ Get output from running process (that was opened with subprocess.Popen).
+
+ :param proc: process to get output from
+ :param read_size: number of bytes of output to read (if None: read all output)
+ :param asynchronous: get output asynchronously
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("get_output_from_process is deprecated, you should stop using it", '6.0')
+
+ if asynchronous:
+ # e=False is set to avoid raising an exception when command has completed;
+ # that's needed to ensure we get all output,
+ # see https://github.com/easybuilders/easybuild-framework/issues/3593
+ output = asyncprocess.recv_some(proc, e=False)
+ elif read_size:
+ output = proc.stdout.read(read_size)
+ else:
+ output = proc.stdout.read()
+
+ # need to be careful w.r.t. encoding since we want to obtain a string value,
+ # and the output may include non UTF-8 characters
+ # * in Python 2, .decode() returns a value of type 'unicode',
+ # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors)
+ # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read()
+ output = str(output.decode('ascii', 'ignore'))
+
+ return output
+
+
+@run_cmd_cache
+def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
+ force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
+ with_hooks=True, with_sysroot=True):
+ """
+ Run specified command (in a subshell)
+ :param cmd: command to run
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param inp: the input given to the command via stdin
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param log_output: indicate whether all output of command should be logged to a separate temporary logfile
+ :param path: path to execute the command in; current working directory is used if unspecified
+ :param force_in_dry_run: force running the command during dry run
+ :param verbose: include message on running the command in dry run output
+ :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
+ :param trace: print command being executed as part of trace output
+ :param stream_output: enable streaming command output to stdout
+ :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
+ :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
+ :param with_sysroot: prepend sysroot to exec_cmd (if defined)
+ """
+
+ _log.deprecated("run_cmd is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0')
+
+ cwd = os.getcwd()
+
+ if isinstance(cmd, str):
+ cmd_msg = cmd.strip()
+ elif isinstance(cmd, list):
+ cmd_msg = ' '.join(cmd)
+ else:
+ raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
+
+ if shell is None:
+ shell = True
+ if isinstance(cmd, list):
+ raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! "
+ "Note that all elements of the list but the first are treated as arguments "
+ "to the shell and NOT to the command to be executed!")
+
+ if log_output or (trace and build_option('trace')):
+ # collect output of running command in temporary log file, if desired
+ fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
+ os.close(fd)
+ try:
+ cmd_log = open(cmd_log_fn, 'w')
+ except IOError as err:
+ raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
+ _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
+ else:
+ cmd_log_fn, cmd_log = None, None
+
+ # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
+ if stream_output is None and build_option('logtostdout'):
+ _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
+ stream_output = True
+
+ if stream_output:
+ print_msg("(streaming) output for command '%s':" % cmd_msg)
+
+ start_time = datetime.now()
+ if trace:
+ trace_txt = "running command:\n"
+ trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
+ trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
+ if inp:
+ trace_txt += "\t[input: %s]\n" % inp
+ trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
+ trace_msg(trace_txt + '\t' + cmd_msg)
+
+ # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
+ if not force_in_dry_run and build_option('extended_dry_run'):
+ if path is None:
+ path = cwd
+ if verbose:
+ dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
+ dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
+
+ # make sure we get the type of the return value right
+ if simple:
+ return True
+ else:
+ # output, exit code
+ return ('', 0)
+
+ try:
+ if path:
+ os.chdir(path)
+
+ _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
+ except OSError as err:
+ _log.warning("Failed to change to %s: %s" % (path, err))
+ _log.info("running cmd %s in non-existing directory, might fail!", cmd)
+
+ if cmd_log:
+ cmd_log.write("# output for command: %s\n\n" % cmd_msg)
+
+ exec_cmd = "/bin/bash"
+
+ # if EasyBuild is configured to use an alternate sysroot,
+ # we should also run shell commands using the bash shell provided in there,
+ # since /bin/bash may not be compatible with the alternate sysroot
+ if with_sysroot:
+ sysroot = build_option('sysroot')
+ if sysroot:
+ sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash')
+ if os.path.exists(sysroot_bin_bash):
+ exec_cmd = sysroot_bin_bash
+
+ if not shell:
+ if isinstance(cmd, list):
+ exec_cmd = None
+ cmd.insert(0, '/usr/bin/env')
+ elif isinstance(cmd, str):
+ cmd = '/usr/bin/env %s' % cmd
+ else:
+ raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
+
+ _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd)
+
+ if with_hooks:
+ hooks = load_hooks(build_option('hooks'))
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})
+ if isinstance(hook_res, str):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
+
+ _log.info('running cmd: %s ' % cmd)
+ try:
+ proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
+ except OSError as err:
+ raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
+
+ if inp:
+ proc.stdin.write(inp.encode())
+ proc.stdin.close()
+
+ if asynchronous:
+ return (proc, cmd, cwd, start_time, cmd_log)
+ else:
+ return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple,
+ regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks,
+ print_deprecation_warning=False)
+
+
+def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
+ """
+ Check status of command that was started asynchronously.
+
+ :param proc: subprocess.Popen instance representing asynchronous command
+ :param cmd: command being run
+ :param owd: original working directory
+ :param start_time: start time of command (datetime instance)
+ :param cmd_log: log file to print command output to
+ :param fail_on_error: raise EasyBuildError when command exited with an error
+ :param output_read_size: number of bytes to read from output
+ :param output: already collected output for this command
+
+ :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
+ """
+
+ _log.deprecated("check_async_cmd is deprecated, you should stop using it", '6.0')
+
+ # use small read size, to avoid waiting for a long time until sufficient output is produced
+ if output_read_size:
+ if not isinstance(output_read_size, int) or output_read_size < 0:
+ raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)")
+ add_out = get_output_from_process(proc, read_size=output_read_size, print_deprecation_warning=False)
+ _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
+ output += add_out
+
+ exit_code = proc.poll()
+ if exit_code is None:
+ _log.debug("Asynchronous command '%s' still running..." % cmd)
+ done = False
+ else:
+ _log.debug("Asynchronous command '%s' completed!", cmd)
+ output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
+ simple=False, trace=False, log_ok=fail_on_error,
+ print_deprecation_warning=False)
+ done = True
+
+ res = {
+ 'done': done,
+ 'exit_code': exit_code,
+ 'output': output,
+ }
+ return res
+
+
+def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
+ regexp=True, stream_output=None, trace=True, output='', with_hook=True,
+ print_deprecation_warning=True):
+ """
+ Complete running of command represented by passed subprocess.Popen instance.
+
+ :param proc: subprocess.Popen instance representing running command
+ :param cmd: command being run
+ :param owd: original working directory
+ :param start_time: start time of command (datetime instance)
+ :param cmd_log: log file to print command output to
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param stream_output: enable streaming command output to stdout
+ :param trace: print command being executed as part of trace output
+ :param with_hook: trigger post run_shell_cmd hooks (if defined)
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("complete_cmd is deprecated, you should stop using it", '6.0')
+
+ # use small read size when streaming output, to make it stream more fluently
+ # read size should not be too small though, to avoid too much overhead
+ if stream_output:
+ read_size = 128
+ else:
+ read_size = 1024 * 8
+
+ stdouterr = output
+
+ try:
+ ec = proc.poll()
+ while ec is None:
+ # need to read from time to time.
+ # - otherwise the stdout/stderr buffer gets filled and it all stops working
+ output = get_output_from_process(proc, read_size=read_size, print_deprecation_warning=False)
+ if cmd_log:
+ cmd_log.write(output)
+ if stream_output:
+ sys.stdout.write(output)
+ stdouterr += output
+ ec = proc.poll()
+
+ # read remaining data (all of it)
+ output = get_output_from_process(proc, print_deprecation_warning=False)
+ finally:
+ proc.stdout.close()
+
+ if cmd_log:
+ cmd_log.write(output)
+ cmd_log.close()
+ if stream_output:
+ sys.stdout.write(output)
+ stdouterr += output
+
+ if with_hook:
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'exit_code': ec,
+ 'output': stdouterr,
+ 'work_dir': os.getcwd(),
+ }
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
+ if trace:
+ trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+
+ try:
+ os.chdir(owd)
+ except OSError as err:
+ raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err)
+
+ return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False)
+
+
+def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
+ maxhits=50, trace=True):
+ """
+ Run specified interactive command (in a subshell)
+ :param cmd: command to run
+ :param qa: dictionary which maps question to answers
+ :param no_qa: list of patters that are not questions
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param std_qa: dictionary which maps question regex patterns to answers
+ :param path: path to execute the command is; current working directory is used if unspecified
+ :param maxhits: maximum number of cycles (seconds) without being able to find a known question
+ :param trace: print command being executed as part of trace output
+ """
+
+ _log.deprecated("run_cmd_qa is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0')
+
+ cwd = os.getcwd()
+
+ if not isinstance(cmd, str) and len(cmd) > 1:
+ # We use shell=True and hence we should really pass the command as a string
+ # When using a list then every element past the first is passed to the shell itself, not the command!
+ raise EasyBuildError("The command passed must be a string!")
+
+ if log_all or (trace and build_option('trace')):
+ # collect output of running command in temporary log file, if desired
+ fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
+ os.close(fd)
+ try:
+ cmd_log = open(cmd_log_fn, 'w')
+ except IOError as err:
+ raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
+ _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
+ else:
+ cmd_log_fn, cmd_log = None, None
+
+ start_time = datetime.now()
+ if trace:
+ trace_txt = "running interactive command:\n"
+ trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
+ trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
+ trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
+ trace_msg(trace_txt + '\t' + cmd.strip())
+
+ # early exit in 'dry run' mode, after printing the command that would be run
+ if build_option('extended_dry_run'):
+ if path is None:
+ path = cwd
+ dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
+ dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
+ if cmd_log:
+ cmd_log.close()
+ if simple:
+ return True
+ else:
+ # output, exit code
+ return ('', 0)
+
+ try:
+ if path:
+ os.chdir(path)
+
+ _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
+ except OSError as err:
+ _log.warning("Failed to change to %s: %s" % (path, err))
+ _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
+
+ # Part 1: process the QandA dictionary
+ # given initial set of Q and A (in dict), return dict of reg. exp. and A
+ #
+ # make regular expression that matches the string with
+ # - replace whitespace
+ # - replace newline
+
+ def escape_special(string):
+ return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
+
+ split = r'[\s\n]+'
+ regSplit = re.compile(r"" + split)
+
+ def process_QA(q, a_s):
+ splitq = [escape_special(x) for x in regSplit.split(q)]
+ regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
+ # add optional split at the end
+ for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
+ a_s[i] += '\n'
+ regQ = re.compile(r"" + regQtxt)
+ if regQ.search(q):
+ return (a_s, regQ)
+ else:
+ raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
+
+ def check_answers_list(answers):
+ """Make sure we have a list of answers (as strings)."""
+ if isinstance(answers, str):
+ answers = [answers]
+ elif not isinstance(answers, list):
+ if cmd_log:
+ cmd_log.close()
+ raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
+ question, type(answers), answers)
+ # list is manipulated when answering matching question, so return a copy
+ return answers[:]
+
+ new_qa = {}
+ _log.debug("new_qa: ")
+ for question, answers in qa.items():
+ answers = check_answers_list(answers)
+ (answers, regQ) = process_QA(question, answers)
+ new_qa[regQ] = answers
+ _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
+
+ new_std_qa = {}
+ if std_qa:
+ for question, answers in std_qa.items():
+ regQ = re.compile(r"" + question + r"[\s\n]*$")
+ answers = check_answers_list(answers)
+ for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
+ answers[i] += '\n'
+ new_std_qa[regQ] = answers
+ _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
+
+ new_no_qa = []
+ if no_qa:
+ # simple statements, can contain wildcards
+ new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
+
+ _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
+
+ # Part 2: Run the command and answer questions
+ # - this needs asynchronous stdout
+
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'interactive': True,
+ 'work_dir': os.getcwd(),
+ }
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+ if isinstance(hook_res, str):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')",
+ RUN_SHELL_CMD, cmd, old_cmd)
+
+ # # Log command output
+ if cmd_log:
+ cmd_log.write("# output for interactive command: %s\n\n" % cmd)
+
+ # Make sure we close the proc handles and the cmd_log file
+ @contextlib.contextmanager
+ def get_proc():
+ try:
+ proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT,
+ stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash')
+ except OSError as err:
+ if cmd_log:
+ cmd_log.close()
+ raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
+ try:
+ yield proc
+ finally:
+ if proc.stdout:
+ proc.stdout.close()
+ if proc.stdin:
+ proc.stdin.close()
+ if cmd_log:
+ cmd_log.close()
+
+ with get_proc() as proc:
+ ec = proc.poll()
+ stdout_err = ''
+ old_len_out = -1
+ hit_count = 0
+
+ while ec is None:
+ # need to read from time to time.
+ # - otherwise the stdout/stderr buffer gets filled and it all stops working
+ try:
+ out = get_output_from_process(proc, asynchronous=True, print_deprecation_warning=False)
+
+ if cmd_log:
+ cmd_log.write(out)
+ stdout_err += out
+ # recv_some used by get_output_from_process for getting asynchronous output may throw exception
+ except (IOError, Exception) as err:
+ _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
+ out = None
+
+ hit = False
+ for question, answers in new_qa.items():
+ res = question.search(stdout_err)
+ if out and res:
+ fa = answers[0] % res.groupdict()
+ # cycle through list of answers
+ last_answer = answers.pop(0)
+ answers.append(last_answer)
+ _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
+
+ _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
+ asyncprocess.send_all(proc, fa)
+ hit = True
+ break
+ if not hit:
+ for question, answers in new_std_qa.items():
+ res = question.search(stdout_err)
+ if out and res:
+ fa = answers[0] % res.groupdict()
+ # cycle through list of answers
+ last_answer = answers.pop(0)
+ answers.append(last_answer)
+ _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
+
+ _log.debug("run_cmd_qa answer %s std question %s out %s",
+ fa, question.pattern, stdout_err[-50:])
+ asyncprocess.send_all(proc, fa)
+ hit = True
+ break
+ if not hit:
+ if len(stdout_err) > old_len_out:
+ old_len_out = len(stdout_err)
+ else:
+ noqa = False
+ for r in new_no_qa:
+ if r.search(stdout_err):
+ _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
+ noqa = True
+ if not noqa:
+ hit_count += 1
+ else:
+ hit_count = 0
+ else:
+ hit_count = 0
+
+ if hit_count > maxhits:
+ # explicitly kill the child process before exiting
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ os.kill(proc.pid, signal.SIGKILL)
+ except OSError as err:
+ _log.debug("run_cmd_qa exception caught when killing child process: %s", err)
+ _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
+ raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
+ cmd, maxhits, stdout_err[-500:])
+
+ # the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
+ time.sleep(1)
+ ec = proc.poll()
+
+ # Process stopped. Read all remaining data
+ try:
+ if proc.stdout:
+ out = get_output_from_process(proc, print_deprecation_warning=False)
+ stdout_err += out
+ if cmd_log:
+ cmd_log.write(out)
+ except IOError as err:
+ _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
+
+ run_hook_kwargs.update({
+ 'interactive': True,
+ 'exit_code': ec,
+ 'output': stdout_err,
+ })
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
+ if trace:
+ trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+
+ try:
+ os.chdir(cwd)
+ except OSError as err:
+ raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
+
+ return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False)
+
+
+def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=True):
+ """
+ Parse command output and construct return value.
+ :param cmd: executed command
+ :param stdouterr: combined stdout/stderr of executed command
+ :param ec: exit code of executed command
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param log_all: always log command output and exit code
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("parse_cmd_output is deprecated, you should stop using it", '6.0')
+
+ if strictness == IGNORE:
+ check_ec = False
+ fail_on_error_match = False
+ elif strictness == WARN:
+ check_ec = True
+ fail_on_error_match = False
+ elif strictness == ERROR:
+ check_ec = True
+ fail_on_error_match = True
+ else:
+ raise EasyBuildError("invalid strictness setting: %s", strictness)
+
+ # allow for overriding the regexp setting
+ if not regexp:
+ fail_on_error_match = False
+
+ if ec and (log_all or log_ok):
+ # We don't want to error if the user doesn't care
+ if check_ec:
+ raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr)
+ else:
+ _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+ elif not ec:
+ if log_all:
+ _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+ else:
+ _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+
+ # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
+ if fail_on_error_match or regexp:
+ res = parse_log_for_error(stdouterr, regexp, stdout=False, print_deprecation_warning=False)
+ if res:
+ errors = "\n\t" + "\n\t".join([r[0] for r in res])
+ error_str = "error" if len(res) == 1 else "errors"
+ if fail_on_error_match:
+ raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
+ else:
+ _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
+ len(res), error_str, cmd, errors)
+
+ if simple:
+ if ec:
+ # If the user does not care -> will return true
+ return not check_ec
+ else:
+ return True
+ else:
+ # Because we are not running in simple mode, we return the output and ec to the user
+ return (stdouterr, ec)
+
+
+def parse_log_for_error(txt, regExp=None, stdout=True, msg=None, print_deprecation_warning=True):
+ """
+ txt is multiline string.
+ - in memory
+ regExp is a one-line regular expression
+ - default
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("parse_log_for_error is deprecated, you should stop using it", '6.0')
+
+ global errors_found_in_log
+
+ if regExp and isinstance(regExp, bool):
+ regExp = r"(?= loose_mv.version[:depth]:
self.raiseException("DEPRECATED (since v%s) functionality used: %s" % (max_ver, msg), exception=exception)
else:
- deprecation_msg = "Deprecated functionality, will no longer work in v%s: %s" % (max_ver, msg)
+ deprecation_msg = "Deprecated functionality, will no longer work in EasyBuild v%s: %s" % (max_ver, msg)
log_callback(deprecation_msg)
def _handleFunction(self, function, levelno, **kwargs):
@@ -588,7 +587,7 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
os.makedirs(directory)
except Exception as ex:
exc, detail, tb = sys.exc_info()
- raise_with_traceback(exc, "Cannot create logdirectory %s: %s \n detail: %s" % (directory, ex, detail), tb)
+ raise exc("Cannot create logdirectory %s: %s \n detail: %s" % (directory, ex, detail)).with_traceback(tb)
return _logToSomething(
logging.handlers.RotatingFileHandler,
@@ -741,7 +740,7 @@ def setLogLevel(level):
"""
Set a global log level for all FancyLoggers
"""
- if isinstance(level, string_type):
+ if isinstance(level, str):
level = getLevelInt(level)
logger = getLogger(fname=False, clsname=False)
logger.setLevel(level)
diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py
index 6bfe91a82b..5d0205687e 100644
--- a/easybuild/base/frozendict.py
+++ b/easybuild/base/frozendict.py
@@ -21,10 +21,10 @@
It can be used as a drop-in replacement for dictionaries where immutability is desired.
"""
import operator
+from collections.abc import Mapping
from functools import reduce
from easybuild.base import fancylogger
-from easybuild.tools.py2vs3 import Mapping
# minor adjustments:
diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py
index c3660f7cbf..820033a420 100644
--- a/easybuild/base/generaloption.py
+++ b/easybuild/base/generaloption.py
@@ -1,5 +1,5 @@
#
-# Copyright 2011-2024 Ghent University
+# Copyright 2011-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,6 +31,7 @@
* Jens Timmerman (Ghent University)
"""
+import configparser
import copy
import difflib
import inspect
@@ -39,13 +40,15 @@
import re
import sys
import textwrap
+from configparser import ConfigParser
from functools import reduce
+from io import StringIO
from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values
from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4
from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
from easybuild.base.optcomplete import autocomplete, CompleterOption
-from easybuild.tools.py2vs3 import StringIO, configparser, ConfigParser, string_type, subprocess_popen_text
+from easybuild.tools.run import subprocess_popen_text
from easybuild.tools.utilities import mk_md_table, mk_rst_table, nub, shell_quote
try:
@@ -138,7 +141,7 @@ def get_empty_add_flex(allvalues, self=None):
empty = None
if isinstance(allvalues, (list, tuple)):
- if isinstance(allvalues[0], string_type):
+ if isinstance(allvalues[0], str):
empty = ''
if empty is None:
@@ -471,7 +474,7 @@ def is_value_a_commandline_option(self, opt, value, index=None):
# --longopt=value, so no issues there either.
# following checks assume that value is a string (not a store_or_None)
- if not isinstance(value, string_type):
+ if not isinstance(value, str):
return None
cmdline_index = None
@@ -1198,7 +1201,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non
# choices
nameds['choices'] = [str(x) for x in extra_detail] # force to strings
hlp += ' (choices: %s)' % ', '.join(nameds['choices'])
- elif isinstance(extra_detail, string_type) and len(extra_detail) == 1:
+ elif isinstance(extra_detail, str) and len(extra_detail) == 1:
args.insert(0, "-%s" % extra_detail)
elif isinstance(extra_detail, (dict,)):
# extract any optcomplete completer hints
@@ -1715,7 +1718,7 @@ class SimpleOption(GeneralOption):
PARSER = SimpleOptionParser
SETROOTLOGGER = True
- def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None):
+ def __init__(self, go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None):
"""Initialisation
:param go_dict: General Option option dict
:param short_groupdescr: short description of main options
@@ -1744,18 +1747,13 @@ def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupde
super(SimpleOption, self).__init__(**kwargs)
- if descr is not None:
- # TODO: as there is no easy/clean way to access the version of the vsc-base package,
- # this is equivalent to a warning
- self.log.deprecated('SimpleOption descr argument', '2.5.0', '3.0.0')
-
def main_options(self):
if self.go_dict is not None:
prefix = None
self.add_group_parser(self.go_dict, self.descr, prefix=prefix)
-def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None):
+def simple_option(go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None):
"""A function that returns a single level GeneralOption option parser
:param go_dict: General Option option dict
@@ -1769,5 +1767,5 @@ def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdes
the generated help will include the docstring
"""
- return SimpleOption(go_dict=go_dict, descr=descr, short_groupdescr=short_groupdescr,
- long_groupdescr=long_groupdescr, config_files=config_files)
+ return SimpleOption(go_dict=go_dict, short_groupdescr=short_groupdescr, long_groupdescr=long_groupdescr,
+ config_files=config_files)
diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py
index 3fa75a6635..ba0f075cf2 100644
--- a/easybuild/base/optcomplete.py
+++ b/easybuild/base/optcomplete.py
@@ -107,7 +107,7 @@
from optparse import OptionParser, Option
from pprint import pformat
-from easybuild.tools.py2vs3 import string_type
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.utilities import shell_quote
debugfn = None # for debugging only
@@ -211,7 +211,7 @@ class FileCompleter(Completer):
CALL_ARGS_OPTIONAL = ['prefix']
def __init__(self, endings=None):
- if isinstance(endings, string_type):
+ if isinstance(endings, str):
endings = [endings]
elif endings is None:
endings = []
@@ -282,11 +282,11 @@ class RegexCompleter(Completer):
def __init__(self, regexlist, always_dirs=True):
self.always_dirs = always_dirs
- if isinstance(regexlist, string_type):
+ if isinstance(regexlist, str):
regexlist = [regexlist]
self.regexlist = []
for regex in regexlist:
- if isinstance(regex, string_type):
+ if isinstance(regex, str):
regex = re.compile(regex)
self.regexlist.append(regex)
@@ -538,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
# Note: this will get filtered properly below.
completer_kwargs = {
- 'pwd': os.getcwd(),
+ 'pwd': get_cwd(),
'cline': cline,
'cpoint': cpoint,
'prefix': prefix,
@@ -547,7 +547,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
# File completion.
if completer and (not prefix or not prefix.startswith('-')):
# Call appropriate completer depending on type.
- if isinstance(completer, (string_type, list, tuple)):
+ if isinstance(completer, (str, list, tuple)):
completer = FileCompleter(completer)
elif not isinstance(completer, (types.FunctionType, types.LambdaType, types.ClassType, types.ObjectType)):
# TODO: what to do here?
@@ -555,7 +555,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
completions = completer(**completer_kwargs)
- if isinstance(completions, string_type):
+ if isinstance(completions, str):
# is a bash command, just run it
if SHELL in (BASH,): # TODO: zsh
print(completions)
diff --git a/easybuild/base/rest.py b/easybuild/base/rest.py
index ed4c187436..4d97eb6183 100644
--- a/easybuild/base/rest.py
+++ b/easybuild/base/rest.py
@@ -40,9 +40,10 @@
import copy
import json
from functools import partial
+from urllib.parse import urlencode
+from urllib.request import HTTPSHandler, Request, build_opener
from easybuild.base import fancylogger
-from easybuild.tools.py2vs3 import HTTPSHandler, Request, build_opener, json_loads, string_type, urlencode
class Client(object):
@@ -180,7 +181,7 @@ def request(self, method, url, body, headers, content_type=None):
else:
body = conn.read()
try:
- pybody = json_loads(body)
+ pybody = json.loads(body)
except ValueError:
pybody = body
fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
@@ -203,10 +204,8 @@ def get_connection(self, method, url, body, headers):
else:
sep = ''
- # value passed to 'data' must be a 'bytes' value (not 'str') in Python 3.x, but a string value in Python 2
- # hence, we encode the value obtained (if needed)
- # this doesn't affect the value type in Python 2, and makes it a 'bytes' value in Python 3
- if isinstance(body, string_type):
+ # value passed to 'data' must be a 'bytes' value (not 'str') hence, we encode the value obtained (if needed)
+ if isinstance(body, str):
body = body.encode('utf-8')
request = Request(self.url + sep + url, data=body)
diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py
index 4b3df7e563..8e6f635b12 100644
--- a/easybuild/base/testing.py
+++ b/easybuild/base/testing.py
@@ -1,5 +1,5 @@
#
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,16 +39,10 @@
import re
import sys
from contextlib import contextmanager
-
-try:
- from cStringIO import StringIO # Python 2
-except ImportError:
- from io import StringIO # Python 3
+from io import StringIO
from unittest import TestCase as OrigTestCase
-from easybuild.tools.py2vs3 import string_type
-
def nicediff(txta, txtb, offset=5):
"""
@@ -61,15 +55,12 @@ def nicediff(txta, txtb, offset=5):
"""
diff = list(difflib.ndiff(txta.splitlines(1), txtb.splitlines(1)))
different_idx = [idx for idx, line in enumerate(diff) if not line.startswith(' ')]
- res_idx = []
+ res_idx = set()
# very bruteforce
for didx in different_idx:
- for idx in range(max(didx - offset, 0), min(didx + offset, len(diff) - 1)):
- if idx not in res_idx:
- res_idx.append(idx)
- res_idx.sort()
+ res_idx.update(range(max(didx - offset, 0), min(didx + offset, len(diff))))
# insert linenumbers too? what are the linenumbers in ndiff?
- newdiff = [diff[idx] for idx in res_idx]
+ newdiff = [diff[idx] for idx in sorted(res_idx)]
return newdiff
@@ -82,30 +73,34 @@ class TestCase(OrigTestCase):
ASSERT_MAX_DIFF = 100
DIFF_OFFSET = 5 # lines of text around changes
- def is_string(self, x):
- """test if the variable x is a string)"""
- try:
- return isinstance(x, string_type)
- except NameError:
- return isinstance(x, str)
+ def _is_diffable(self, x):
+ """Test if it makes sense to show a diff for x"""
+ if isinstance(x, (int, float, bool, type(None))):
+ return False
+ if isinstance(x, str) and '\n' not in x:
+ return False
+ return True
- # pylint: disable=arguments-differ
+ # pylint: disable=arguments-differ,arguments-renamed
def assertEqual(self, a, b, msg=None):
"""Make assertEqual always print useful messages"""
try:
super(TestCase, self).assertEqual(a, b)
except AssertionError as e:
+ if not self._is_diffable(a) or not self._is_diffable(b):
+ raise
+
if msg is None:
msg = str(e)
else:
- msg = "%s: %s" % (msg, e)
+ msg = "%s: %s" % (msg, str(e))
- if self.is_string(a):
+ if isinstance(a, str):
txta = a
else:
txta = pprint.pformat(a)
- if self.is_string(b):
+ if isinstance(b, str):
txtb = b
else:
txtb = pprint.pformat(b)
@@ -116,7 +111,7 @@ def assertEqual(self, a, b, msg=None):
else:
limit = ''
- raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF])))
+ raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF]))) from None
def assertExists(self, path, msg=None):
"""Assert the given path exists"""
@@ -130,6 +125,11 @@ def assertNotExists(self, path, msg=None):
msg = "'%s' should not exist" % path
self.assertFalse(os.path.exists(path), msg)
+ def assertAllExist(self, paths, msg=None):
+ """Assert that all paths in the given list exist"""
+ for path in paths:
+ self.assertExists(path, msg)
+
def setUp(self):
"""Prepare test case."""
super(TestCase, self).setUp()
@@ -173,7 +173,7 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs):
self.fail("Expected errors with %s(%s) call should occur" % (call.__name__, str_args))
except error as err:
msg = self.convert_exception_to_str(err)
- if self.is_string(regex):
+ if isinstance(regex, str):
regex = re.compile(regex)
self.assertTrue(regex.search(msg), "Pattern '%s' is found in '%s'" % (regex.pattern, msg))
diff --git a/easybuild/base/wrapper.py b/easybuild/base/wrapper.py
index e73c8d22c5..bc44d9c494 100644
--- a/easybuild/base/wrapper.py
+++ b/easybuild/base/wrapper.py
@@ -6,7 +6,24 @@
Original code by http://stackoverflow.com/users/416467/kindall from answer 4 of
http://stackoverflow.com/questions/9057669/how-can-i-intercept-calls-to-pythons-magic-methods-in-new-style-classes
"""
-from easybuild.tools.py2vs3 import mk_wrapper_baseclass
+
+
+# based on six's 'with_metaclass' function
+# see also https://stackoverflow.com/questions/18513821/python-metaclass-understanding-the-with-metaclass
+def create_base_metaclass(base_class_name, metaclass, *bases):
+ """Create new class with specified metaclass based on specified base class(es)."""
+ return metaclass(base_class_name, bases, {})
+
+
+def mk_wrapper_baseclass(metaclass):
+
+ class WrapperBase(object, metaclass=metaclass):
+ """
+ Wrapper class that provides proxy access to an instance of some internal instance.
+ """
+ __wraps__ = None
+
+ return WrapperBase
class WrapperMeta(type):
diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py
index f03298abca..56ffd37de2 100644
--- a/easybuild/framework/__init__.py
+++ b/easybuild/framework/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 0b76f5bff2..4d9b4fbdd7 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,19 +40,25 @@
* Maxime Boissonneault (Compute Canada)
* Davide Vanzo (Vanderbilt University)
* Caspar van Leeuwen (SURF)
+* Jan Andre Reuter (Juelich Supercomputing Centre)
"""
-
+import concurrent
import copy
import glob
import inspect
import json
import os
+import random
import re
import stat
+import sys
import tempfile
import time
import traceback
+from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
+from string import ascii_letters
+from textwrap import indent
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
@@ -66,39 +72,43 @@
from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
from easybuild.framework.extension import Extension, resolve_exts_filter_template
-from easybuild.tools import LooseVersion, config, run
+from easybuild.tools import LooseVersion, config
from easybuild.tools.build_details import get_build_stats
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs
from easybuild.tools.build_log import print_error, print_msg, print_warning
from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
+from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa
from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
-from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
-from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
+from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
+from easybuild.tools.config import build_option, build_path, get_failed_install_build_dirs_path
+from easybuild.tools.config import get_failed_install_logs_path, get_log_filename, get_repository, get_repositorypath
from easybuild.tools.config import install_path, log_path, package_path, source_paths
from easybuild.tools.environment import restore_env, sanitize_env
-from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256
from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
-from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
-from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
-from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
-from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP
-from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP
-from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook
-from easybuild.tools.run import check_async_cmd, run_cmd
+from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock
+from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files
+from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file
+from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs
+from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum
+from easybuild.tools.filetools import weld_paths, write_file
+from easybuild.tools.hooks import (
+ BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
+ MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
+ SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
+)
+from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
from easybuild.tools.jenkins import write_to_xml
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
-from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
+from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment, MODULE_LOAD_ENV_HEADERS
+from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root
from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar
from easybuild.tools.package.utilities import package
-from easybuild.tools.py2vs3 import extract_method_name, string_type
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group
@@ -106,7 +116,7 @@
from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
-DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
+DEFAULT_BIN_LIB_SUBDIRS = SEARCH_PATH_BIN_DIRS + SEARCH_PATH_LIB_DIRS
MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -141,14 +151,15 @@ def extra_options(extra=None):
#
# INIT
#
- def __init__(self, ec):
+ def __init__(self, ec, logfile=None):
"""
Initialize the EasyBlock instance.
:param ec: a parsed easyconfig file (EasyConfig instance)
+ :param logfile: pass logfile from other EasyBlock. If not passed, create logfile (optional)
"""
# keep track of original working directory, so we can go back there
- self.orig_workdir = os.getcwd()
+ self.orig_workdir = get_cwd()
# dict of all hooks (mapping of name to function)
self.hooks = load_hooks(build_option('hooks'))
@@ -200,6 +211,21 @@ def __init__(self, ec):
if modules_header_path is not None:
self.modules_header = read_file(modules_header_path)
+ # environment variables on module load
+ mod_load_aliases = {}
+ # apply --module-search-path-headers: easyconfig parameter has precedence
+ mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers')
+
+ try:
+ mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers]
+ except KeyError as err:
+ raise EasyBuildError(
+ f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. "
+ f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}"
+ ) from err
+
+ self.module_load_environment = ModuleLoadEnvironment(aliases=mod_load_aliases)
+
# determine install subdirectory, based on module name
self.install_subdir = None
@@ -214,7 +240,8 @@ def __init__(self, ec):
# logging
self.log = None
- self.logfile = None
+ self.logfile = logfile
+ self.external_logfile = logfile is not None
self.logdebug = build_option('debug')
self.postmsg = '' # allow a post message to be set, which can be shown as last output
self.current_step = None
@@ -294,6 +321,7 @@ def post_init(self):
# but needs to be correct if the build is performed in the installation directory
self.log.info("Changing build dir to %s", self.installdir)
self.builddir = self.installdir
+ self.set_parallel()
# INIT/CLOSE LOG
def _init_log(self):
@@ -303,11 +331,11 @@ def _init_log(self):
if self.log is not None:
return
- self.logfile = get_log_filename(self.name, self.version, add_salt=True)
- fancylogger.logToFile(self.logfile, max_bytes=0)
+ if self.logfile is None:
+ self.logfile = get_log_filename(self.name, self.version, add_salt=True)
+ fancylogger.logToFile(self.logfile, max_bytes=0)
self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False)
-
self.log.info(this_is_easybuild())
this_module = inspect.getmodule(self)
@@ -323,6 +351,9 @@ def close_log(self):
"""
Shutdown the logger.
"""
+ # only close log if we created a logfile
+ if self.external_logfile:
+ return
self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
fancylogger.logToFile(self.logfile, enable=False)
@@ -357,34 +388,57 @@ def get_checksum_for(self, checksums, filename=None, index=None):
:param filename: name of the file to obtain checksum for
:param index: index of file in list
"""
- checksum = None
-
- # sometimes, filename are specified as a dict
+ chksum_input = filename
+ chksum_input_git = None
+ # if filename is provided as dict, take 'filename' key
if isinstance(filename, dict):
- filename = filename['filename']
+ chksum_input = filename.get('filename', None)
+ chksum_input_git = filename.get('git_config', None)
+ # early return if no filename given
+ if chksum_input is None:
+ self.log.debug("Cannot get checksum without a file name")
+ return None
+ checksum = None
# if checksums are provided as a dict, lookup by source filename as key
if isinstance(checksums, dict):
- if filename is not None and filename in checksums:
- checksum = checksums[filename]
- else:
- checksum = None
- elif isinstance(checksums, (list, tuple)):
- if index is not None and index < len(checksums) and (index >= 0 or abs(index) <= len(checksums)):
+ try:
+ checksum = checksums[chksum_input]
+ except KeyError:
+ self.log.debug("Checksum not found for file: %s", chksum_input)
+ elif isinstance(checksums, (list, tuple)) and index is not None:
+ try:
checksum = checksums[index]
- else:
- checksum = None
- elif checksums is None:
- checksum = None
- else:
+ except IndexError:
+ self.log.debug("Checksum not found for index list: %s", index)
+ elif checksums is not None:
raise EasyBuildError("Invalid type for checksums (%s), should be dict, list, tuple or None.",
type(checksums))
if checksum is None or build_option("checksum_priority") == CHECKSUM_PRIORITY_JSON:
json_checksums = self.get_checksums_from_json()
- return json_checksums.get(filename, None)
- else:
- return checksum
+ checksum = json_checksums.get(chksum_input, None)
+
+ if checksum and chksum_input_git is not None:
+ # ignore any checksum for given filename due to changes in https://github.com/python/cpython/issues/90021
+ # tarballs made for git repos are not reproducible when created with Python < 3.9
+ if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+ self.log.deprecated(
+ "Reproducible tarballs of Git repos are only possible when using Python 3.9+ to run EasyBuild. "
+ f"Skipping checksum verification of {chksum_input} since Python < 3.9 is used.",
+ '6.0'
+ )
+ return None
+ # not all archives formats of git repos are reproducible
+ # warn users that checksum might fail for non-reproducible archives
+ _, file_ext = os.path.splitext(chksum_input)
+ if file_ext not in ['', '.tar', '.txz', '.xz']:
+ print_warning(
+ f"Checksum verification may fail! Archive file '{chksum_input}' contains sources of a git repo "
+ "in a non-reproducible format. Please re-create that archive with XZ compression instead."
+ )
+
+ return checksum
def get_checksums_from_json(self, always_read=False):
"""
@@ -418,7 +472,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
if source is None:
raise EasyBuildError("fetch_source called with empty 'source' argument")
- elif isinstance(source, string_type):
+ elif isinstance(source, str):
filename = source
elif isinstance(source, dict):
# Making a copy to avoid modifying the object with pops
@@ -476,7 +530,7 @@ def fetch_sources(self, sources=None, checksums=None):
# Single source should be re-wrapped as a list, and checksums with it
if isinstance(sources, dict):
sources = [sources]
- if isinstance(checksums, string_type):
+ if isinstance(checksums, str):
checksums = [checksums]
# Loop over the list of sources; list of checksums must match >= in size
@@ -532,14 +586,6 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
else:
self.log.info("Added patches: %s", self.patches)
- def fetch_extension_sources(self, skip_checksums=False):
- """
- Fetch source and patch files for extensions (DEPRECATED, use collect_exts_file_info instead).
- """
- depr_msg = "EasyBlock.fetch_extension_sources is deprecated, use EasyBlock.collect_exts_file_info instead"
- self.log.deprecated(depr_msg, '5.0')
- return self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums)
-
def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
"""
Collect information on source and patch files for extensions.
@@ -597,19 +643,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
template_values = copy.deepcopy(self.cfg.template_values)
template_values.update(template_constant_dict(ext_src))
- # resolve templates in extension options
- ext_options = resolve_template(ext_options, template_values)
+ source_urls = resolve_template(ext_options.get('source_urls', []), template_values)
+ checksums = resolve_template(ext_options.get('checksums', []), template_values)
- source_urls = ext_options.get('source_urls', [])
- checksums = ext_options.get('checksums', [])
-
- download_instructions = ext_options.get('download_instructions')
+ download_instructions = resolve_template(ext_options.get('download_instructions'), template_values)
if ext_options.get('nosource', None):
self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name)
elif ext_options.get('sources', None):
- sources = ext_options['sources']
+ sources = resolve_template(ext_options['sources'], template_values)
# only a single source file is supported for extensions currently,
# see https://github.com/easybuilders/easybuild-framework/issues/3463
@@ -624,7 +667,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
# always pass source spec as dict value to fetch_source method,
# mostly so we can inject stuff like source URLs
- if isinstance(source, string_type):
+ if isinstance(source, str):
source = {'filename': source}
elif not isinstance(source, dict):
raise EasyBuildError("Incorrect value type for source of extension %s: %s",
@@ -646,17 +689,18 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
})
else:
- # use default template for name of source file if none is specified
- default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
# if no sources are specified via 'sources', fall back to 'source_tmpl'
src_fn = ext_options.get('source_tmpl')
if src_fn is None:
- src_fn = default_source_tmpl
- elif not isinstance(src_fn, string_type):
+ # use default template for name of source file if none is specified
+ src_fn = '%(name)s-%(version)s.tar.gz'
+ elif not isinstance(src_fn, str):
error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
+ src_fn = resolve_template(src_fn, template_values)
+
if fetch_files:
src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
force_download=force_download,
@@ -671,9 +715,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
src_path = ext_src['src']
src_fn = os.path.basename(src_path)
- # report both MD5 and SHA256 checksums, since both are valid default checksum types
src_checksums = {}
- for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
src_checksums[checksum_type] = src_checksum
self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
@@ -686,10 +729,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % src_fn)
else:
- raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
+ raise EasyBuildError(
+ 'Checksum verification for extension source %s failed', src_fn,
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
# locate extension patches (if any), and verify checksums
- ext_patches = ext_options.get('patches', [])
+ ext_patches = resolve_template(ext_options.get('patches', []), template_values)
if fetch_files:
ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True)
else:
@@ -704,9 +750,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
for patch in ext_patches:
patch = patch['path']
computed_checksums[patch] = {}
- # report both MD5 and SHA256 checksums,
- # since both are valid default checksum types
- for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
checksum = compute_checksum(patch, checksum_type=checksum_type)
computed_checksums[patch][checksum_type] = checksum
self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
@@ -723,14 +767,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % patch_fn)
else:
- raise EasyBuildError("Checksum verification for extension patch %s failed",
- patch_fn)
+ raise EasyBuildError(
+ "Checksum verification for extension patch %s failed", patch_fn,
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
else:
self.log.debug('No patches found for extension %s.' % ext_name)
exts_sources.append(ext_src)
- elif isinstance(ext, string_type):
+ elif isinstance(ext, str):
exts_sources.append({'name': ext})
else:
@@ -795,12 +841,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
return fullpath
except IOError as err:
+ msg = f"Downloading file {filename} from url {url} to {fullpath} failed: {err}"
if not warning_only:
- raise EasyBuildError("Downloading file %s "
- "from url %s to %s failed: %s", filename, url, fullpath, err)
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_DOWNLOAD)
else:
- self.log.warning("Downloading file %s "
- "from url %s to %s failed: %s", filename, url, fullpath, err)
+ self.log.warning(msg)
return None
else:
@@ -873,116 +918,124 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
if self.dry_run:
self.dry_run_msg(" * %s found at %s", filename, foundfile)
return foundfile
- elif no_download:
+
+ if no_download:
if self.dry_run:
self.dry_run_msg(" * %s (MISSING)", filename)
return filename
- else:
- if not warning_only:
- raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
- "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
- else:
- self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
- "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
- return None
- elif git_config:
- return get_source_tarball_from_git(filename, targetdir, git_config)
- else:
- # try and download source files from specified source URLs
- if urls:
- source_urls = urls[:]
- else:
- source_urls = []
- source_urls.extend(self.cfg['source_urls'])
- # Add additional URLs as configured.
- for url in build_option("extra_source_urls"):
- url += "/" + name_letter + "/" + location
- source_urls.append(url)
+ failedpaths_msg = "\n * ".join([""]+failedpaths)
+ file_notfound_msg = (
+ f"Couldn't find file '{filename}' anywhere, and downloading it is disabled... "
+ f"Paths attempted (in order): {failedpaths_msg}"
+ )
- mkdir(targetdir, parents=True)
+ if warning_only:
+ self.log.warning(file_notfound_msg)
+ return None
- for url in source_urls:
+ raise EasyBuildError(file_notfound_msg, exit_code=EasyBuildExit.MISSING_SOURCES)
- if extension:
- targetpath = os.path.join(targetdir, "extensions", filename)
- else:
- targetpath = os.path.join(targetdir, filename)
+ if git_config:
+ return get_source_tarball_from_git(filename, targetdir, git_config)
- url_filename = download_filename or filename
+ # try and download source files from specified source URLs
+ if urls:
+ source_urls = urls[:]
+ else:
+ source_urls = []
+ source_urls.extend(self.cfg['source_urls'])
- if isinstance(url, string_type):
- if url[-1] in ['=', '/']:
- fullurl = "%s%s" % (url, url_filename)
- else:
- fullurl = "%s/%s" % (url, url_filename)
- elif isinstance(url, tuple):
- # URLs that require a suffix, e.g., SourceForge download links
- # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download
- fullurl = "%s/%s/%s" % (url[0], url_filename, url[1])
- else:
- self.log.warning("Source URL %s is of unknown type, so ignoring it." % url)
- continue
+ # Add additional URLs as configured.
+ for url in build_option("extra_source_urls"):
+ url += "/" + name_letter + "/" + location
+ source_urls.append(url)
- # PyPI URLs may need to be converted due to change in format of these URLs,
- # cfr. https://bitbucket.org/pypa/pypi/issues/438
- if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl):
- alt_url = derive_alt_pypi_url(fullurl)
- if alt_url:
- _log.debug("Using alternate PyPI URL for %s: %s", fullurl, alt_url)
- fullurl = alt_url
- else:
- _log.debug("Failed to derive alternate PyPI URL for %s, so retaining the original", fullurl)
+ mkdir(targetdir, parents=True)
- if self.dry_run:
- self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath)
- if extension and urls:
- # extensions typically have custom source URLs specified, only mention first
- self.dry_run_msg(" (from %s, ...)", fullurl)
- downloaded = True
+ for url in source_urls:
- else:
- self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
- downloaded = False
- try:
- if download_file(filename, fullurl, targetpath):
- downloaded = True
+ if extension:
+ targetpath = os.path.join(targetdir, "extensions", filename)
+ else:
+ targetpath = os.path.join(targetdir, filename)
- except IOError as err:
- self.log.debug("Failed to download %s from %s: %s" % (filename, url, err))
- failedpaths.append(fullurl)
- continue
+ url_filename = download_filename or filename
- if downloaded:
- # if fetching from source URL worked, we're done
- self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl))
- return targetpath
+ if isinstance(url, str):
+ if url[-1] in ['=', '/']:
+ fullurl = "%s%s" % (url, url_filename)
else:
- failedpaths.append(fullurl)
+ fullurl = "%s/%s" % (url, url_filename)
+ elif isinstance(url, tuple):
+ # URLs that require a suffix, e.g., SourceForge download links
+ # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download
+ fullurl = "%s/%s/%s" % (url[0], url_filename, url[1])
+ else:
+ self.log.warning("Source URL %s is of unknown type, so ignoring it." % url)
+ continue
+
+ # PyPI URLs may need to be converted due to change in format of these URLs,
+ # cfr. https://bitbucket.org/pypa/pypi/issues/438
+ if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl):
+ alt_url = derive_alt_pypi_url(fullurl)
+ if alt_url:
+ _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url)
+ fullurl = alt_url
+ else:
+ _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original",
+ fullurl)
if self.dry_run:
- self.dry_run_msg(" * %s (MISSING)", filename)
- return filename
+ self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath)
+ if extension and urls:
+ # extensions typically have custom source URLs specified, only mention first
+ self.dry_run_msg(" (from %s, ...)", fullurl)
+ downloaded = True
+
else:
- error_msg = "Couldn't find file %s anywhere, "
- if download_instructions is None:
- download_instructions = self.cfg['download_instructions']
- if download_instructions is not None and download_instructions != "":
- msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
- print_msg(msg, prefix=False, stderr=True)
- error_msg += "please follow the download instructions above, and make the file available "
- error_msg += "in the active source path (%s)" % ':'.join(source_paths())
- else:
- # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
- failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
- error_msg += "and downloading it didn't work either... "
- error_msg += "Paths attempted (in order): %s " % failedpaths_msg
+ self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
+ downloaded = False
+ try:
+ if download_file(filename, fullurl, targetpath):
+ downloaded = True
- if not warning_only:
- raise EasyBuildError(error_msg, filename)
- else:
- self.log.warning(error_msg, filename)
- return None
+ except IOError as err:
+ self.log.debug("Failed to download %s from %s: %s" % (filename, url, err))
+ failedpaths.append(fullurl)
+ continue
+
+ if downloaded:
+ # if fetching from source URL worked, we're done
+ self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl))
+ return targetpath
+ else:
+ failedpaths.append(fullurl)
+
+ if self.dry_run:
+ self.dry_run_msg(" * %s (MISSING)", filename)
+ return filename
+ else:
+ error_msg = "Couldn't find file %s anywhere, "
+ if download_instructions is None:
+ download_instructions = self.cfg['download_instructions']
+ if download_instructions is not None and download_instructions != "":
+ msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n'
+ msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
+ print_msg(msg, prefix=False, stderr=True)
+ error_msg += "please follow the download instructions above, and make the file available "
+ error_msg += "in the active source path (%s)" % ':'.join(source_paths())
+ else:
+ # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
+ failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
+ error_msg += "and downloading it didn't work either... "
+ error_msg += "Paths attempted (in order): %s " % failedpaths_msg
+
+ if not warning_only:
+ raise EasyBuildError(error_msg, filename, exit_code=EasyBuildExit.FAIL_DOWNLOAD)
+ else:
+ self.log.warning(error_msg, filename)
+ return None
#
# GETTER/SETTER UTILITY FUNCTIONS
@@ -1217,6 +1270,8 @@ def make_devel_module(self, create_in_builddir=False):
# these should be all the dependencies and we should load them
recursive_unload = self.cfg['recursive_module_unload']
depends_on = self.cfg['module_depends_on']
+ if depends_on is not None:
+ self.log.deprecated("'module_depends_on' easyconfig parameter should not be used anymore", '6.0')
for key in os.environ:
# legacy support
if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX):
@@ -1344,6 +1399,8 @@ def make_module_dep(self, unload_info=None):
# include load statements for retained dependencies
recursive_unload = self.cfg['recursive_module_unload']
depends_on = self.cfg['module_depends_on']
+ if depends_on is not None:
+ self.log.deprecated("'module_depends_on' easyconfig parameter should not be used anymore", '6.0')
loads = []
for dep in deps:
unload_modules = []
@@ -1368,8 +1425,7 @@ def make_module_dep(self, unload_info=None):
multi_dep_mod_names = {}
for deplist in self.cfg.multi_deps:
for dep in deplist:
- multi_dep_mod_names.setdefault(dep['name'], [])
- multi_dep_mod_names[dep['name']].append(dep['short_mod_name'])
+ multi_dep_mod_names.setdefault(dep['name'], []).append(dep['short_mod_name'])
multi_dep_load_defaults = []
for _, depmods in sorted(multi_dep_mod_names.items()):
@@ -1388,6 +1444,46 @@ def make_module_description(self):
"""
return self.module_generator.get_description()
+ def make_module_pythonpath(self):
+ """
+ Add lines for module file to update $PYTHONPATH or $EBPYTHONPREFIXES,
+ if they aren't already present and the standard lib/python*/site-packages subdirectory exists
+ """
+ if os.path.isfile(os.path.join(self.installdir, 'bin', 'python')): # only needed when not a python install
+ return []
+
+ python_subdir_pattern = os.path.join(self.installdir, 'lib', 'python*', 'site-packages')
+ candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern))
+ python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
+ if not python_paths:
+ return []
+
+ # determine whether Python is a runtime dependency;
+ # if so, we assume it was installed with EasyBuild, and hence is aware of $EBPYTHONPREFIXES
+ runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+
+ # don't use $EBPYTHONPREFIXES unless we can and it's preferred or necesary (due to use of multi_deps)
+ use_ebpythonprefixes = False
+ multi_deps = self.cfg['multi_deps']
+
+ if 'Python' in runtime_deps:
+ self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+
+ if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+ self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+ use_ebpythonprefixes = True
+
+ elif multi_deps and 'Python' in multi_deps:
+ self.log.info("Python is listed in 'multi_deps', so using $EBPYTHONPREFIXES instead of $PYTHONPATH")
+ use_ebpythonprefixes = True
+
+ if use_ebpythonprefixes:
+ path = '' # EBPYTHONPREFIXES are relative to the install dir
+ lines = self.module_generator.prepend_paths(EBPYTHONPREFIXES, path, warn_exists=False)
+ else:
+ lines = self.module_generator.prepend_paths(PYTHONPATH, python_paths, warn_exists=False)
+ return [lines] if lines else []
+
def make_module_extra(self, altroot=None, altversion=None):
"""
Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.
@@ -1420,21 +1516,8 @@ def make_module_extra(self, altroot=None, altversion=None):
for (key, value) in self.cfg['modextravars'].items():
lines.append(self.module_generator.set_environment(key, value))
- for (key, value) in self.cfg['modextrapaths'].items():
- if isinstance(value, string_type):
- value = [value]
- elif not isinstance(value, (tuple, list)):
- raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple",
- value, type(value))
- lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
-
- for (key, value) in self.cfg['modextrapaths_append'].items():
- if isinstance(value, string_type):
- value = [value]
- elif not isinstance(value, (tuple, list)):
- raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple",
- value, type(value))
- lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
+ # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES
+ lines.extend(self.make_module_pythonpath())
modloadmsg = self.cfg['modloadmsg']
if modloadmsg:
@@ -1468,8 +1551,8 @@ def make_module_extra_extensions(self):
# set environment variable that specifies list of extensions
# We need only name and version, so don't resolve templates
exts_list = self.make_extension_string(ext_sep=',', sort=False)
- env_var_name = convert_name(self.name, upper=True)
- lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
+ env_var_name = 'EBEXTSLIST' + convert_name(self.name, upper=True)
+ lines.append(self.module_generator.set_environment(env_var_name, exts_list))
return ''.join(lines)
@@ -1553,108 +1636,186 @@ def make_module_group_check(self):
return txt
- def make_module_req(self):
+ def make_module_req(self, fake=False):
"""
- Generate the environment-variables to run the module.
+ Generate the environment-variables required to run the module.
"""
- requirements = self.make_module_req_guess()
+ if self.make_module_req_guess.__qualname__ != "EasyBlock.make_module_req_guess":
+ # Deprecated make_module_req_guess method used in child Easyblock
+ # adjust environment with custom make_module_req_guess
+ self.log.deprecated(
+ "make_module_req_guess() is deprecated, use EasyBlock.module_load_environment instead.",
+ "6.0",
+ )
+ self.module_load_environment.replace(self.make_module_req_guess())
- lines = ['\n']
- if os.path.isdir(self.installdir):
- old_dir = change_dir(self.installdir)
- else:
- old_dir = None
+ # update module load environment with extra paths defined in easyconfig
+ self.inject_module_extra_paths()
+
+ # Expand and inject path-like environment variables into module file
+ env_var_requirements = {
+ envar_name: envar_val
+ for envar_name, envar_val in sorted(self.module_load_environment.items())
+ if envar_val.is_path
+ }
+ self.log.debug(f"Tentative module environment requirements before path expansion: {env_var_requirements}")
+ # TODO: handle non-path variables in make_module_extra
+ # in the meantime, just report if any is found
+ non_path_envars = set(self.module_load_environment) - set(env_var_requirements)
+ if non_path_envars:
+ self.log.warning(
+ f"Non-path variables found in module load environment: {non_path_envars}."
+ "This is not yet supported by this version of EasyBuild."
+ )
+
+ mod_lines = ['\n']
if self.dry_run:
self.dry_run_msg("List of paths that would be searched and added to module file:\n")
note = "note: glob patterns are not expanded and existence checks "
note += "for paths are skipped for the statements below due to dry run"
- lines.append(self.module_generator.comment(note))
-
- # For these environment variables, the corresponding directory must include at least one file.
- # The values determine if detection is done recursively, i.e. if it accepts directories where files
- # are only in subdirectories.
- keys_requiring_files = {
- 'PATH': False,
- 'LD_LIBRARY_PATH': False,
- 'LIBRARY_PATH': True,
- 'CPATH': True,
- 'CMAKE_PREFIX_PATH': True,
- 'CMAKE_LIBRARY_PATH': True,
- }
+ mod_lines.append(self.module_generator.comment(note))
- for key, reqs in sorted(requirements.items()):
- if isinstance(reqs, string_type):
- self.log.warning("Hoisting string value %s into a list before iterating over it", reqs)
- reqs = [reqs]
- if self.dry_run:
- self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs)))
- # Don't expand globs or do any filtering below for dry run
- paths = reqs
+ for env_var, search_paths in env_var_requirements.items():
+ if self.dry_run or fake:
+ # Don't expand globs or do any filtering for dry run
+ mod_req_paths = search_paths
+ if self.dry_run:
+ self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}")
else:
- # Expand globs but only if the string is non-empty
- # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME)
- paths = sum((glob.glob(path) if path else [path] for path in reqs), []) # sum flattens to list
-
- # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates
- lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) and
- os.path.samefile('lib', 'lib64'))
- if lib64_is_symlink:
- fixed_paths = []
- for path in paths:
- if (path + os.path.sep).startswith('lib64' + os.path.sep):
- # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path, so skip symlink
- if key == 'CMAKE_LIBRARY_PATH':
- continue
- path = path.replace('lib64', 'lib', 1)
- fixed_paths.append(path)
- if fixed_paths != paths:
- self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths)
- paths = fixed_paths
- # remove duplicate paths preserving order
- paths = nub(paths)
- if key in keys_requiring_files:
- # only retain paths that contain at least one file
- recursive = keys_requiring_files[key]
- retained_paths = []
- for pth in paths:
- fullpath = os.path.join(self.installdir, pth)
- if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive):
- retained_paths.append(pth)
- if retained_paths != paths:
- self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
- key, paths, retained_paths)
- paths = retained_paths
-
- if paths:
- lines.append(self.module_generator.prepend_paths(key, paths))
+ mod_req_paths = [
+ expanded_path for unexpanded_path in search_paths
+ for expanded_path in self.expand_module_search_path(unexpanded_path, path_type=search_paths.type)
+ ]
+
+ if mod_req_paths:
+ mod_req_paths = nub(mod_req_paths) # remove duplicates
+ extra_mod_lines = self.module_generator.update_paths(env_var, mod_req_paths, allow_abs=True,
+ prepend=search_paths.mod_prepend,
+ delim=search_paths.delimiter)
+ mod_lines.append(extra_mod_lines)
+
if self.dry_run:
self.dry_run_msg('')
- if old_dir is not None:
- change_dir(old_dir)
+ return ''.join(mod_lines)
+
+ def inject_module_extra_paths(self):
+ """
+ Parse search paths and delimiters defined in 'modextrapaths' easyconfig parameter
+ and inject them into module load environment
+ """
+ ec_param = 'modextrapaths'
+ known_aliases = [MODULE_LOAD_ENV_HEADERS]
+
+ for env_var_name, extra_opts in self.cfg[ec_param].items():
+ # default settings for environment variables
+ env_var_opts = {
+ 'delimiter': os.pathsep, # ':'
+ 'var_type': ModEnvVarType.PATH_WITH_FILES,
+ 'prepend': True,
+ }
+
+ if not isinstance(extra_opts, dict):
+ extra_opts = {'paths': extra_opts}
+
+ env_var_opts.update(extra_opts)
+
+ if 'paths' not in env_var_opts:
+ error_msg = f"'paths' key not set for ${env_var_name} in '{ec_param}' easyconfig parameter"
+ raise EasyBuildError(error_msg)
+
+ # make sure that we have a list of paths, even if there's only one
+ if isinstance(env_var_opts['paths'], str):
+ env_var_opts['paths'] = [env_var_opts['paths']]
+
+ existing_env_vars = []
+ if env_var_name in self.module_load_environment:
+ # update existing variable
+ existing_env_vars.append(env_var_name)
+ elif env_var_name in known_aliases:
+ # update existing alias to variables
+ existing_env_vars = self.module_load_environment.alias_vars(env_var_name)
+
+ for existing_env_var in existing_env_vars:
+ env_var = getattr(self.module_load_environment, existing_env_var)
+ env_var.extend(env_var_opts['paths'])
+ env_var.delimiter = env_var_opts['delimiter']
+ env_var.prepend = env_var_opts['prepend']
+ env_var.type = env_var_opts['var_type']
+ msg = f"Variable ${existing_env_var} from '{ec_param}' extended in module load environment with "
+ msg += f"delimiter='{env_var.delimiter}', prepend='{env_var.prepend}', type='{env_var.type}' "
+ msg += f"and paths='{env_var}'"
+ self.log.debug(msg)
+
+ if not existing_env_vars:
+ # rename 'modextrapaths' options to match ModuleEnvironmentVariable constructor parameters
+ env_var_opts['contents'] = env_var_opts.pop('paths')
+ setattr(self.module_load_environment, env_var_name, env_var_opts)
+ env_var = getattr(self.module_load_environment, env_var_name)
+ msg = f"Variable ${env_var_name} from '{ec_param}' added to module load environment with "
+ msg += f"delimiter='{env_var.delimiter}', prepend='{env_var.prepend}', type='{env_var.type}' "
+ msg += f"and paths='{env_var}'"
+ self.log.debug(msg)
+
+ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES):
+ """
+ Expand given path glob and return list of suitable paths to be used as search paths:
+ - Paths must point to existing files/directories
+ - Relative paths are relative to installation prefix root and are kept relative after expansion
+ - Absolute paths are kept as absolute paths after expansion
+ - Follow symlinks and resolve their paths (avoids duplicate paths through symlinks)
+ - :path_type: ModEnvVarType that controls requirements for population of directories
+ - PATH: no requirements, can be empty
+ - PATH_WITH_FILES: must contain at least one file in them (default)
+ - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory
+ """
+ if os.path.isabs(search_path):
+ abs_glob = search_path
+ else:
+ real_installdir = os.path.realpath(self.installdir)
+ abs_glob = os.path.join(real_installdir, search_path)
+
+ exp_search_paths = glob.glob(abs_glob, recursive=True)
+
+ retained_search_paths = []
+ for abs_path in exp_search_paths:
+ check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES)
+ if os.path.isdir(abs_path) and check_dir_files:
+ # only retain paths to directories that contain at least one file
+ recursive = path_type == ModEnvVarType.PATH_WITH_FILES
+ if not dir_contains_files(abs_path, recursive=recursive):
+ self.log.debug("Discarded search path to empty directory: %s", abs_path)
+ continue
+
+ if os.path.isabs(search_path):
+ retain_path = abs_path
+ else:
+ # recover relative path
+ retain_path = os.path.relpath(os.path.realpath(abs_path), start=real_installdir)
+ if retain_path == '.':
+ retain_path = '' # use empty string to represent root of install dir
- return ''.join(lines)
+ if retain_path.startswith('..' + os.path.sep):
+ raise EasyBuildError(
+ f"Expansion of search path glob pattern '{search_path}' resulted in a relative path "
+ f"pointing outside of install directory: {retain_path}"
+ )
+
+ retained_search_paths.append(retain_path)
+
+ return retained_search_paths
def make_module_req_guess(self):
"""
- A dictionary of possible directories to look for.
- """
- lib_paths = ['lib', 'lib32', 'lib64']
- return {
- 'PATH': ['bin', 'sbin'],
- 'LD_LIBRARY_PATH': lib_paths,
- 'LIBRARY_PATH': lib_paths,
- 'CPATH': ['include'],
- 'MANPATH': ['man', os.path.join('share', 'man')],
- 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']],
- 'ACLOCAL_PATH': [os.path.join('share', 'aclocal')],
- 'CLASSPATH': ['*.jar'],
- 'XDG_DATA_DIRS': ['share'],
- 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths],
- 'CMAKE_PREFIX_PATH': [''],
- 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above
- }
+ A dictionary of common search path variables to be loaded by environment modules
+ Each key contains the list of known directories related to the search path
+ """
+ self.log.deprecated(
+ "make_module_req_guess() is deprecated, use EasyBlock.module_load_environment instead",
+ '6.0',
+ )
+ return self.module_load_environment.as_dict
def load_module(self, mod_paths=None, purge=True, extra_modules=None, verbose=True):
"""
@@ -1748,33 +1909,41 @@ def load_dependency_modules(self):
# EXTENSIONS UTILITY FUNCTIONS
#
- def _make_extension_list(self):
+ def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
+ """
+ Generate a string with a list of extensions returned by make_extension_list.
+
+ The name and version are separated by name_version_sep and each extension is separated by ext_sep.
+ For customization of extensions the make_extension_list method should be used.
+ """
+ exts_list = (name_version_sep.join(ext) for ext in self.make_extension_list())
+ if sort:
+ exts_list = sorted(exts_list, key=str.lower)
+ return ext_sep.join(exts_list)
+
+ def make_extension_list(self):
"""
Return a list of extension names and their versions included in this installation
- Each entry should be a (name, version) tuple or just (name, ) if no version exists
+ Each entry should be a (name, version) tuple or just (name, ) if no version exists.
+ Custom EasyBlocks may override this to add extensions that cannot be found automatically.
"""
# Each extension in exts_list is either a string or a list/tuple with name, version as first entries
# As name can be a templated value we must resolve templates
+ if hasattr(self, '_make_extension_list'):
+ self.log.nosupport("self._make_extension_list is replaced by self.make_extension_list", '5.0')
+ if type(self).make_extension_string != EasyBlock.make_extension_string:
+ self.log.nosupport("self.make_extension_string should not be overridden", '5.0')
+
exts_list = []
for ext in self.cfg.get_ref('exts_list'):
- if isinstance(ext, string_type):
+ if isinstance(ext, str):
exts_list.append((resolve_template(ext, self.cfg.template_values), ))
else:
- exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1]))
+ exts_list.append((resolve_template(ext[0], self.cfg.template_values),
+ resolve_template(ext[1], self.cfg.template_values)))
return exts_list
- def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
- """
- Generate a string with a list of extensions.
-
- The name and version are separated by name_version_sep and each extension is separated by ext_sep
- """
- exts_list = (name_version_sep.join(ext) for ext in self._make_extension_list())
- if sort:
- exts_list = sorted(exts_list, key=str.lower)
- return ext_sep.join(exts_list)
-
def prepare_for_extensions(self):
"""Ran before installing extensions (eg to set templates)"""
@@ -1807,21 +1976,20 @@ def skip_extensions_sequential(self, exts_filter):
exts_cnt = len(self.ext_instances)
- res = []
+ exts = []
for idx, ext_inst in enumerate(self.ext_instances):
cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst)
- (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin,
- regexp=False, trace=False)
- self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out)
- if ec == 0:
- print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log)
+ res = run_shell_cmd(cmd, stdin=stdin, fail_on_error=False, hidden=True)
+ self.log.info(f"exts_filter result for {ext_inst.name}: exit code {res.exit_code}; output: {res.output}")
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ print_msg(f"skipping extension {ext_inst.name}", silent=self.silent, log=self.log)
else:
- self.log.info("Not skipping %s", ext_inst.name)
- res.append(ext_inst)
+ self.log.info(f"Not skipping {ext_inst.name}")
+ exts.append(ext_inst)
- self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt))
+ self.update_exts_progress_bar(f"skipping installed extensions ({idx + 1}/{exts_cnt} checked)")
- self.ext_instances = res
+ self.ext_instances = exts
self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances))
def skip_extensions_parallel(self, exts_filter):
@@ -1829,44 +1997,33 @@ def skip_extensions_parallel(self, exts_filter):
Skip already installed extensions (checking in parallel),
by removing them from list of Extension instances to install (self.ext_instances).
"""
- self.log.experimental("Skipping installed extensions in parallel")
print_msg("skipping installed extensions (in parallel)", log=self.log)
- async_cmd_info_cache = {}
- running_checks_ids = []
installed_exts_ids = []
- exts_queue = list(enumerate(self.ext_instances[:]))
checked_exts_cnt = 0
exts_cnt = len(self.ext_instances)
+ cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances]
- # asynchronously run checks to see whether extensions are already installed
- while exts_queue or running_checks_ids:
+ with ThreadPoolExecutor(max_workers=self.cfg.parallel) as thread_pool:
- # first handle completed checks
- for idx in running_checks_ids[:]:
+ # list of command to run asynchronously
+ async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False,
+ asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)]
+
+ # process result of commands as they have completed running
+ for done_task in concurrent.futures.as_completed(async_cmds):
+ res = done_task.result()
+ idx = res.task_id
ext_name = self.ext_instances[idx].name
- # don't read any output, just check whether command completed
- async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False)
- if async_cmd_info['done']:
- out, ec = async_cmd_info['output'], async_cmd_info['exit_code']
- self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out)
- running_checks_ids.remove(idx)
- if ec == 0:
- print_msg("skipping extension %s" % ext_name, log=self.log)
- installed_exts_ids.append(idx)
-
- checked_exts_cnt += 1
- exts_pbar_label = "skipping installed extensions "
- exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt)
- self.update_exts_progress_bar(exts_pbar_label)
-
- # start additional checks asynchronously
- while exts_queue and len(running_checks_ids) < self.cfg['parallel']:
- idx, ext = exts_queue.pop(0)
- cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
- async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin,
- regexp=False, trace=False, asynchronous=True)
- running_checks_ids.append(idx)
+ self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}")
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ print_msg(f"skipping extension {ext_name}", log=self.log)
+ installed_exts_ids.append(idx)
+
+ checked_exts_cnt += 1
+ exts_pbar_label = "skipping installed extensions "
+ exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt)
+ self.update_exts_progress_bar(exts_pbar_label)
# compose new list of extensions, skip over the ones that are already installed;
# note: original order in extensions list should be preserved!
@@ -1878,7 +2035,15 @@ def skip_extensions_parallel(self, exts_filter):
self.ext_instances = retained_ext_instances
- def install_extensions(self, install=True):
+ def install_extensions(self, *args, **kwargs):
+ """[DEPRECATED] Install extensions."""
+ self.log.deprecated(
+ "EasyBlock.install_extensions() is deprecated, use EasyBlock.install_all_extensions() instead.",
+ '6.0',
+ )
+ self.install_all_extensions(*args, **kwargs)
+
+ def install_all_extensions(self, install=True):
"""
Install extensions.
@@ -1888,13 +2053,12 @@ def install_extensions(self, install=True):
self.log.debug("List of loaded modules: %s", self.modules_tool.list())
if build_option('parallel_extensions_install'):
- self.log.experimental("installing extensions in parallel")
try:
self.install_extensions_parallel(install=install)
except NotImplementedError:
# If parallel extension install is not supported for this type of extension then install sequentially
msg = "Parallel extensions install not supported for %s - using sequential install" % self.name
- self.log.experimental(msg)
+ self.log.info(msg)
self.install_extensions_sequential(install=install)
else:
self.install_extensions_sequential(install=install)
@@ -1910,7 +2074,6 @@ def install_extensions_sequential(self, install=True):
exts_cnt = len(self.ext_instances)
for idx, ext in enumerate(self.ext_instances):
-
self.log.info("Starting extension %s", ext.name)
run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext])
@@ -1942,15 +2105,15 @@ def install_extensions_sequential(self, install=True):
ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
rpath_filter_dirs=self.rpath_filter_dirs)
- # real work
+ # actual installation of the extension
if install:
try:
- ext.prerun()
+ ext.install_extension_substep("pre_install_extension")
with self.module_generator.start_module_creation():
- txt = ext.run()
+ txt = ext.install_extension_substep("install_extension")
if txt:
self.module_extra_extensions += txt
- ext.postrun()
+ ext.install_extension_substep("post_install_extension")
finally:
if not self.dry_run:
ext_duration = datetime.now() - start_time
@@ -1971,6 +2134,8 @@ def install_extensions_parallel(self, install=True):
"""
self.log.info("Installing extensions in parallel...")
+ thread_pool = ThreadPoolExecutor(max_workers=self.cfg.parallel)
+
running_exts = []
installed_ext_names = []
@@ -2007,16 +2172,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
# check for extension installations that have completed
if running_exts:
- self.log.info("Checking for completed extension installations (%d running)...", len(running_exts))
+ self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...")
for ext in running_exts[:]:
- if self.dry_run or ext.async_cmd_check():
- self.log.info("Installation of %s completed!", ext.name)
- ext.postrun()
- running_exts.remove(ext)
- installed_ext_names.append(ext.name)
- update_exts_progress_bar_helper(running_exts, 1)
+ if self.dry_run or ext.async_cmd_task.done():
+ res = ext.async_cmd_task.result()
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ self.log.info(f"Installation of extension {ext.name} completed!")
+ # run post-install method for extension from same working dir as installation of extension
+ cwd = change_dir(res.work_dir)
+ ext.install_extension_substep("post_install_extension")
+ change_dir(cwd)
+ running_exts.remove(ext)
+ installed_ext_names.append(ext.name)
+ update_exts_progress_bar_helper(running_exts, 1)
+ else:
+ raise_run_shell_cmd_error(res)
else:
- self.log.debug("Installation of %s is still running...", ext.name)
+ self.log.debug(f"Installation of extension {ext.name} is still running...")
# try to start as many extension installations as we can, taking into account number of available cores,
# but only consider first 100 extensions still in the queue
@@ -2024,7 +2196,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
for _ in range(max_iter):
- if not (exts_queue and len(running_exts) < self.cfg['parallel']):
+ if not (exts_queue and len(running_exts) < self.cfg.parallel):
break
# check whether extension at top of the queue is ready to install
@@ -2060,20 +2232,33 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
# if some of the required dependencies are not installed yet, requeue this extension
elif pending_deps:
- # make sure all required dependencies are actually going to be installed,
- # to avoid getting stuck in an infinite loop!
+ # check whether all required dependency extensions are actually going to be installed;
+ # if not, we assume that they are provided by dependencies;
missing_deps = [x for x in required_deps if x not in all_ext_names]
if missing_deps:
- raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s",
- ext.name, ', '.join(missing_deps))
- else:
- self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...",
- ext.name, ', '.join(pending_deps))
+ msg = f"Missing required extensions for {ext.name} not found "
+ msg += "in list of extensions being installed, let's assume they are provided by "
+ msg += "dependencies and proceed: " + ', '.join(missing_deps)
+ self.log.info(msg)
+
+ msg = f"Pending dependencies for {ext.name} before taking into account missing dependencies: "
+ self.log.debug(msg + ', '.join(pending_deps))
+ pending_deps = [x for x in pending_deps if x not in missing_deps]
+ msg = f"Pending dependencies for {ext.name} after taking into account missing dependencies: "
+ self.log.debug(msg + ', '.join(pending_deps))
+
+ if pending_deps:
+ msg = f"Required dependencies not installed yet for extension {ext.name} ("
+ msg += ', '.join(pending_deps)
+ msg += "), adding it back to queue..."
+ self.log.info(msg)
# purposely adding extension back in the queue at Nth place rather than at the end,
# since we assume that the required dependencies will be installed soon...
exts_queue.insert(max_iter, ext)
- else:
+ # list of pending dependencies may be empty now after taking into account required extensions
+ # that are not being installed above, so extension may be ready to install
+ if not pending_deps:
tup = (ext.name, ext.version or '')
print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log)
@@ -2082,10 +2267,10 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
rpath_filter_dirs=self.rpath_filter_dirs)
if install:
- ext.prerun()
- ext.run_async()
+ ext.install_extension_substep("pre_install_extension")
+ ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
running_exts.append(ext)
- self.log.info("Started installation of extension %s in the background...", ext.name)
+ self.log.info(f"Started installation of extension {ext.name} in the background...")
update_exts_progress_bar_helper(running_exts, 0)
# print progress info after every iteration (unless that info is already shown via progress bar)
@@ -2098,6 +2283,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..."
print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log)
+ thread_pool.shutdown()
+
#
# MISCELLANEOUS UTILITY FUNCTIONS
#
@@ -2248,19 +2435,36 @@ def set_parallel(self):
"""Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
# set level of parallelism for build
par = build_option('parallel')
- cfg_par = self.cfg['parallel']
- if cfg_par is None:
- self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
- elif par is None:
- par = cfg_par
- self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
+ if par is not None:
+ self.log.debug(f"Desired parallelism specified via 'parallel' build option: {par}")
+
+ # Transitional only in case some easyblocks still set/change cfg['parallel']
+ # Use _parallelLegacy to avoid deprecation warnings
+ par_ec = self.cfg['_parallelLegacy']
+ if par_ec is not None:
+ if par is None:
+ par = par_ec
+ else:
+ par = min(int(par), int(par_ec))
+
+ # --max-parallel specifies global maximum for parallelism
+ max_par_global = int(build_option('max_parallel'))
+ # note: 'max_parallel' and 'maxparallel; are the same easyconfig parameter,
+ # since 'max_parallel' is an alternative name for 'maxparallel'
+ max_par_ec = self.cfg['max_parallel']
+ # take into account that False is a valid value for max_parallel
+ if max_par_ec is False:
+ max_par_ec = 1
+ # if max_parallel is not specified in easyconfig, we take the global value
+ if max_par_ec is None:
+ max_par = max_par_global
+ # take minimum value if both are specified
else:
- par = min(int(par), int(cfg_par))
- self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
+ max_par = min(int(max_par_ec), max_par_global)
- par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
- self.log.info("Setting parallelism: %s" % par)
- self.cfg['parallel'] = par
+ par = det_parallelism(par=par, maxpar=max_par)
+ self.log.info(f"Setting parallelism: {par}")
+ self.cfg.parallel = par
def remove_module_file(self):
"""Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""
@@ -2306,8 +2510,6 @@ def check_readiness_step(self):
"""
Verify if all is ok to start build.
"""
- self.set_parallel()
-
# check whether modules are loaded
loadedmods = self.modules_tool.loaded_modules()
if len(loadedmods) > 0:
@@ -2412,8 +2614,7 @@ def fetch_step(self, skip_checksums=False):
# compute checksums for all source and patch files
if not (skip_checksums or self.dry_run):
for fil in self.src + self.patches:
- # report both MD5 and SHA256 checksums, since both are valid default checksum types
- for checksum_type in [CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256]:
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
fil[checksum_type] = compute_checksum(fil['path'], checksum_type=checksum_type)
self.log.info("%s checksum for %s: %s", checksum_type, fil['path'], fil[checksum_type])
@@ -2472,7 +2673,10 @@ def checksum_step(self):
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % fil['name'])
else:
- raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
+ raise EasyBuildError(
+ "Checksum verification for %s using %s failed.", fil['path'], fil['checksum'],
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
def check_checksums_for(self, ent, sub='', source_cnt=None):
"""
@@ -2481,13 +2685,22 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
ec_fn = os.path.basename(self.cfg.path)
checksum_issues = []
- sources = ent.get('sources', [])
- patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
- checksums = ent.get('checksums', [])
+ # try to get value with templates resolved, but fall back to not resolving templates;
+ # this is better for error reporting that includes names of source files
+ try:
+ sources = ent.get('sources', [])
+ patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
+ checksums = ent.get('checksums', [])
+ except EasyBuildError:
+ if isinstance(ent, EasyConfig):
+ sources = ent.get_ref('sources')
+ patches = ent.get_ref('patches') + ent.get_ref('postinstallpatches')
+ checksums = ent.get_ref('checksums')
+
# Single source should be re-wrapped as a list, and checksums with it
if isinstance(sources, dict):
sources = [sources]
- if isinstance(checksums, string_type):
+ if isinstance(checksums, str):
checksums = [checksums]
if not checksums:
@@ -2562,10 +2775,10 @@ def check_checksums(self):
checksum_issues.extend(self.check_checksums_for(self.cfg))
# also check checksums for extensions
- for ext in self.cfg['exts_list']:
+ for ext in self.cfg.get_ref('exts_list'):
# just skip extensions for which only a name is specified
# those are just there to check for things that are in the "standard library"
- if not isinstance(ext, string_type):
+ if not isinstance(ext, str):
ext_name = ext[0]
# take into account that extension may be a 2-tuple with just name/version
ext_opts = ext[2] if len(ext) == 3 else {}
@@ -2655,7 +2868,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
if build_option('rpath_override_dirs') is not None:
# make sure we have a list
rpath_overrides = build_option('rpath_override_dirs')
- if isinstance(rpath_overrides, string_type):
+ if isinstance(rpath_overrides, str):
rpath_override_dirs = rpath_overrides.split(':')
# Filter out any empty values
rpath_override_dirs = list(filter(None, rpath_override_dirs))
@@ -2741,18 +2954,19 @@ def test_step(self):
"""Run unit tests provided by software (if any)."""
unit_test_cmd = self.cfg['runtest']
if unit_test_cmd:
-
- self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd)
- (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False)
-
- return out
+ self.log.debug(f"Trying to execute {unit_test_cmd} as a command for running unit tests...")
+ res = run_shell_cmd(unit_test_cmd)
+ return res.output
def _test_step(self):
"""Run the test_step and handles failures"""
try:
self.test_step()
- except EasyBuildError as err:
- self.report_test_failure(err)
+ except RunShellCmdError as err:
+ err.print()
+ ec_path = os.path.basename(self.cfg.path)
+ error_msg = f"shell command '{err.cmd_name} ...' failed in test step for {ec_path}"
+ self.report_test_failure(error_msg)
def stage_install_step(self):
"""Install in a stage directory before actual installation."""
@@ -2782,7 +2996,7 @@ def init_ext_instances(self):
# obtain name and module path for default extention class
exts_defaultclass = self.cfg['exts_defaultclass']
- if isinstance(exts_defaultclass, string_type):
+ if isinstance(exts_defaultclass, str):
# proper way: derive module path from specified class name
default_class = exts_defaultclass
default_class_modpath = get_module_path(default_class, generic=True)
@@ -2793,7 +3007,6 @@ def init_ext_instances(self):
exts_cnt = len(self.exts)
self.update_exts_progress_bar("creating internal datastructures for extensions")
-
for idx, ext in enumerate(self.exts):
ext_name = ext['name']
self.log.debug("Creating class instance for extension %s...", ext_name)
@@ -2886,7 +3099,7 @@ def extensions_step(self, fetch=False, install=True):
fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods)
- start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list']))
+ start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg.get_ref('exts_list')))
self.prepare_for_extensions()
@@ -2910,7 +3123,7 @@ def extensions_step(self, fetch=False, install=True):
if self.skip:
self.skip_extensions()
- self.install_extensions(install=install)
+ self.install_all_extensions(install=install)
# cleanup (unload fake module, remove fake module dir)
if fake_mod_data:
@@ -2959,7 +3172,7 @@ def fix_shebang(self):
shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang)
fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang]
if fix_shebang_for:
- if isinstance(fix_shebang_for, string_type):
+ if isinstance(fix_shebang_for, str):
fix_shebang_for = [fix_shebang_for]
shebang = '#!%s %s' % (env_for_shebang, lang)
@@ -3001,17 +3214,17 @@ def run_post_install_commands(self, commands=None):
commands = self.cfg['postinstallcmds']
if commands:
- self.log.debug("Specified post install commands: %s", commands)
+ self.log.debug(f"Specified post install commands: {commands}")
# make sure we have a list of commands
if not isinstance(commands, (list, tuple)):
- error_msg = "Invalid value for 'postinstallcmds', should be list or tuple of strings: %s"
- raise EasyBuildError(error_msg, commands)
+ error_msg = f"Invalid value for 'postinstallcmds', should be list or tuple of strings: {commands}"
+ raise EasyBuildError(error_msg)
for cmd in commands:
- if not isinstance(cmd, string_type):
- raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd)
- run_cmd(cmd, simple=True, log_ok=True, log_all=True)
+ if not isinstance(cmd, str):
+ raise EasyBuildError(f"Invalid element in 'postinstallcmds', not a string: {cmd}")
+ run_shell_cmd(cmd)
def apply_post_install_patches(self, patches=None):
"""
@@ -3036,15 +3249,12 @@ def print_post_install_messages(self):
def post_install_step(self):
"""
- Do some postprocessing
+ [DEPRECATED] Do some postprocessing
- run post install commands if any were specified
"""
-
- self.run_post_install_commands()
- self.apply_post_install_patches()
- self.print_post_install_messages()
-
- self.fix_shebang()
+ # even though post_install_step is deprecated in easyblocks we need to keep this here until it is
+ # removed in 6.0 for easyblocks calling super(EB_xxx, self).post_install_step()
+ # The deprecation warning for those is below, in post_processing_step().
lib_dir = os.path.join(self.installdir, 'lib')
lib64_dir = os.path.join(self.installdir, 'lib64')
@@ -3053,18 +3263,44 @@ def post_install_step(self):
# However for each in $LIBRARY_PATH (where is often /lib) it searches /../lib64 first.
# So we create /lib64 as a symlink to /lib to make it prefer EB installed libraries.
# See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776
- if build_option('lib64_lib_symlink'):
- if os.path.exists(lib_dir) and not os.path.exists(lib64_dir):
- # create *relative* 'lib64' symlink to 'lib';
- # see https://github.com/easybuilders/easybuild-framework/issues/3564
- symlink('lib', lib64_dir, use_abspath_source=False)
+ if build_option('lib64_lib_symlink') and os.path.exists(lib_dir) and not os.path.exists(lib64_dir):
+ # create *relative* 'lib64' symlink to 'lib';
+ # see https://github.com/easybuilders/easybuild-framework/issues/3564
+ symlink('lib', lib64_dir, use_abspath_source=False)
# symlink lib to lib64, which is helpful on OpenSUSE;
# see https://github.com/easybuilders/easybuild-framework/issues/3549
- if build_option('lib_lib64_symlink'):
- if os.path.exists(lib64_dir) and not os.path.exists(lib_dir):
- # create *relative* 'lib' symlink to 'lib64';
- symlink('lib64', lib_dir, use_abspath_source=False)
+ if build_option('lib_lib64_symlink') and os.path.exists(lib64_dir) and not os.path.exists(lib_dir):
+ # create *relative* 'lib' symlink to 'lib64';
+ symlink('lib64', lib_dir, use_abspath_source=False)
+
+ self.run_post_install_commands()
+ self.apply_post_install_patches()
+ self.print_post_install_messages()
+
+ self.fix_shebang()
+
+ def post_processing_step(self):
+ """
+ Do some postprocessing
+ - run post install commands if any were specified
+ """
+ # if post_install_step() is present in the easyblock print a deprecation warning
+ # with EB 6.0, post_install_step() can be renamed to post_processing_step, and this method deleted.
+
+ if self.post_install_step.__qualname__ != "EasyBlock.post_install_step":
+ self.log.deprecated(
+ "EasyBlock.post_install_step() is deprecated, use EasyBlock.post_processing_step() instead.",
+ '6.0',
+ )
+ return self.post_install_step()
+
+ def _dispatch_sanity_check_step(self, *args, **kwargs):
+ """Decide whether to run the dry-run or the real version of the sanity-check step"""
+ if self.dry_run:
+ self._sanity_check_step_dry_run(*args, **kwargs)
+ else:
+ self._sanity_check_step(*args, **kwargs)
def sanity_check_step(self, *args, **kwargs):
"""
@@ -3072,15 +3308,11 @@ def sanity_check_step(self, *args, **kwargs):
- if *any* of the files/subdirectories in the installation directory listed
in sanity_check_paths are non-existent (or empty), the sanity check fails
"""
- if self.dry_run:
- self._sanity_check_step_dry_run(*args, **kwargs)
-
# handling of extensions that were installed for multiple dependency versions is done in ExtensionEasyBlock
- elif self.cfg['multi_deps'] and not self.is_extension:
+ if self.cfg['multi_deps'] and not self.is_extension:
self._sanity_check_step_multi_deps(*args, **kwargs)
-
else:
- self._sanity_check_step(*args, **kwargs)
+ self._dispatch_sanity_check_step(*args, **kwargs)
def _sanity_check_step_multi_deps(self, *args, **kwargs):
"""Perform sanity check for installations that iterate over a list a versions for particular dependencies."""
@@ -3112,7 +3344,7 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs):
self.log.info(info_msg)
kwargs['extra_modules'] = extra_modules
- self._sanity_check_step(*args, **kwargs)
+ self._dispatch_sanity_check_step(*args, **kwargs)
# restore list of lists of build dependencies & stop iterating again
self.cfg['builddependencies'] = builddeps
@@ -3125,46 +3357,52 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
fails = []
- # hard reset $LD_LIBRARY_PATH before running RPATH sanity check
- orig_env = env.unset_env_vars(['LD_LIBRARY_PATH'])
+ if build_option('strict_rpath_sanity_check'):
+ self.log.info("Unsetting $LD_LIBRARY_PATH since strict RPATH sanity check is enabled...")
+ # hard reset $LD_LIBRARY_PATH before running RPATH sanity check
+ orig_env = env.unset_env_vars(['LD_LIBRARY_PATH'])
+ else:
+ self.log.info("Not unsetting $LD_LIBRARY_PATH since strict RPATH sanity check is disabled...")
+ orig_env = None
- self.log.debug("$LD_LIBRARY_PATH during RPATH sanity check: %s", os.getenv('LD_LIBRARY_PATH', '(empty)'))
- self.log.debug("List of loaded modules: %s", self.modules_tool.list())
+ ld_library_path = os.getenv('LD_LIBRARY_PATH', '(empty)')
+ self.log.debug(f"$LD_LIBRARY_PATH during RPATH sanity check: {ld_library_path}")
+ modules_list = self.modules_tool.list()
+ self.log.debug(f"List of loaded modules: {modules_list}")
not_found_regex = re.compile(r'(\S+)\s*\=\>\s*not found')
- readelf_rpath_regex = re.compile('(RPATH)', re.M)
+ lib_path_regex = re.compile(r'\S+\s*\=\>\s*(\S+)')
+ readelf_rpath_regex = re.compile(r'\(RPATH\)', re.M)
# List of libraries that should be exempt from the RPATH sanity check;
# For example, libcuda.so.1 should never be RPATH-ed by design,
# see https://github.com/easybuilders/easybuild-framework/issues/4095
filter_rpath_sanity_libs = build_option('filter_rpath_sanity_libs')
- msg = "Ignoring the following libraries if they are not found by RPATH sanity check: %s"
- self.log.info(msg, filter_rpath_sanity_libs)
+ self.log.info("Ignoring the following libraries if they are not found by RPATH sanity check: %s",
+ filter_rpath_sanity_libs)
if rpath_dirs is None:
rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs()
if not rpath_dirs:
rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS
- self.log.info("Using default subdirectories for binaries/libraries to verify RPATH linking: %s",
- rpath_dirs)
+ self.log.info(f"Using default subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}")
else:
- self.log.info("Using specified subdirectories for binaries/libraries to verify RPATH linking: %s",
- rpath_dirs)
+ self.log.info(f"Using specified subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}")
for dirpath in [os.path.join(self.installdir, d) for d in rpath_dirs]:
if os.path.exists(dirpath):
- self.log.debug("Sanity checking RPATH for files in %s", dirpath)
+ self.log.debug(f"Sanity checking RPATH for files in {dirpath}")
for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
- self.log.debug("Sanity checking RPATH for %s", path)
+ self.log.debug(f"Sanity checking RPATH for {path}")
out = get_linked_libs_raw(path)
if out is None:
- msg = "Failed to determine dynamically linked libraries for %s, "
+ msg = "Failed to determine dynamically linked libraries for {path}, "
msg += "so skipping it in RPATH sanity check"
- self.log.debug(msg, path)
+ self.log.debug(msg)
else:
# check whether all required libraries are found via 'ldd'
matches = re.findall(not_found_regex, out)
@@ -3172,36 +3410,46 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
# For each match, check if the library is in the exception list
for match in matches:
if match in filter_rpath_sanity_libs:
- msg = "Library %s not found for %s, but ignored "
- msg += "since it is on the rpath exception list: %s"
- self.log.info(msg, match, path, filter_rpath_sanity_libs)
+ msg = f"Library {match} not found for {path}, but ignored "
+ msg += f"since it is on the rpath exception list: {filter_rpath_sanity_libs}"
+ self.log.info(msg)
else:
- fail_msg = "Library %s not found for %s" % (match, path)
+ fail_msg = f"Library {match} not found for {path}"
self.log.warning(fail_msg)
fails.append(fail_msg)
+
+ # if any libraries were not found, log whether dependency libraries have an RPATH section
+ if fails:
+ lib_paths = re.findall(lib_path_regex, out)
+ for lib_path in lib_paths:
+ self.log.info(f"Checking whether dependency library {lib_path} has RPATH section")
+ res = run_shell_cmd(f"readelf -d {lib_path}", fail_on_error=False)
+ if res.exit_code:
+ self.log.info(f"No RPATH section found in {lib_path}")
else:
- self.log.debug("Output of 'ldd %s' checked, looks OK", path)
+ self.log.debug(f"Output of 'ldd {path}' checked, looks OK")
# check whether RPATH section in 'readelf -d' output is there
if check_readelf_rpath:
fail_msg = None
- out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
- if ec:
- fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
- elif not readelf_rpath_regex.search(out):
- fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out)
+ res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False, hidden=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ fail_msg = f"Failed to run 'readelf -d {path}': {res.output}"
+ elif not readelf_rpath_regex.search(res.output):
+ fail_msg = f"No '(RPATH)' found in 'readelf -d' output for {path}"
if fail_msg:
self.log.warning(fail_msg)
fails.append(fail_msg)
else:
- self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
+ self.log.debug(f"Output of 'readelf -d {path}' checked, looks OK")
else:
self.log.debug("Skipping the RPATH section check with 'readelf -d', as requested")
else:
- self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
+ self.log.debug(f"Not sanity checking files in non-existing directory {dirpath}")
- env.restore_env_vars(orig_env)
+ if orig_env:
+ env.restore_env_vars(orig_env)
return fails
@@ -3330,6 +3578,19 @@ def regex_for_lib(lib):
return fail_msg
+ def sanity_check_mod_files(self):
+ """
+ Check installation for Fortran .mod files
+ """
+ self.log.debug(f"Checking for .mod files in install directory {self.installdir}...")
+ mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True)
+
+ fail_msg = None
+ if mod_files:
+ fail_msg = f"One or more .mod files found in {self.installdir}: " + ', '.join(mod_files)
+
+ return fail_msg
+
def _sanity_check_step_common(self, custom_paths, custom_commands):
"""
Determine sanity check paths and commands to use.
@@ -3359,7 +3620,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
# if no sanity_check_paths are specified in easyconfig,
# we fall back to the ones provided by the easyblock via custom_paths
if custom_paths:
- paths = custom_paths
+ paths = self.cfg.resolve_template(custom_paths)
self.log.info("Using customized sanity check paths: %s", paths)
# if custom_paths is empty, we fall back to a generic set of paths:
# non-empty bin/ + /lib or /lib64 directories
@@ -3373,14 +3634,13 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
# if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig,
# those paths are used to enhance the paths provided by the easyblock
if enhance_sanity_check and ec_paths:
- for key in ec_paths:
- val = ec_paths[key]
+ for key, val in ec_paths.items():
if isinstance(val, list):
paths[key] = paths.get(key, []) + val
else:
- error_pattern = "Incorrect value type in sanity_check_paths, should be a list: "
- error_pattern += "%s (type: %s)" % (val, type(val))
- raise EasyBuildError(error_pattern)
+ error_msg = "Incorrect value type in sanity_check_paths, should be a list: "
+ error_msg += "%s (type: %s)" % (val, type(val))
+ raise EasyBuildError(error_msg)
self.log.info("Enhanced sanity check paths after taking into account easyconfig file: %s", paths)
sorted_keys = sorted(paths.keys())
@@ -3410,7 +3670,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
self.log.info("Using (only) sanity check commands specified by easyconfig file: %s", commands)
else:
if custom_commands:
- commands = custom_commands
+ commands = self.cfg.resolve_template(custom_commands)
self.log.info("Using customised sanity check commands: %s", commands)
else:
commands = []
@@ -3424,7 +3684,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
for i, command in enumerate(commands):
# set command to default. This allows for config files with
# non-tuple commands
- if isinstance(command, string_type):
+ if isinstance(command, str):
self.log.debug("Using %s as sanity check command" % command)
commands[i] = command
else:
@@ -3584,7 +3844,7 @@ def xs2str(xs):
(typ, check_fn) = path_keys_and_check[key]
for xs in paths[key]:
- if isinstance(xs, string_type):
+ if isinstance(xs, str):
xs = (xs,)
elif not isinstance(xs, tuple):
raise EasyBuildError("Unsupported type %s encountered in '%s', not a string or tuple",
@@ -3608,6 +3868,7 @@ def xs2str(xs):
if not found:
sanity_check_fail_msg = "no %s found at %s in %s" % (typ, xs2str(xs), self.installdir)
self.sanity_check_fail_msgs.append(sanity_check_fail_msg)
+ self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK
self.log.warning("Sanity check: %s", sanity_check_fail_msg)
trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found]))
@@ -3620,24 +3881,26 @@ def xs2str(xs):
if self.toolchain.mpi_family() and self.toolchain.mpi_family() in toolchain.OPENMPI:
env.setvar('OMPI_MCA_rmaps_base_oversubscribe', '1')
- # change to install directory (better environment for running tests)
- if os.path.isdir(self.installdir):
- change_dir(self.installdir)
+ # run sanity checks from an empty temp directory
+ # using the build or installation directory can produce false positives and polute them with files
+ sanity_check_work_dir = tempfile.mkdtemp(prefix='eb-sanity-check-')
# run sanity check commands
- for command in commands:
+ for cmd in commands:
- trace_msg("running command '%s' ..." % command)
+ trace_msg(f"running command '{cmd}' ...")
- out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False, trace=False)
- if ec != 0:
- fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out)
+ res = run_shell_cmd(cmd, work_dir=sanity_check_work_dir, fail_on_error=False, hidden=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ fail_msg = f"sanity check command {cmd} failed with exit code {res.exit_code} (output: {res.output})"
self.sanity_check_fail_msgs.append(fail_msg)
- self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1])
+ self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK
+ self.log.warning(f"Sanity check: {fail_msg}")
else:
- self.log.info("sanity check command %s ran successfully! (output: %s)" % (command, out))
+ self.log.info(f"sanity check command {cmd} ran successfully! (output: {res.output})")
- trace_msg("result for command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0]))
+ cmd_result_str = ('FAILED', 'OK')[res.exit_code == EasyBuildExit.SUCCESS]
+ trace_msg(f"result for command '{cmd}': {cmd_result_str}")
# also run sanity check for extensions (unless we are an extension ourselves)
if not extension:
@@ -3651,6 +3914,16 @@ def xs2str(xs):
self.log.warning("Check for required/banned linked shared libraries failed!")
self.sanity_check_fail_msgs.append(linked_shared_lib_fails)
+ # software installed with GCCcore toolchain should not have Fortran module files (.mod),
+ # unless that's explicitly allowed
+ if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']:
+ mod_files_found_msg = self.sanity_check_mod_files()
+ if mod_files_found_msg:
+ if build_option('fail_on_mod_files_gcccore'):
+ self.sanity_check_fail_msgs.append(mod_files_found_msg)
+ else:
+ print_warning(mod_files_found_msg)
+
# cleanup
if self.fake_mod_data:
self.clean_up_fake_module(self.fake_mod_data)
@@ -3667,9 +3940,12 @@ def xs2str(xs):
# pass or fail
if self.sanity_check_fail_msgs:
- raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs))
- else:
- self.log.debug("Sanity check passed!")
+ raise EasyBuildError(
+ "Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs),
+ exit_code=EasyBuildExit.FAIL_SANITY_CHECK,
+ )
+
+ self.log.debug("Sanity check passed!")
def _set_module_as_default(self, fake=False):
"""
@@ -3699,7 +3975,7 @@ def cleanup_step(self):
# make sure we're out of the dir we're removing
change_dir(self.orig_workdir)
- self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd())
+ self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd())
try:
remove_dir(self.builddir)
@@ -3754,7 +4030,7 @@ def make_module_step(self, fake=False):
txt += self.make_module_deppaths()
txt += self.make_module_dep()
txt += self.make_module_extend_modpath()
- txt += self.make_module_req()
+ txt += self.make_module_req(fake=fake)
txt += self.make_module_extra()
txt += self.make_module_footer()
@@ -3769,8 +4045,13 @@ def make_module_step(self, fake=False):
for line in txt.split('\n'):
self.dry_run_msg(INDENT_4SPACES + line)
else:
- write_file(mod_filepath, txt)
- self.log.info("Module file %s written: %s", mod_filepath, txt)
+ try:
+ write_file(mod_filepath, txt)
+ self.log.info("Module file %s written: %s", mod_filepath, txt)
+ except EasyBuildError:
+ raise EasyBuildError(
+ f"Unable to write Module file {mod_filepath}", exit_code=EasyBuildExit.FAIL_MODULE_WRITE
+ )
# if backup module file is there, print diff with newly generated module file
if self.mod_file_backup and not fake:
@@ -3878,26 +4159,26 @@ def test_cases_step(self):
for test in self.cfg['tests']:
change_dir(self.orig_workdir)
if os.path.isabs(test):
- path = test
+ test_cmd = test
else:
for source_path in source_paths():
- path = os.path.join(source_path, self.name, test)
- if os.path.exists(path):
+ test_cmd = os.path.join(source_path, self.name, test)
+ if os.path.exists(test_cmd):
break
- if not os.path.exists(path):
- raise EasyBuildError("Test specifies invalid path: %s", path)
+ if not os.path.exists(test_cmd):
+ raise EasyBuildError(f"Test specifies invalid path: {test_cmd}")
try:
- self.log.debug("Running test %s" % path)
- run_cmd(path, log_all=True, simple=True)
+ self.log.debug(f"Running test {test_cmd}")
+ run_shell_cmd(test_cmd)
except EasyBuildError as err:
- raise EasyBuildError("Running test %s failed: %s", path, err)
+ raise EasyBuildError(f"Running test {test_cmd} failed: {err}")
def update_config_template_run_step(self):
"""Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names"""
for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- self.cfg.template_values[name[0]] = str(getattr(self, name[0], None))
+ self.cfg.template_values[name] = str(getattr(self, name, None))
self.cfg.generate_template_values()
def skip_step(self, step, skippable):
@@ -3966,7 +4247,7 @@ def run_step(self, step, step_methods):
for step_method in step_methods:
# Remove leading underscore from e.g. "_test_step"
- method_name = extract_method_name(step_method).lstrip('_')
+ method_name = '_'.join(step_method.__code__.co_names).lstrip('_')
self.log.info("Running method %s part of step %s", method_name, step)
if self.dry_run:
@@ -4018,15 +4299,6 @@ def ready_step_spec(initial):
return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False,
initial=initial)
- source_substeps = [
- (False, lambda x: x.checksum_step),
- (True, lambda x: x.extract_step),
- ]
-
- def source_step_spec(initial):
- """Return source step specified."""
- return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial)
-
install_substeps = [
(False, lambda x: x.stage_install_step),
(False, lambda x: x.make_installdir),
@@ -4040,6 +4312,7 @@ def install_step_spec(initial):
# format for step specifications: (step_name, description, list of functions, skippable)
# core steps that are part of the iterated loop
+ extract_step_spec = (EXTRACT_STEP, "unpacking", [lambda x: x.extract_step], True)
patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True)
prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
@@ -4049,9 +4322,10 @@ def install_step_spec(initial):
# part 1: pre-iteration + first iteration
steps_part1 = [
- (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step], False),
+ (FETCH_STEP, "fetching files and verifying checksums",
+ [lambda x: x.fetch_step, lambda x: x.checksum_step], False),
ready_step_spec(True),
- source_step_spec(True),
+ extract_step_spec,
patch_step_spec,
prepare_step_spec,
configure_step_spec,
@@ -4065,7 +4339,7 @@ def install_step_spec(initial):
# not all parts of all steps need to be rerun (see e.g., ready, prepare)
steps_part2 = [
ready_step_spec(False),
- source_step_spec(False),
+ extract_step_spec,
patch_step_spec,
prepare_step_spec,
configure_step_spec,
@@ -4077,7 +4351,7 @@ def install_step_spec(initial):
# part 3: post-iteration part
steps_part3 = [
(POSTITER_STEP, 'restore after iterating', [lambda x: x.post_iter_step], False),
- (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step], True),
+ (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_processing_step], True),
(SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], True),
(CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step], False),
(MODULE_STEP, 'creating module', [lambda x: x.make_module_step], False),
@@ -4153,6 +4427,17 @@ def run_all_steps(self, run_test_cases):
start_time = datetime.now()
try:
self.run_step(step_name, step_methods)
+ except RunShellCmdError as err:
+ err.print()
+ error_msg = (
+ f"shell command '{err.cmd_name} ...' failed with exit code {err.exit_code} "
+ f"in {step_name} step for {os.path.basename(self.cfg.path)}"
+ )
+ try:
+ step_exit_code = EasyBuildExit[f"FAIL_{step_name.upper()}_STEP"]
+ except KeyError:
+ step_exit_code = EasyBuildExit.ERROR
+ raise EasyBuildError(error_msg, exit_code=step_exit_code) from err
finally:
if not self.dry_run:
step_duration = datetime.now() - start_time
@@ -4190,6 +4475,54 @@ def print_dry_run_note(loc, silent=True):
dry_run_msg(msg, silent=silent)
+def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig):
+ """
+ Copy build directories and log files for failed installation (if desired)
+ """
+ logs_path = get_failed_install_logs_path(easyconfig)
+ build_dirs_path = get_failed_install_build_dirs_path(easyconfig)
+
+ # there may be multiple log files, or the file name may be different due to zipping
+ logs = glob.glob(f"{application_log}*")
+
+ timestamp = time.strftime('%Y%m%d-%H%M%S')
+ salt = ''.join(random.choices(ascii_letters, k=5))
+ unique_subdir = f'{timestamp}-{salt}'
+
+ operation_args = []
+
+ if logs_path and logs:
+ logs_path = os.path.join(logs_path, unique_subdir)
+
+ if is_parent_path(app.builddir, logs_path):
+ msg = "Path to copy log files of failed installs to is subdirectory of build directory; not copying"
+ print_warning(msg, log=_log, silent=silent)
+ else:
+ msg = f"Log file(s) of failed installation copied to {logs_path}"
+ operation_args.append((copy_file, logs, logs_path, msg))
+
+ if build_dirs_path and os.path.isdir(app.builddir):
+ build_dirs_path = os.path.join(build_dirs_path, unique_subdir)
+
+ if is_parent_path(app.builddir, build_dirs_path):
+ msg = "Path to copy build dirs of failed installs to is subdirectory of build directory; not copying"
+ print_warning(msg, log=_log, silent=silent)
+ else:
+ msg = f"Build directory of failed installation copied to {build_dirs_path}"
+
+ def operation(src, dest):
+ copy_dir(src, dest, dirs_exist_ok=True)
+
+ operation_args.append((operation, [app.builddir], build_dirs_path, msg))
+
+ persistence_paths = create_non_existing_paths(target_path for (_, _, target_path, _) in operation_args)
+
+ for idx, (operation, paths, _, msg) in enumerate(operation_args):
+ for path in paths:
+ operation(path, persistence_paths[idx])
+ print_msg(msg, log=_log, silent=silent)
+
+
def build_and_install_one(ecdict, init_env):
"""
Build the software
@@ -4220,11 +4553,10 @@ def build_and_install_one(ecdict, init_env):
# restore original environment, and then sanitize it
_log.info("Resetting environment")
- run.errors_found_in_log = 0
restore_env(init_env)
sanitize_env()
- cwd = os.getcwd()
+ cwd = get_cwd()
# load easyblock
easyblock = build_option('easyblock')
@@ -4237,7 +4569,7 @@ def build_and_install_one(ecdict, init_env):
try:
app_class = get_easyblock_class(easyblock, name=name)
app = app_class(ecdict['ec'])
- _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
+ _log.info("Obtained application instance for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError as err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
silent=silent)
@@ -4254,7 +4586,8 @@ def build_and_install_one(ecdict, init_env):
app.cfg['skip'] = skip
# build easyconfig
- errormsg = '(no error)'
+ error_msg = '(no error)'
+ exit_code = None
# timing info
start_time = time.time()
try:
@@ -4292,9 +4625,11 @@ def build_and_install_one(ecdict, init_env):
adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True)
except EasyBuildError as err:
- first_n = 300
- errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n])
- _log.warning(errormsg)
+ error_msg = err.msg
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.ERROR
result = False
ended = 'ended'
@@ -4309,12 +4644,14 @@ def build_and_install_one(ecdict, init_env):
def ensure_writable_log_dir(log_dir):
"""Make sure we can write into the log dir"""
if build_option('read_only_installdir'):
- # temporarily re-enable write permissions for copying log/easyconfig to install dir
+ # temporarily re-enable write permissions for copying log/easyconfig to install dir,
+ # ensuring that we resolve symlinks
+ log_dir = os.path.realpath(log_dir)
if os.path.exists(log_dir):
adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
else:
parent_dir = os.path.dirname(log_dir)
- if os.path.exists(parent_dir):
+ if os.path.exists(parent_dir) and not (os.stat(parent_dir).st_mode & stat.S_IWUSR):
adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
mkdir(log_dir, parents=True)
adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
@@ -4395,7 +4732,7 @@ def ensure_writable_log_dir(log_dir):
copy_file(patch['path'], target)
_log.debug("Copied patch %s to %s", patch['path'], target)
- if build_option('read_only_installdir'):
+ if build_option('read_only_installdir') and not app.cfg['stop']:
# take away user write permissions (again)
perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
adjust_permissions(new_log_dir, perms, add=False, recursive=True)
@@ -4412,15 +4749,13 @@ def ensure_writable_log_dir(log_dir):
success = True
summary = 'COMPLETED'
succ = 'successfully'
+ exit_code = EasyBuildExit.SUCCESS
else:
# build failed
success = False
summary = 'FAILED'
-
- build_dir = ''
- if app.builddir:
- build_dir = " (build directory: %s)" % (app.builddir)
- succ = "unsuccessfully%s: %s" % (build_dir, errormsg)
+ succ = "unsuccessfully: " + error_msg
+ exit_code = EasyBuildExit.ERROR if exit_code is None else exit_code
# cleanup logs
app.close_log()
@@ -4429,11 +4764,6 @@ def ensure_writable_log_dir(log_dir):
req_time = time2str(end_timestamp - start_timestamp)
print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent)
- # check for errors
- if run.errors_found_in_log > 0:
- _log.warning("%d possible error(s) were detected in the "
- "build logs, please verify the build.", run.errors_found_in_log)
-
if app.postmsg:
print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent)
@@ -4451,9 +4781,12 @@ def ensure_writable_log_dir(log_dir):
logs = glob.glob('%s*' % application_log)
print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent)
+ if not success:
+ copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec'])
+
del app
- return (success, application_log, errormsg)
+ return (success, application_log, error_msg, exit_code)
def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
@@ -4463,7 +4796,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
for easyblock_class in inspect.getmro(type(easyblock_instance)):
easyblock_path = inspect.getsourcefile(easyblock_class)
# if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done
- # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import)
+ # (Extension and ExtensionEasyBlock are hardcoded to avoid a cyclical import)
if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']:
break
else:
@@ -4540,7 +4873,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
instance = get_easyblock_instance(ec)
apps.append(instance)
- base_dir = os.getcwd()
+ base_dir = get_cwd()
# keep track of environment right before initiating builds
# note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now
@@ -4674,7 +5007,7 @@ def inject_checksums(ecs, checksum_type):
def make_list_lines(values, indent_level):
"""Make lines for list of values."""
def to_str(s):
- if isinstance(s, string_type):
+ if isinstance(s, str):
return "'%s'" % s
else:
return str(s)
diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py
index b19017e023..47385a8862 100644
--- a/easybuild/framework/easyconfig/__init__.py
+++ b/easybuild/framework/easyconfig/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py
index 4fe8c9d7b5..e4ba2c3c6f 100644
--- a/easybuild/framework/easyconfig/constants.py
+++ b/easybuild/framework/easyconfig/constants.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,6 +37,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import print_warning
+from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS
from easybuild.tools.systemtools import KNOWN_ARCH_CONSTANTS, get_os_name, get_os_type, get_os_version
@@ -67,6 +68,7 @@ def _get_arch_constant():
'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
+ 'MODULE_LOAD_ENV_HEADERS': (MODULE_LOAD_ENV_HEADERS, "Environment variables with search paths to CPP headers"),
'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
'OS_VERSION': (get_os_version(), "System version"),
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index ce11d0457b..66ed417b61 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -110,8 +110,6 @@
'installopts': ['', 'Extra options for installation', BUILD],
'maxparallel': [None, 'Max degree of parallelism', BUILD],
'module_only': [False, 'Only generate module file', BUILD],
- 'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
- 'cores, active cpuset and restrictions in ulimit)'), BUILD],
'patches': [[], "List of patches to apply", BUILD],
'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],
@@ -131,6 +129,7 @@
'sanity_check_paths': [{}, ("List of files and directories to check "
"(format: {'files':, 'dirs':})"), BUILD],
'skip': [False, "Skip existing software", BUILD],
+ 'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD],
'skipsteps': [[], "Skip these steps", BUILD],
'source_urls': [[], "List of URLs for source files", BUILD],
'sources': [[], "List of source files", BUILD],
@@ -155,8 +154,8 @@
FILEMANAGEMENT],
'keeppreviousinstall': [False, ('Boolean to keep the previous installation with identical '
'name. Experts only!'), FILEMANAGEMENT],
- 'keepsymlinks': [False, ('Boolean to determine whether symlinks are to be kept during copying '
- 'or if the content of the files pointed to should be copied'),
+ 'keepsymlinks': [True, ('Boolean to determine whether symlinks are to be kept during copying '
+ 'or if the content of the files pointed to should be copied'),
FILEMANAGEMENT],
'start_dir': [None, ('Path to start the make in. If the path is absolute, use that path. '
'If not, this is added to the guessed path.'), FILEMANAGEMENT],
@@ -190,12 +189,9 @@
'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS],
# MODULES easyconfig parameters
- 'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES],
- 'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES],
'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES],
'modaliases': [{}, "Aliases to be defined in module file", MODULES],
'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES],
- 'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES],
'modextravars': [{}, "Extra environment variables to be added to module file", MODULES],
'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES],
'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES],
@@ -205,12 +201,13 @@
'moduleclass': [MODULECLASS_BASE, 'Module class to be used for this software', MODULES],
'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES],
'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
- 'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
- '(implies recursive unloading of modules).', MODULES],
+ 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
+ '(implies recursive unloading of modules) [DEPRECATED]', MODULES],
+ 'module_search_path_headers': [None, "Environment variable set by modules on load "
+ "with search paths to header files (if None, use $CPATH)", MODULES],
'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module "
- "(True/False to hard enable/disable; None implies honoring "
- "the --recursive-module-unload EasyBuild configuration setting",
- MODULES],
+ "(True/False to hard enable/disable; None implies honoring the "
+ "--recursive-module-unload EasyBuild configuration setting", MODULES],
# MODULES documentation easyconfig parameters
# (docurls is part of MANDATORY)
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index a8c9a488b9..4665a1d1e8 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -51,31 +51,32 @@
import easybuild.tools.filetools as filetools
from easybuild.base import fancylogger
+from easybuild.base.wrapper import create_base_metaclass
from easybuild.framework.easyconfig import MANDATORY
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS, EXTERNAL_MODULE_MARKER
from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
from easybuild.framework.easyconfig.format.convert import Dependency
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec
-from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION, is_yeb_format
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
-from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS
+from easybuild.framework.easyconfig.parser import REPLACED_PARAMETERS, EasyConfigParser
+from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg
from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
from easybuild.tools.filetools import convert_name, copy_file, create_index, decode_class_name, encode_class_name
from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, load_index
from easybuild.tools.filetools import read_file, write_file
-from easybuild.tools.hooks import PARSE, load_hooks, run_hook
+from easybuild.tools.hooks import PARSE, EXTRACT_STEP, STEP_NAMES, load_hooks, run_hook
from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
from easybuild.tools.modules import modules_tool, NoModulesTool
-from easybuild.tools.py2vs3 import create_base_metaclass, string_type
from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
@@ -119,11 +120,24 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
def new_ec_method(self, key, *args, **kwargs):
"""Check whether any replace easyconfig parameters are still used"""
# map deprecated parameters to their replacements, issue deprecation warning(/error)
- if key in DEPRECATED_PARAMETERS:
+ if key == 'parallel':
+ _log.deprecated("Easyconfig parameter 'parallel' is deprecated, "
+ "use 'max_parallel' or the parallel property instead.", '6.0')
+ # This hidden parameter allows easyblocks to continue using self.cfg['parallel'] which contains
+ # the computed parallelism after the ready step but the easyconfig parameter before that step.
+
+ # Easyblocks using `max_parallel` always get the value from the easyconfig unmodified.
+ # Easyblocks should use either the parallel property or `max_parallel` such that the semantic is clear.
+ # In particular writes to self.cfg['parallel'] do NOT update the %(parallel)s template.
+ # It can be removed when the deprecation expires.
+ key = '_parallelLegacy'
+ elif key in ALTERNATIVE_EASYCONFIG_PARAMETERS:
+ key = ALTERNATIVE_EASYCONFIG_PARAMETERS[key]
+ elif key in DEPRECATED_EASYCONFIG_PARAMETERS:
depr_key = key
- key, ver = DEPRECATED_PARAMETERS[depr_key]
- _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver)
- if key in REPLACED_PARAMETERS:
+ key, ver = DEPRECATED_EASYCONFIG_PARAMETERS[depr_key]
+ _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
+ elif key in REPLACED_PARAMETERS:
_log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
return ec_method(self, key, *args, **kwargs)
@@ -141,7 +155,10 @@ def is_local_var_name(name):
"""
res = False
if name.startswith(LOCAL_VAR_PREFIX) or name.startswith('_'):
- res = True
+ # make exception for '_parallelLegacy' hidden easyconfig parameter,
+ # which is used to deprecate use of 'paralell' easyconfig parameter
+ if name != '_parallelLegacy':
+ res = True
# __builtins__ is always defined as a 'local' variables
# single-letter local variable names are allowed (mainly for use in list comprehensions)
# in Python 2, variables defined in list comprehensions leak to the outside (no longer the case in Python 3)
@@ -180,7 +197,8 @@ def triage_easyconfig_params(variables, ec):
for key in variables:
# validations are skipped, just set in the config
- if key in ec:
+ if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(),
+ ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())):
ec_params[key] = variables[key]
_log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
elif key in REPLACED_PARAMETERS:
@@ -236,7 +254,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains
subtoolchain_version = None
# ensure we always have a tuple of alternative subtoolchain names, which makes things easier below
- if isinstance(subtoolchain_names, string_type):
+ if isinstance(subtoolchain_names, str):
subtoolchain_names = (subtoolchain_names,)
system_subtoolchain = False
@@ -248,11 +266,6 @@ def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains
# system toolchain: bottom of the hierarchy
if is_system_toolchain(subtoolchain_name):
add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains')
- if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'):
- depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
- _log.deprecated(depr_msg, '5.0')
- add_system_to_minimal_toolchains = True
-
system_subtoolchain = True
if add_system_to_minimal_toolchains and not incl_capabilities:
@@ -409,20 +422,6 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
return toolchain_hierarchy
-@contextmanager
-def disable_templating(ec):
- """Temporarily disable templating on the given EasyConfig
-
- Usage:
- with disable_templating(ec):
- # Do what you want without templating
- # Templating set to previous value
- """
- _log.deprecated("disable_templating(ec) was replaced by ec.disable_templating()", '5.0')
- with ec.disable_templating() as old_value:
- yield old_value
-
-
class EasyConfig(object):
"""
Class which handles loading, reading, validation of easyconfigs
@@ -443,7 +442,11 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
:param local_var_naming_check: mode to use when checking if local variables use the recommended naming scheme
"""
self.template_values = None
- self.enable_templating = True # a boolean to control templating
+ # a boolean to control templating, can be (temporarily) disabled via disable_templating context manager
+ self._templating_enabled = True
+ # boolean to control whether all template values must be resolvable on access,
+ # can be (temporarily) disabled via allow_unresolved_templates context manager
+ self._expect_resolved_template_values = True
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
@@ -514,6 +517,12 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
self.iterate_options = []
self.iterating = False
+ # Storage for parallel property. Mark as unset initially
+ self._parallel = None
+ # introduce hidden '_parallelLegacy' easyconfig parameter,
+ # used to deprecate use of 'parallel' easyconfig parameter
+ self._config['_parallelLegacy'] = [None, '', ('', )]
+
# parse easyconfig file
self.build_specs = build_specs
self.parse()
@@ -557,12 +566,48 @@ def disable_templating(self):
# Do what you want without templating
# Templating set to previous value
"""
- old_enable_templating = self.enable_templating
- self.enable_templating = False
+ old_templating_enabled = self._templating_enabled
+ self._templating_enabled = False
+ try:
+ yield old_templating_enabled
+ finally:
+ self._templating_enabled = old_templating_enabled
+
+ @property
+ def templating_enabled(self):
+ """Check whether templating is enabled on this EasyConfig"""
+ return self._templating_enabled
+
+ def _enable_templating(self, *_):
+ self.log.nosupport("self.enable_templating is replaced by self.templating_enabled. "
+ "To disable it use the self.disable_templating context manager", '5.0')
+ enable_templating = property(_enable_templating, _enable_templating)
+
+ @contextmanager
+ def allow_unresolved_templates(self):
+ """Temporarily allow templates to be not (fully) resolved.
+
+ This should only be used when it is intended to use partially resolved templates.
+ Otherwise `ec.get(key, resolve=False)` should be used.
+ See also @ref disable_templating.
+
+ Usage:
+ with ec.allow_unresolved_templates():
+ value = ec.get('key') # This will not raise an error
+ print(value % {'extra_key': exta_value})
+ # Resolving is enforced again if it was before
+ """
+ old_expect_resolved_template_values = self._expect_resolved_template_values
+ self._expect_resolved_template_values = False
try:
- yield old_enable_templating
+ yield old_expect_resolved_template_values
finally:
- self.enable_templating = old_enable_templating
+ self._expect_resolved_template_values = old_expect_resolved_template_values
+
+ @property
+ def expect_resolved_template_values(self):
+ """Check whether resolving all template values on access is enforced."""
+ return self._expect_resolved_template_values
def __str__(self):
"""Return a string representation of this EasyConfig instance"""
@@ -574,10 +619,7 @@ def __str__(self):
def filename(self):
"""Determine correct filename for this easyconfig file."""
- if is_yeb_format(self.path, self.rawtxt):
- ext = YEB_FORMAT_EXTENSION
- else:
- ext = EB_FORMAT_EXTENSION
+ ext = EB_FORMAT_EXTENSION
return '%s-%s%s' % (self.name, det_full_ec_version(self), ext)
@@ -613,6 +655,7 @@ def copy(self, validate=None):
ec = EasyConfig(self.path, validate=validate, hidden=self.hidden, rawtxt=self.rawtxt)
# take a copy of the actual config dictionary (which already contains the extra options)
ec._config = copy.deepcopy(self._config)
+ ec._parallel = self._parallel # Might be already set, e.g. for extensions
# since rawtxt is defined, self.path may not get inherited, make sure it does
if self.path:
ec.path = self.path
@@ -627,7 +670,7 @@ def update(self, key, value, allow_duplicate=True):
Update an easyconfig parameter with the specified value (i.e. append to it).
Note: For dictionary easyconfig parameters, 'allow_duplicate' is ignored (since it's meaningless).
"""
- if isinstance(value, string_type):
+ if isinstance(value, str):
inval = [value]
elif isinstance(value, (list, dict, tuple)):
inval = value
@@ -645,7 +688,7 @@ def update(self, key, value, allow_duplicate=True):
# Grab current parameter value so we can modify it
param_value = copy.deepcopy(self[key])
- if isinstance(param_value, string_type):
+ if isinstance(param_value, str):
for item in inval:
# re.search: only add value to string if it's not there yet (surrounded by whitespace)
if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), param_value)):
@@ -683,7 +726,8 @@ def set_keys(self, params):
with self.disable_templating():
for key in sorted(params.keys()):
# validations are skipped, just set in the config
- if key in self._config.keys():
+ if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS,
+ DEPRECATED_EASYCONFIG_PARAMETERS)):
self[key] = params[key]
self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
key, self[key], type(self[key]))
@@ -716,6 +760,12 @@ def parse(self):
if missing_mandatory_keys:
raise EasyBuildError("mandatory parameters not provided in %s: %s", self.path, missing_mandatory_keys)
+ if 'parallel' in ec_vars:
+ # Replace value and issue better warning for easyconfig parameters,
+ # as opposed to warnings meant for easyblocks)
+ self.log.deprecated("Easyconfig parameter 'parallel' is deprecated, use 'max_parallel' instead.", '6.0')
+ ec_vars['_parallelLegacy'] = ec_vars.pop('parallel')
+
# provide suggestions for typos. Local variable names are excluded from this check
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in ec_vars if not is_local_var_name(key) and key not in self]
@@ -782,12 +832,16 @@ def count_files(self):
"""
Determine number of files (sources + patches) required for this easyconfig.
"""
- cnt = len(self['sources']) + len(self['patches'])
- for ext in self['exts_list']:
+ # No need to resolve templates as we only need a count not the names
+ with self.disable_templating():
+ cnt = len(self['sources']) + len(self['patches'])
+ exts = self['exts_list']
+
+ for ext in exts:
if isinstance(ext, tuple) and len(ext) >= 3:
ext_opts = ext[2]
- # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources
+ # check for 'sources' first, since that's also considered first by EasyBlock.collect_exts_file_info
if 'sources' in ext_opts:
cnt += len(ext_opts['sources'])
elif 'source_tmpl' in ext_opts:
@@ -817,7 +871,7 @@ def local_var_naming(self, local_var_naming_check):
msg = "Use of %d unknown easyconfig parameters detected %s: %s\n" % (cnt, in_fn, unknown_keys_msg)
msg += "If these are just local variables please rename them to start with '%s', " % LOCAL_VAR_PREFIX
msg += "or try using --fix-deprecated-easyconfigs to do this automatically.\nFor more information, see "
- msg += "https://easybuild.readthedocs.io/en/latest/Easyconfig-files-local-variables.html ."
+ msg += "https://docs.easybuild.io/easyconfig-files-local-variables/ ."
# always log a warning if local variable that don't follow recommended naming scheme are found
self.log.warning(msg)
@@ -836,7 +890,7 @@ def check_deprecated(self, path):
deprecated = self['deprecated']
if deprecated:
- if isinstance(deprecated, string_type):
+ if isinstance(deprecated, str):
if 'easyconfig' not in build_option('silence_deprecation_warnings'):
depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated))
else:
@@ -852,10 +906,10 @@ def check_deprecated(self, path):
if depr_msgs:
depr_msg = ', '.join(depr_msgs)
- depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+ depr_maj_ver = int(str(VERSION).split('.', maxsplit=1)[0]) + 1
depr_ver = '%s.0' % depr_maj_ver
- more_info_depr_ec = " (see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)"
+ more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)"
self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent'))
@@ -867,8 +921,8 @@ def validate(self, check_osdeps=True):
- check license
"""
self.log.info("Validating easyconfig")
- for attr in self.validations:
- self._validate(attr, self.validations[attr])
+ for attr, valid_values in self.validations.items():
+ self._validate(attr, valid_values)
if check_osdeps:
self.log.info("Checking OS dependencies")
@@ -877,9 +931,25 @@ def validate(self, check_osdeps=True):
self.log.info("Not checking OS dependencies")
self.log.info("Checking skipsteps")
- if not isinstance(self._config['skipsteps'][0], (list, tuple,)):
+ skipsteps = self._config['skipsteps'][0]
+ if not isinstance(skipsteps, (list, tuple,)):
raise EasyBuildError('Invalid type for skipsteps. Allowed are list or tuple, got %s (%s)',
- type(self._config['skipsteps'][0]), self._config['skipsteps'][0])
+ type(skipsteps), skipsteps)
+ unknown_step_names = [step for step in skipsteps if step not in STEP_NAMES]
+ if unknown_step_names:
+ error_lines = ["Found one or more unknown step names in 'skipsteps' easyconfig parameter:"]
+ for step in unknown_step_names:
+ error_line = "* %s" % step
+ # try to find close match, may be just a typo in the step name
+ close_matches = difflib.get_close_matches(step, STEP_NAMES, 2, 0.8)
+ # 'source' step was renamed to 'extract' in EasyBuild 5.0, see provide a useful suggestion in that case;
+ # see https://github.com/easybuilders/easybuild-framework/pull/4629
+ if not close_matches and step == 'source':
+ close_matches.append(EXTRACT_STEP)
+ if close_matches:
+ error_line += " (did you mean %s?)" % ', or '.join("'%s'" % s for s in close_matches)
+ error_lines.append(error_line)
+ raise EasyBuildError('\n'.join(error_lines))
self.log.info("Checking build option lists")
self.validate_iterate_opts_lists()
@@ -914,7 +984,7 @@ def validate_os_deps(self):
not_found = []
for dep in self['osdependencies']:
# make sure we have a tuple
- if isinstance(dep, string_type):
+ if isinstance(dep, str):
dep = (dep,)
elif not isinstance(dep, tuple):
raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)",
@@ -924,9 +994,12 @@ def validate_os_deps(self):
not_found.append(dep)
if not_found:
- raise EasyBuildError("One or more OS dependencies were not found: %s", not_found)
- else:
- self.log.info("OS dependencies ok: %s" % self['osdependencies'])
+ raise EasyBuildError(
+ "One or more OS dependencies were not found: %s", not_found,
+ exit_code=EasyBuildExit.MISSING_SYSTEM_DEPENDENCY
+ )
+
+ self.log.info("OS dependencies ok: %s" % self['osdependencies'])
return True
@@ -1119,15 +1192,19 @@ def filter_deps(self, deps):
return retained_deps
- def dependencies(self, build_only=False):
+ def dependencies(self, build_only=False, runtime_only=False):
"""
Returns an array of parsed dependencies (after filtering, if requested)
dependency = {'name': '', 'version': '', 'system': (False|True), 'versionsuffix': '', 'toolchain': ''}
Iterable builddependencies are flattened when not iterating.
:param build_only: only return build dependencies, discard others
+ :param runtime_only: only return runtime dependencies, discard others
"""
- deps = self.builddependencies()
+ if runtime_only:
+ deps = []
+ else:
+ deps = self.builddependencies()
if not build_only:
# use += rather than .extend to get a new list rather than updating list of build deps in place...
@@ -1222,6 +1299,32 @@ def all_dependencies(self):
return self._all_dependencies
+ @property
+ def is_parallel_set(self):
+ """Return if the desired parallelism has been determined yet"""
+ return self._parallel is not None
+
+ @property
+ def parallel(self):
+ """Degree of parallellism (number of cores) to be used for building, etc."""
+ if not self.is_parallel_set:
+ raise ValueError("Parallelism not set yet, 'set_parallel' method of easyblock must be called first")
+
+ # This gets set when an easyblock changes ec['parallel'].
+ # It also gets set/updated in set_parallel to mirror the old behavior during the deprecation phase
+ parallelLegacy = self._config['_parallelLegacy'][0]
+ if parallelLegacy is not None:
+ return max(1, parallelLegacy)
+ return self._parallel
+
+ @parallel.setter
+ def parallel(self, value):
+ # Update backstorage and template value
+ self._parallel = max(1, value) # Also handles False
+ self.template_values['parallel'] = self._parallel
+ # Backwards compatibility, only for easyblocks still reading self.cfg['parallel']
+ self._config['_parallelLegacy'][0] = self._parallel
+
def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False):
"""
Dump this easyconfig to file, with the given filename.
@@ -1232,11 +1335,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
# templated values should be dumped unresolved
with self.disable_templating():
# build dict of default values
- default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG}
- default_values.update({key: self.extra_options[key][0] for key in self.extra_options})
+ default_values = {key: value[0] for key, value in DEFAULT_CONFIG.items()}
+ default_values.update({key: value[0] for key, value in self.extra_options.items()})
self.generate_template_values()
- templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}
+ templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()}
# create reverse map of templates, to inject template values where possible
# longer template values are considered first, shorter template keys get preference over longer ones
@@ -1289,7 +1392,10 @@ def _validate(self, attr, values): # private method
if values is None:
values = []
if self[attr] and self[attr] not in values:
- raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values)
+ raise EasyBuildError(
+ "%s provided '%s' is not valid: %s", attr, self[attr], values,
+ exit_code=EasyBuildExit.VALUE_ERROR
+ )
def probe_external_module_metadata(self, mod_name, existing_metadata=None):
"""
@@ -1485,13 +1591,13 @@ def get_parsed_multi_deps(self):
# all multi_deps entries should be listed in builddependencies (if not, something is very wrong)
if isinstance(builddeps, list) and all(isinstance(x, list) for x in builddeps):
- for iter_id in range(len(builddeps)):
+ for iter_id, current_builddeps in enumerate(builddeps):
# only build dependencies that correspond to multi_deps entries should be loaded as extra modules
# (other build dependencies should not be required to make sanity check pass for this iteration)
iter_deps = []
for key in self['multi_deps']:
- hits = [d for d in builddeps[iter_id] if d['name'] == key]
+ hits = [d for d in current_builddeps if d['name'] == key]
if len(hits) == 1:
iter_deps.append(hits[0])
else:
@@ -1644,8 +1750,9 @@ def _finalize_dependencies(self):
filter_deps_specs = self.parse_filter_deps()
for key in DEPENDENCY_PARAMETERS:
- # loop over a *copy* of dependency dicts (with resolved templates);
- deps = self[key]
+ # loop over a *copy* of dependency dicts with resolved templates,
+ # although some templates may not resolve yet (e.g. those relying on dependencies like %(pyver)s)
+ deps = resolve_template(self.get_ref(key), self.template_values, expect_resolved=False)
# to update the original dep dict, we need to get a reference with templating disabled...
deps_ref = self.get_ref(key)
@@ -1755,6 +1862,12 @@ def _generate_template_values(self, ignore=None):
if self.template_values[key] is None:
del self.template_values[key]
+ def resolve_template(self, value):
+ """Resolve all templates in the given value using this easyconfig"""
+ if not self.template_values:
+ self.generate_template_values()
+ return resolve_template(value, self.template_values, expect_resolved=self.expect_resolved_template_values)
+
@handle_deprecated_or_replaced_easyconfig_parameters
def __contains__(self, key):
"""Check whether easyconfig parameter is defined"""
@@ -1763,16 +1876,13 @@ def __contains__(self, key):
@handle_deprecated_or_replaced_easyconfig_parameters
def __getitem__(self, key):
"""Return value of specified easyconfig parameter (without help text, etc.)"""
- value = None
- if key in self._config:
+ try:
value = self._config[key][0]
- else:
+ except KeyError:
raise EasyBuildError("Use of unknown easyconfig parameter '%s' when getting parameter value", key)
- if self.enable_templating:
- if self.template_values is None or len(self.template_values) == 0:
- self.generate_template_values()
- value = resolve_template(value, self.template_values)
+ if self.templating_enabled:
+ value = self.resolve_template(value)
return value
@@ -1817,11 +1927,14 @@ def get(self, key, default=None, resolve=True):
# see also https://docs.python.org/2/reference/datamodel.html#object.__eq__
def __eq__(self, ec):
"""Is this EasyConfig instance equivalent to the provided one?"""
- return self.asdict() == ec.asdict()
+ # Compare raw values to check that templates used are the same
+ with self.disable_templating():
+ with ec.disable_templating():
+ return self.asdict() == ec.asdict()
def __ne__(self, ec):
"""Is this EasyConfig instance equivalent to the provided one?"""
- return self.asdict() != ec.asdict()
+ return not self == ec
def __hash__(self):
"""Return hash value for a hashable representation of this EasyConfig instance."""
@@ -1834,8 +1947,9 @@ def make_hashable(val):
return val
lst = []
- for (key, val) in sorted(self.asdict().items()):
- lst.append((key, make_hashable(val)))
+ with self.disable_templating():
+ for (key, val) in sorted(self.asdict().items()):
+ lst.append((key, make_hashable(val)))
# a list is not hashable, but a tuple is
return hash(tuple(lst))
@@ -1845,13 +1959,13 @@ def asdict(self):
Return dict representation of this EasyConfig instance.
"""
res = {}
- for key, tup in self._config.items():
- value = tup[0]
- if self.enable_templating:
- if not self.template_values:
- self.generate_template_values()
- value = resolve_template(value, self.template_values)
- res[key] = value
+ # Not all values can be resolved, e.g. %(installdir)s
+ with self.allow_unresolved_templates():
+ for key, tup in self._config.items():
+ value = tup[0]
+ if self.templating_enabled:
+ value = self.resolve_template(value)
+ res[key] = value
return res
def get_cuda_cc_template_value(self, key):
@@ -1861,7 +1975,7 @@ def get_cuda_cc_template_value(self, key):
Returns user-friendly error message in case neither are defined,
or if an unknown key is used.
"""
- if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC):
+ if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
try:
return self.template_values[key]
except KeyError:
@@ -1880,19 +1994,10 @@ def det_installversion(version, toolchain_name, toolchain_version, prefix, suffi
_log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0')
-def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=None, **kwargs):
+def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=True, **kwargs):
"""
Get class for a particular easyblock (or use default)
"""
- if 'default_fallback' in kwargs:
- msg = "Named argument 'default_fallback' for get_easyblock_class is deprecated, "
- msg += "use 'error_on_missing_easyblock' instead"
- _log.deprecated(msg, '4.0')
- if error_on_missing_easyblock is None:
- error_on_missing_easyblock = kwargs['default_fallback']
- elif error_on_missing_easyblock is None:
- error_on_missing_easyblock = True
-
cls = None
try:
if easyblock:
@@ -1948,12 +2053,20 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
error_re = re.compile(r"No module named '?.*/?%s'?" % modname)
_log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern)
if error_re.match(str(err)):
+ # Missing easyblock type of error
if error_on_missing_easyblock:
- raise EasyBuildError("No software-specific easyblock '%s' found for %s", class_name, name)
- elif error_on_failed_import:
- raise EasyBuildError("Failed to import %s easyblock: %s", class_name, err)
+ raise EasyBuildError(
+ "No software-specific easyblock '%s' found for %s", class_name, name,
+ exit_code=EasyBuildExit.MISSING_EASYBLOCK
+ ) from err
else:
- _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
+ # Broken import
+ if error_on_failed_import:
+ raise EasyBuildError(
+ "Failed to import %s easyblock: %s", class_name, err,
+ exit_code=EasyBuildExit.EASYBLOCK_ERROR
+ ) from err
+ _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
if cls is not None:
_log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')",
@@ -1967,13 +2080,10 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
# simply reraise rather than wrapping it into another error
raise err
except Exception as err:
- raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err)
-
-
-def is_generic_easyblock(easyblock):
- """Return whether specified easyblock name is a generic easyblock or not."""
- _log.deprecated("is_generic_easyblock function was moved to easybuild.tools.filetools", '5.0')
- return filetools.is_generic_easyblock(easyblock)
+ raise EasyBuildError(
+ "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err,
+ exit_code=EasyBuildExit.EASYBLOCK_ERROR
+ )
def get_module_path(name, generic=None, decode=True):
@@ -2002,12 +2112,13 @@ def get_module_path(name, generic=None, decode=True):
return '.'.join(modpath + [module_name])
-def resolve_template(value, tmpl_dict):
+def resolve_template(value, tmpl_dict, expect_resolved=True):
"""Given a value, try to susbstitute the templated strings with actual values.
- value: some python object (supported are string, tuple/list, dict or some mix thereof)
- tmpl_dict: template dictionary
+ - expect_resolved: Expects that all templates get resolved
"""
- if isinstance(value, string_type):
+ if isinstance(value, str):
# simple escaping, making all '%foo', '%%foo', '%%%foo' post-templates values available,
# but ignore a string like '%(name)s'
# behaviour of strings like '%(name)s',
@@ -2031,14 +2142,48 @@ def resolve_template(value, tmpl_dict):
# '%(name)s' -> '%(name)s'
# '%%(name)s' -> '%%(name)s'
if '%' in value:
- orig_value = value
+ raw_value = value
value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value)
try:
value = value % tmpl_dict
except KeyError:
- _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict)
- value = orig_value # Undo "%"-escaping
+ # check if any alternative and/or deprecated templates resolve
+ try:
+ orig_value = value
+ # map old templates to new values for alternative and deprecated templates
+ alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in
+ ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+ alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in
+ ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()}
+ depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in
+ DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+
+ # try templating with alternative and deprecated templates included
+ value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map}
+
+ for old_tmpl, val in depr_map.items():
+ # check which deprecated templates were replaced, and issue deprecation warnings
+ if old_tmpl in orig_value and val in value:
+ new_tmpl, ver = DEPRECATED_EASYCONFIG_TEMPLATES[old_tmpl]
+ _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead",
+ ver)
+ except KeyError:
+ if expect_resolved:
+ msg = (f'Failed to resolve all templates in "{value}" using template dictionary: {tmpl_dict}. '
+ 'This might cause failures or unexpected behavior, '
+ 'check for correct escaping if this is intended!')
+ if build_option('allow_unresolved_templates'):
+ print_warning(msg)
+ else:
+ raise EasyBuildError(msg)
+ value = raw_value # Undo "%"-escaping
+
+ for key in tmpl_dict:
+ if key in DEPRECATED_EASYCONFIG_TEMPLATES:
+ new_key, ver = DEPRECATED_EASYCONFIG_TEMPLATES[key]
+ _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver)
+
else:
# this block deals with references to objects and returns other references
# for reading this is ok, but for self['x'] = {}
@@ -2052,11 +2197,12 @@ def resolve_template(value, tmpl_dict):
# self._config['x']['y'] = z
# it can not be intercepted with __setitem__ because the set is done at a deeper level
if isinstance(value, list):
- value = [resolve_template(val, tmpl_dict) for val in value]
+ value = [resolve_template(val, tmpl_dict, expect_resolved) for val in value]
elif isinstance(value, tuple):
- value = tuple(resolve_template(list(value), tmpl_dict))
+ value = tuple(resolve_template(list(value), tmpl_dict, expect_resolved))
elif isinstance(value, dict):
- value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()}
+ value = {resolve_template(k, tmpl_dict, expect_resolved): resolve_template(v, tmpl_dict, expect_resolved)
+ for k, v in value.items()}
return value
@@ -2091,7 +2237,11 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
try:
ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden)
except EasyBuildError as err:
- raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg)
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.EASYCONFIG_ERROR
+ raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code)
name = ec['name']
@@ -2505,8 +2655,6 @@ def copy_patch_files(patch_specs, target_dir):
def fix_deprecated_easyconfigs(paths):
"""Fix use of deprecated functionality in easyconfigs at specified locations."""
- dummy_tc_regex = re.compile(r'^toolchain\s*=\s*{.*name.*dummy.*}', re.M)
-
easyconfig_paths = []
for path in paths:
easyconfig_paths.extend(find_easyconfigs(path))
@@ -2519,11 +2667,6 @@ def fix_deprecated_easyconfigs(paths):
fixed = False
- # fix use of 'dummy' toolchain, use SYSTEM constant instead
- if dummy_tc_regex.search(ectxt):
- ectxt = dummy_tc_regex.sub("toolchain = SYSTEM", ectxt)
- fixed = True
-
# fix use of local variables with a name other than a single letter or 'local_*'
ec = EasyConfig(path, local_var_naming_check=LOCAL_VAR_NAMING_CHECK_LOG)
for key in ec.unknown_keys:
diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py
index c2a81766ad..7ca361b39a 100644
--- a/easybuild/framework/easyconfig/format/__init__.py
+++ b/easybuild/framework/easyconfig/format/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py
index 82613d42e0..1fd3182621 100644
--- a/easybuild/framework/easyconfig/format/convert.py
+++ b/easybuild/framework/easyconfig/format/convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py
index 3aeae419a2..eec432967e 100644
--- a/easybuild/framework/easyconfig/format/format.py
+++ b/easybuild/framework/easyconfig/format/format.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -41,7 +41,6 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.configobj import Section
from easybuild.tools.utilities import get_subclasses
-from easybuild.tools.py2vs3 import string_type
# format is mandatory major.minor
@@ -69,11 +68,11 @@
['preconfigopts', 'configopts'],
['prebuildopts', 'buildopts'],
['preinstallopts', 'installopts'],
- ['parallel', 'maxparallel'],
+ ['maxparallel'],
]
LAST_PARAMS = ['exts_default_options', 'exts_list',
'sanity_check_paths', 'sanity_check_commands',
- 'modextrapaths', 'modextrapaths_append', 'modextravars',
+ 'modextrapaths', 'modextravars',
'moduleclass']
SANITY_CHECK_PATHS_DIRS = 'dirs'
@@ -323,7 +322,7 @@ def parse_sections(self, toparse, current):
value_type = self.VERSION_OPERATOR_VALUE_TYPES[key]
# list of supported toolchains/versions
# first one is default
- if isinstance(value, string_type):
+ if isinstance(value, str):
# so the split should be unnecessary
# (if it's not a list already, it's just one value)
# TODO this is annoying. check if we can force this in configobj
@@ -434,7 +433,7 @@ def _squash(self, vt_tuple, processed, sanity):
# walk over dictionary of parsed sections, and check for marker conflicts (using .add())
for key, value in processed.items():
if isinstance(value, NestedDict):
- tmp = self._squash_netsed_dict(key, value, squashed, sanity, vt_tuple)
+ tmp = self._squash_nested_dict(key, value, squashed, sanity, vt_tuple)
res_sections.update(tmp)
elif key in self.VERSION_OPERATOR_VALUE_TYPES:
self.log.debug("Found VERSION_OPERATOR_VALUE_TYPES entry (%s)" % key)
@@ -454,7 +453,7 @@ def _squash(self, vt_tuple, processed, sanity):
(processed, squashed.versions, squashed.result))
return squashed
- def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple):
+ def _squash_nested_dict(self, key, nested_dict, squashed, sanity, vt_tuple):
"""
Squash NestedDict instance, returns dict with already squashed data
from possible higher sections
diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py
index 868e218925..a45c855436 100644
--- a/easybuild/framework/easyconfig/format/one.py
+++ b/easybuild/framework/easyconfig/format/one.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -49,7 +49,6 @@
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.filetools import read_file, write_file
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import INDENT_4SPACES, quote_py_str
@@ -255,7 +254,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
- if isinstance(param_val, string_type) and param_name not in DEPENDENCY_PARAMETERS:
+ if isinstance(param_val, str) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res
diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
index 8227a40958..b7e7206536 100644
--- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
+++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,7 +39,8 @@
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS, TEMPLATE_CONSTANTS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.configobj import ConfigObj
from easybuild.tools.systemtools import get_shared_lib_ext
@@ -51,8 +52,8 @@
def build_easyconfig_constants_dict():
"""Make a dictionary with all constants that can be used"""
all_consts = [
- ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}),
- ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}),
+ ('TEMPLATE_CONSTANTS', {name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}),
+ ('EASYCONFIG_CONSTANTS', {name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}),
('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}),
]
err = []
@@ -86,6 +87,58 @@ def build_easyconfig_variables_dict():
return vars_dict
+def handle_deprecated_constants(method):
+ """Decorator to handle deprecated easyconfig template constants"""
+ def wrapper(self, key, *args, **kwargs):
+ """Check whether any deprecated constants are used"""
+ alternative = ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS
+ deprecated = DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS
+ if key in alternative:
+ key = alternative[key]
+ elif key in deprecated:
+ depr_key = key
+ key, ver = deprecated[depr_key]
+ _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver)
+ return method(self, key, *args, **kwargs)
+ return wrapper
+
+
+class DeprecatedDict(dict):
+ """Custom dictionary that handles deprecated easyconfig template constants gracefully"""
+
+ def __init__(self, *args, **kwargs):
+ self.clear()
+ self.update(*args, **kwargs)
+
+ @handle_deprecated_constants
+ def __contains__(self, key):
+ return super().__contains__(key)
+
+ @handle_deprecated_constants
+ def __delitem__(self, key):
+ return super().__delitem__(key)
+
+ @handle_deprecated_constants
+ def __getitem__(self, key):
+ return super().__getitem__(key)
+
+ @handle_deprecated_constants
+ def __setitem__(self, key, value):
+ return super().__setitem__(key, value)
+
+ def update(self, *args, **kwargs):
+ if args:
+ if isinstance(args[0], dict):
+ for key, value in args[0].items():
+ self.__setitem__(key, value)
+ else:
+ for key, value in args[0]:
+ self.__setitem__(key, value)
+
+ for key, value in kwargs.items():
+ self.__setitem__(key, value)
+
+
class EasyConfigFormatConfigObj(EasyConfigFormat):
"""
Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed).
@@ -176,7 +229,7 @@ def parse_header(self, header):
def parse_pyheader(self, pyheader):
"""Parse the python header, assign to docstring and cfg"""
- global_vars = self.pyheader_env()
+ global_vars = DeprecatedDict(self.pyheader_env())
self.log.debug("pyheader initial global_vars %s", global_vars)
self.log.debug("pyheader text being exec'ed: %s", pyheader)
diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py
index 652b1c9b80..e84d807bd9 100644
--- a/easybuild/framework/easyconfig/format/two.py
+++ b/easybuild/framework/easyconfig/format/two.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py
index b8f181e5aa..4b3cdb6098 100644
--- a/easybuild/framework/easyconfig/format/version.py
+++ b/easybuild/framework/easyconfig/format/version.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,7 +37,6 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.toolchain.utilities import search_toolchain
@@ -46,7 +45,16 @@
class EasyVersion(LooseVersion):
- """Exact LooseVersion. No modifications needed (yet)"""
+ """Represent a version"""
+
+ def __init__(self, vstring, is_default=False):
+ super().__init__(vstring)
+ self._is_default = is_default
+
+ @property
+ def is_default(self):
+ """Return whether this is the default version used when no explicit version is specified"""
+ return self._is_default
def __len__(self):
"""Determine length of this EasyVersion instance."""
@@ -75,7 +83,7 @@ class VersionOperator(object):
OPERATOR_FAMILIES = [['>', '>='], ['<', '<=']] # similar operators
# default version and operator when version is undefined
- DEFAULT_UNDEFINED_VERSION = EasyVersion('0.0.0')
+ DEFAULT_UNDEFINED_VERSION = EasyVersion('0.0', is_default=True)
DEFAULT_UNDEFINED_VERSION_OPERATOR = OPERATOR_MAP['>']
# default operator when operator is undefined (but version is)
DEFAULT_UNDEFINED_OPERATOR = OPERATOR_MAP['==']
@@ -144,7 +152,7 @@ def test(self, test_version):
if not self:
raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__)
- if isinstance(test_version, string_type):
+ if isinstance(test_version, str):
test_version = self._convert(test_version)
elif not isinstance(test_version, EasyVersion):
raise EasyBuildError("test: argument should be a string or EasyVersion (type %s)", type(test_version))
@@ -257,7 +265,7 @@ def _convert_operator(self, operator_str, version=None):
"""Return the operator"""
operator = None
if operator_str is None:
- if version == self.DEFAULT_UNDEFINED_VERSION or version is None:
+ if version is None or version.is_default:
operator = self.DEFAULT_UNDEFINED_VERSION_OPERATOR
else:
operator = self.DEFAULT_UNDEFINED_OPERATOR
@@ -640,7 +648,7 @@ def add(self, versop_new, data=None, update=None):
:param update: if versop_new already exist and has data set, try to update the existing data with the new data;
instead of overriding the existing data with the new data (method used for updating is .update)
"""
- if isinstance(versop_new, string_type):
+ if isinstance(versop_new, str):
versop_new = VersionOperator(versop_new)
elif not isinstance(versop_new, VersionOperator):
raise EasyBuildError("add: argument must be a VersionOperator instance or string: %s; type %s",
diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py
deleted file mode 100644
index 675d0f0a86..0000000000
--- a/easybuild/framework/easyconfig/format/yeb.py
+++ /dev/null
@@ -1,169 +0,0 @@
-# #
-# Copyright 2013-2024 Ghent University
-#
-# This file is part of EasyBuild,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/easybuilders/easybuild
-#
-# EasyBuild is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation v2.
-#
-# EasyBuild is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with EasyBuild. If not, see .
-# #
-"""
-YAML easyconfig format (.yeb)
-Useful: http://www.yaml.org/spec/1.2/spec.html
-
-Authors:
-
-* Caroline De Brouwer (Ghent University)
-* Kenneth Hoste (Ghent University)
-"""
-import copy
-import os
-import platform
-
-from easybuild.base import fancylogger
-from easybuild.framework.easyconfig.format.format import EasyConfigFormat
-from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
-from easybuild.tools import LooseVersion
-from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.utilities import INDENT_4SPACES, only_if_module_is_available, quote_str
-
-
-_log = fancylogger.getLogger('easyconfig.format.yeb', fname=False)
-
-
-YAML_DIR = r'%YAML'
-YAML_SEP = '---'
-YEB_FORMAT_EXTENSION = '.yeb'
-YAML_SPECIAL_CHARS = set(":{}[],&*#?|-<>=!%@\\")
-
-
-def yaml_join(loader, node):
- """
- defines custom YAML join function.
- see http://stackoverflow.com/questions/5484016/
- how-can-i-do-string-concatenation-or-string-replacement-in-yaml/23212524#23212524
- :param loader: the YAML Loader
- :param node: the YAML (sequence) node
- """
- seq = loader.construct_sequence(node)
- return ''.join([str(i) for i in seq])
-
-
-try:
- import yaml
- # register the tag handlers
- if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'):
- yaml.add_constructor('!join', yaml_join)
- else:
- yaml.add_constructor(u'!join', yaml_join, Loader=yaml.SafeLoader)
-except ImportError:
- pass
-
-
-class FormatYeb(EasyConfigFormat):
- """Support for easyconfig YAML format"""
- USABLE = True
-
- def __init__(self):
- """FormatYeb constructor"""
- super(FormatYeb, self).__init__()
- self.log.experimental("Parsing .yeb easyconfigs")
-
- def validate(self):
- """Format validation"""
- _log.info(".yeb format validation isn't implemented (yet) - validation always passes")
- return True
-
- def get_config_dict(self):
- """
- Return parsed easyconfig as a dictionary, based on specified arguments.
- """
- # avoid passing anything by reference, so next time get_config_dict is called
- # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file
- return copy.deepcopy(self.parsed_yeb)
-
- @only_if_module_is_available('yaml')
- def parse(self, txt):
- """
- Process YAML file
- """
- txt = self._inject_constants_dict(txt)
- if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'):
- self.parsed_yeb = yaml.load(txt)
- else:
- self.parsed_yeb = yaml.load(txt, Loader=yaml.SafeLoader)
-
- def _inject_constants_dict(self, txt):
- """Inject constants so they are resolved when actually parsing the YAML text."""
- constants_dict = build_easyconfig_constants_dict()
-
- lines = txt.splitlines()
-
- # extract possible YAML header, for example
- # %YAML 1.2
- # ---
- yaml_header = []
- for i, line in enumerate(lines):
- if line.startswith(YAML_DIR):
- if lines[i + 1].startswith(YAML_SEP):
- yaml_header.extend([lines.pop(i), lines.pop(i)])
-
- injected_constants = ['__CONSTANTS__: ']
- for key, value in constants_dict.items():
- injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))
-
- full_txt = '\n'.join(yaml_header + injected_constants + lines)
-
- return full_txt
-
- def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None):
- """Dump parsed easyconfig in .yeb format"""
- raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet")
-
- def extract_comments(self, txt):
- """Extract comments from easyconfig file"""
- self.log.debug("Not extracting comments from .yeb easyconfigs")
-
-
-def is_yeb_format(filename, rawcontent):
- """
- Determine whether easyconfig is in .yeb format.
- If filename is None, rawcontent will be used to check the format.
- """
- isyeb = False
- if filename:
- isyeb = os.path.splitext(filename)[-1] == YEB_FORMAT_EXTENSION
- else:
- # if one line like 'name: ' is found, this must be YAML format
- for line in rawcontent.splitlines():
- if line.startswith('name: '):
- isyeb = True
- return isyeb
-
-
-def quote_yaml_special_chars(val):
- """
- Single-quote values that contain special characters, specifically to be used in YAML context (.yeb files)
- Single quotes inside the string are escaped by doubling them.
- (see: http://symfony.com/doc/current/components/yaml/yaml_format.html#strings)
- """
- if isinstance(val, string_type):
- if "'" in val or YAML_SPECIAL_CHARS.intersection(val):
- val = "'%s'" % val.replace("'", "''")
-
- return val
diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py
index e6286d54e3..d1f7ac787a 100644
--- a/easybuild/framework/easyconfig/licenses.py
+++ b/easybuild/framework/easyconfig/licenses.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py
index d6521b17dc..f5914d65a4 100644
--- a/easybuild/framework/easyconfig/parser.py
+++ b/easybuild/framework/easyconfig/parser.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,16 +37,74 @@
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION
from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes
-from easybuild.framework.easyconfig.format.yeb import FormatYeb, is_yeb_format
from easybuild.framework.easyconfig.types import PARAMETER_TYPES, check_type_of_param_value
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file, write_file
-from easybuild.tools.py2vs3 import string_type
+# alternative easyconfig parameters, and their non-deprecated equivalents
+ALTERNATIVE_EASYCONFIG_PARAMETERS = {
+ # : ,
+ 'build_deps': 'builddependencies',
+ 'build_in_install_dir': 'buildininstalldir',
+ 'build_opts': 'buildopts',
+ 'build_stats': 'buildstats',
+ 'clean_up_old_build': 'cleanupoldbuild',
+ 'clean_up_old_install': 'cleanupoldinstall',
+ 'configure_opts': 'configopts',
+ 'deps': 'dependencies',
+ 'doc_paths': 'docpaths',
+ 'doc_urls': 'docurls',
+ 'do_not_create_install_dir': 'dontcreateinstalldir',
+ 'exts_class_map': 'exts_classmap',
+ 'exts_default_class': 'exts_defaultclass',
+ 'exts_default_opts': 'exts_default_options',
+ 'hidden_deps': 'hiddendependencies',
+ 'include_modulepath_exts': 'include_modpath_extensions',
+ 'install_opts': 'installopts',
+ 'keep_previous_install': 'keeppreviousinstall',
+ 'keep_symlinks': 'keepsymlinks',
+ 'max_parallel': 'maxparallel',
+ 'env_mod_aliases': 'modaliases',
+ 'env_mod_alt_soft_name': 'modaltsoftname',
+ 'modulepath_prepend_paths': 'moddependpaths',
+ 'env_mod_extra_paths': 'modextrapaths',
+ 'env_mod_extra_vars': 'modextravars',
+ 'env_mod_load_msg': 'modloadmsg',
+ 'env_mod_lua_footer': 'modluafooter',
+ 'env_mod_tcl_footer': 'modtclfooter',
+ 'env_mod_category': 'moduleclass',
+ 'env_mod_depends_on': 'module_depends_on',
+ 'env_mod_force_unload': 'moduleforceunload',
+ 'env_mod_load_no_conflict': 'moduleloadnoconflict',
+ 'env_mod_unload_msg': 'modunloadmsg',
+ 'only_toolchain_env_mod': 'onlytcmod',
+ 'os_deps': 'osdependencies',
+ 'post_install_cmds': 'postinstallcmds',
+ 'post_install_msgs': 'postinstallmsgs',
+ 'post_install_patches': 'postinstallpatches',
+ 'pre_build_opts': 'prebuildopts',
+ 'pre_configure_opts': 'preconfigopts',
+ 'pre_install_opts': 'preinstallopts',
+ 'pre_test_opts': 'pretestopts',
+ 'recursive_env_mod_unload': 'recursive_module_unload',
+ 'run_test': 'runtest',
+ 'sanity_check_cmds': 'sanity_check_commands',
+ 'skip_fortran_mod_files_sanity_check': 'skip_mod_files_sanity_check',
+ 'skip_steps': 'skipsteps',
+ 'test_opts': 'testopts',
+ 'toolchain_opts': 'toolchainopts',
+ 'unpack_opts': 'unpack_options',
+ 'version_prefix': 'versionprefix',
+ 'version_suffix': 'versionsuffix',
+}
+
# deprecated easyconfig parameters, and their replacements
-DEPRECATED_PARAMETERS = {
+DEPRECATED_EASYCONFIG_PARAMETERS = {
# : (, ),
+ 'allow_append_abs_path': ('modextrapaths', '6.0'),
+ 'allow_prepend_abs_path': ('modextrapaths', '6.0'),
+ 'modextrapaths_append': ('modextrapaths', '6.0'),
}
# replaced easyconfig parameters, and their replacements
@@ -161,7 +219,7 @@ def _read(self, filename=None):
except IOError as err:
raise EasyBuildError('Failed to obtain content with %s: %s', self.get_fn, err)
- if not isinstance(self.rawcontent, string_type):
+ if not isinstance(self.rawcontent, str):
msg = 'rawcontent is not a string: type %s, content %s' % (type(self.rawcontent), self.rawcontent)
raise EasyBuildError("Unexpected result for raw content: %s", msg)
@@ -189,11 +247,8 @@ def _get_format_version_class(self):
def _set_formatter(self, filename):
"""Obtain instance of the formatter"""
if self._formatter is None:
- if is_yeb_format(filename, self.rawcontent):
- self._formatter = FormatYeb()
- else:
- klass = self._get_format_version_class()
- self._formatter = klass()
+ klass = self._get_format_version_class()
+ self._formatter = klass()
self._formatter.parse(self.rawcontent)
def set_format_text(self):
diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py
index bad6ab5bda..87c6780e12 100644
--- a/easybuild/framework/easyconfig/style.py
+++ b/easybuild/framework/easyconfig/style.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,24 +31,18 @@
* Ward Poelmans (Ghent University)
"""
import re
-import sys
+from importlib import reload
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.easyconfig import EasyConfig
from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.py2vs3 import reload, string_type
from easybuild.tools.utilities import only_if_module_is_available
try:
import pycodestyle
from pycodestyle import StyleGuide, register_check, trailing_whitespace
except ImportError:
- try:
- # fallback to importing from 'pep8', which was renamed to pycodestyle in 2016
- import pep8
- from pep8 import StyleGuide, register_check, trailing_whitespace
- except ImportError:
- pass
+ pass
_log = fancylogger.getLogger('easyconfig.style', fname=False)
@@ -100,13 +94,13 @@ def _eb_check_trailing_whitespace(physical_line, lines, line_number, checker_sta
# if the warning is about the multiline string of description
# we will not issue a warning
- if checker_state.get('eb_last_key') == 'description':
+ if checker_state.get('eb_last_key') in ['description', 'examples', 'citing']:
result = None
return result
-@only_if_module_is_available(('pycodestyle', 'pep8'))
+@only_if_module_is_available('pycodestyle')
def check_easyconfigs_style(easyconfigs, verbose=False):
"""
Check the given list of easyconfigs for style
@@ -114,12 +108,7 @@ def check_easyconfigs_style(easyconfigs, verbose=False):
:param verbose: print our statistics and be verbose about the errors and warning
:return: the number of warnings and errors
"""
- # importing autopep8 changes some pep8 functions.
- # We reload it to be sure to get the real pep8 functions.
- if 'pycodestyle' in sys.modules:
- reload(pycodestyle)
- else:
- reload(pep8)
+ reload(pycodestyle)
# register the extra checks before using pep8:
# any function in this module starting with `_eb_check_` will be used.
@@ -137,6 +126,7 @@ def check_easyconfigs_style(easyconfigs, verbose=False):
# note that W291 has been replaced by our custom W299
options.ignore = (
'W291', # replaced by W299
+ 'E741', # 'l' is considered an ambiguous name, but we use it often for 'lib'
)
options.verbose = int(verbose)
@@ -161,7 +151,7 @@ def cmdline_easyconfigs_style_check(ecs):
# if an EasyConfig instance is provided, just grab the corresponding file path
if isinstance(ec, EasyConfig):
path = ec.path
- elif isinstance(ec, string_type):
+ elif isinstance(ec, str):
path = ec
else:
raise EasyBuildError("Value of unknown type encountered in cmdline_easyconfigs_style_check: %s (type: %s)",
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
index be9bb28bd3..46fe4c5970 100644
--- a/easybuild/framework/easyconfig/templates.py
+++ b/easybuild/framework/easyconfig/templates.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,7 +38,6 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.systemtools import get_shared_lib_ext, pick_dep_version
from easybuild.tools.config import build_option
@@ -46,21 +45,20 @@
_log = fancylogger.getLogger('easyconfig.templates', fname=False)
# derived from easyconfig, but not from ._config directly
-TEMPLATE_NAMES_EASYCONFIG = [
- ('module_name', "Module name"),
- ('nameletter', "First letter of software name"),
- ('toolchain_name', "Toolchain name"),
- ('toolchain_version', "Toolchain version"),
- ('version_major_minor', "Major.Minor version"),
- ('version_major', "Major version"),
- ('version_minor', "Minor version"),
-]
+TEMPLATE_NAMES_EASYCONFIG = {
+ 'module_name': 'Module name',
+ 'nameletter': 'First letter of software name',
+ 'toolchain_name': 'Toolchain name',
+ 'toolchain_version': 'Toolchain version',
+ 'version_major_minor': "Major.Minor version",
+ 'version_major': 'Major version',
+ 'version_minor': 'Minor version',
+}
# derived from EasyConfig._config
TEMPLATE_NAMES_CONFIG = [
'bitbucket_account',
'github_account',
'name',
- 'parallel',
'version',
'versionsuffix',
'versionprefix',
@@ -72,105 +70,202 @@
'nameletter',
]
# values taken from the EasyBlock before each step
-TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [
- ('builddir', "Build directory"),
- ('installdir', "Installation directory"),
- ('start_dir', "Directory in which the build process begins"),
-]
+TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = {
+ 'builddir': 'Build directory',
+ 'installdir': 'Installation directory',
+ 'start_dir': 'Directory in which the build process begins',
+}
# software names for which to define ver, majver and shortver templates
-TEMPLATE_SOFTWARE_VERSIONS = [
- # software name, prefix for *ver, *majver and *shortver
- ('CUDA', 'cuda'),
- ('CUDAcore', 'cuda'),
- ('Java', 'java'),
- ('Perl', 'perl'),
- ('Python', 'py'),
- ('R', 'r'),
-]
+TEMPLATE_SOFTWARE_VERSIONS = {
+ # software name -> prefix for *ver, *majver and *shortver
+ 'CUDA': 'cuda',
+ 'CUDAcore': 'cuda',
+ 'Java': 'java',
+ 'Perl': 'perl',
+ 'Python': 'py',
+ 'R': 'r',
+}
# template values which are only generated dynamically
-TEMPLATE_NAMES_DYNAMIC = [
- ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
- ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
- "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
- ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
- ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
- ('cuda_cc_space_sep_no_period',
- "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."),
- ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
- ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
- ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
- ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
- ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"),
- ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
- "as specified by the --sysroot configuration option"),
-]
+TEMPLATE_NAMES_DYNAMIC = {
+ 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)',
+ 'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via "
+ "--cuda-compute-capabilities configuration option or "
+ "via cuda_compute_capabilities easyconfig parameter",
+ 'cuda_cc_cmake': 'List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+',
+ 'cuda_cc_space_sep': 'Space-separated list of CUDA compute capabilities',
+ 'cuda_cc_space_sep_no_period':
+ "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90').",
+ 'cuda_cc_semicolon_sep': 'Semicolon-separated list of CUDA compute capabilities',
+ 'cuda_int_comma_sep': 'Comma-separated list of integer CUDA compute capabilities',
+ 'cuda_int_space_sep': 'Space-separated list of integer CUDA compute capabilities',
+ 'cuda_int_semicolon_sep': 'Semicolon-separated list of integer CUDA compute capabilities',
+ 'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities',
+ 'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities',
+ 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)',
+ 'parallel': "Degree of parallelism for e.g. make",
+ # can't be a boolean (True/False), must be a string value since it's a string template
+ 'rpath_enabled': "String value indicating whether or not RPATH linking is used ('true' or 'false')",
+ 'software_commit': "Git commit id to use for the software as specified by --software-commit command line option",
+ 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
+ "as specify by the --sysroot configuration option",
+}
# constant templates that can be used in easyconfigs
-TEMPLATE_CONSTANTS = [
+# Entry: constant -> (value, doc)
+TEMPLATE_CONSTANTS = {
# source url constants
- ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s',
- 'apache.org source url'),
- ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
- 'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
- 'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib',
- 'CRAN (contrib) source url'),
- ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
- 'http download for gnome ftp server'),
- ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive',
- 'GitHub source URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
- ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
- 'GitHub source URL with lowercase name (if github_account easyconfig '
- 'parameter is not specified, namelower is used in its place)'),
- ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
- 'GitHub release URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
- ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
- 'GitHub release URL with lowercase name (if github_account easyconfig '
- 'parameter is not specified, namelower is used in its place)'),
- ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
- 'download.savannah.gnu.org source url'),
- ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
- 'gnu.org source url (ftp mirror)'),
- ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s',
- 'gnu.org source url (main ftp)'),
- ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
- 'googlecode.com source url'),
- ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
- 'launchpad.net source url'),
- ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s',
- 'pypi source url'), # e.g., Cython, Sphinx
- ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
- 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
- ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s',
- 'cran.r-project.org (base) source url'),
- ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s',
- 'sourceforge.net source url'),
- ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/',
- 'xorg data source url'),
- ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/',
- 'xorg lib source url'),
- ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/',
- 'xorg proto source url'),
- ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/',
- 'xorg util source url'),
- ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/',
- 'xorg xcb source url'),
+ 'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s',
+ 'apache.org source url'),
+ 'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
+ 'bitbucket.org source url '
+ '(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
+ 'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
+ 'bitbucket.org downloads url '
+ '(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
+ 'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib',
+ 'CRAN (contrib) source url'),
+ 'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
+ 'http download for gnome ftp server'),
+ 'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive',
+ 'GitHub source URL '
+ '(namelower is used if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_LOWER_SOURCE': ('https://github.com/%(github_account)s/%(namelower)s/archive',
+ 'GitHub source URL with lowercase name '
+ '(namelower is used if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_RELEASE': ('https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
+ 'GitHub release URL '
+ '(namelower is use if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_LOWER_RELEASE': ('https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
+ 'GitHub release URL with lowercase name (if github_account easyconfig '
+ 'parameter is not specified, namelower is used in its place)'),
+ 'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
+ 'download.savannah.gnu.org source url'),
+ 'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s',
+ 'gnu.org source url (ftp mirror)'),
+ 'GNU_FTP_SOURCE': ('https://ftp.gnu.org/gnu/%(namelower)s',
+ 'gnu.org source url (main ftp)'),
+ 'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files',
+ 'googlecode.com source url'),
+ 'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
+ 'launchpad.net source url'),
+ 'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s',
+ 'pypi source url'), # e.g., Cython, Sphinx
+ 'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
+ 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
+ 'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s',
+ 'cran.r-project.org (base) source url'),
+ 'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s',
+ 'sourceforge.net source url'),
+ 'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/',
+ 'xorg data source url'),
+ 'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/',
+ 'xorg lib source url'),
+ 'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/',
+ 'xorg proto source url'),
+ 'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/',
+ 'xorg util source url'),
+ 'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/',
+ 'xorg xcb source url'),
# TODO, not urgent, yet nice to have:
# CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL
# other constants
- ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'),
-]
-
-extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
-for ext in extensions:
+ 'SHLIB_EXT': (get_shared_lib_ext(), 'extension for shared libraries'),
+}
+
+# alternative templates, and their equivalents
+ALTERNATIVE_EASYCONFIG_TEMPLATES = {
+ # : ,
+ 'build_dir': 'builddir',
+ 'cuda_cc_comma_sep': 'cuda_compute_capabilities',
+ 'cuda_maj_ver': 'cudamajver',
+ 'cuda_short_ver': 'cudashortver',
+ 'cuda_ver': 'cudaver',
+ 'install_dir': 'installdir',
+ 'java_maj_ver': 'javamajver',
+ 'java_short_ver': 'javashortver',
+ 'java_ver': 'javaver',
+ 'name_letter_lower': 'nameletterlower',
+ 'name_letter': 'nameletter',
+ 'name_lower': 'namelower',
+ 'perl_maj_ver': 'perlmajver',
+ 'perl_short_ver': 'perlshortver',
+ 'perl_ver': 'perlver',
+ 'py_maj_ver': 'pymajver',
+ 'py_short_ver': 'pyshortver',
+ 'py_ver': 'pyver',
+ 'r_maj_ver': 'rmajver',
+ 'r_short_ver': 'rshortver',
+ 'r_ver': 'rver',
+ 'toolchain_ver': 'toolchain_version',
+ 'ver_maj_min': 'version_major_minor',
+ 'ver_maj': 'version_major',
+ 'ver_min': 'version_minor',
+ 'version_prefix': 'versionprefix',
+ 'version_suffix': 'versionsuffix',
+}
+
+# deprecated templates, and their replacements
+DEPRECATED_EASYCONFIG_TEMPLATES = {
+ # : (, ),
+}
+
+# alternative template constants, and their equivalents
+ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS = {
+ # : ,
+ 'APACHE_URL': 'APACHE_SOURCE',
+ 'BITBUCKET_GET_URL': 'BITBUCKET_SOURCE',
+ 'BITBUCKET_DOWNLOADS_URL': 'BITBUCKET_DOWNLOADS',
+ 'CRAN_URL': 'CRAN_SOURCE',
+ 'FTP_GNOME_URL': 'FTPGNOME_SOURCE',
+ 'GITHUB_URL': 'GITHUB_SOURCE',
+ 'GITHUB_URL_LOWER': 'GITHUB_LOWER_SOURCE',
+ 'GITHUB_RELEASE_URL': 'GITHUB_RELEASE',
+ 'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE',
+ 'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE',
+ 'GNU_FTP_URL': 'GNU_FTP_SOURCE',
+ 'GNU_URL': 'GNU_SOURCE',
+ 'GOOGLECODE_URL': 'GOOGLECODE_SOURCE',
+ 'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE',
+ 'PYPI_URL': 'PYPI_SOURCE',
+ 'PYPI_URL_LOWER': 'PYPI_LOWER_SOURCE',
+ 'R_URL': 'R_SOURCE',
+ 'SOURCEFORGE_URL': 'SOURCEFORGE_SOURCE',
+ 'XORG_DATA_URL': 'XORG_DATA_SOURCE',
+ 'XORG_LIB_URL': 'XORG_LIB_SOURCE',
+ 'XORG_PROTO_URL': 'XORG_PROTO_SOURCE',
+ 'XORG_UTIL_URL': 'XORG_UTIL_SOURCE',
+ 'XORG_XCB_URL': 'XORG_XCB_SOURCE',
+ 'SOURCE_LOWER_TAR_GZ': 'SOURCELOWER_TAR_GZ',
+ 'SOURCE_LOWER_TAR_XZ': 'SOURCELOWER_TAR_XZ',
+ 'SOURCE_LOWER_TAR_BZ2': 'SOURCELOWER_TAR_BZ2',
+ 'SOURCE_LOWER_TGZ': 'SOURCELOWER_TGZ',
+ 'SOURCE_LOWER_TXZ': 'SOURCELOWER_TXZ',
+ 'SOURCE_LOWER_TBZ2': 'SOURCELOWER_TBZ2',
+ 'SOURCE_LOWER_TB2': 'SOURCELOWER_TB2',
+ 'SOURCE_LOWER_GTGZ': 'SOURCELOWER_GTGZ',
+ 'SOURCE_LOWER_ZIP': 'SOURCELOWER_ZIP',
+ 'SOURCE_LOWER_TAR': 'SOURCELOWER_TAR',
+ 'SOURCE_LOWER_XZ': 'SOURCELOWER_XZ',
+ 'SOURCE_LOWER_TAR_Z': 'SOURCELOWER_TAR_Z',
+ 'SOURCE_LOWER_WHL': 'SOURCELOWER_WHL',
+ 'SOURCE_LOWER_PY2_WHL': 'SOURCELOWER_PY2_WHL',
+ 'SOURCE_LOWER_PY3_WHL': 'SOURCELOWER_PY3_WHL',
+}
+
+# deprecated template constants, and their replacements
+DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS = {
+ # : (, ),
+}
+
+EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
+for ext in EXTENSIONS:
suffix = ext.replace('.', '_').upper()
- TEMPLATE_CONSTANTS += [
- ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
- ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
- ]
+ TEMPLATE_CONSTANTS.update({
+ 'SOURCE_%s' % suffix: ('%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
+ 'SOURCELOWER_%s' % suffix: ('%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
+ })
for pyver in ('py2.py3', 'py2', 'py3'):
if pyver == 'py2.py3':
desc = 'Python 2 & Python 3'
@@ -178,26 +273,22 @@
else:
desc = 'Python ' + pyver[-1]
name_infix = pyver.upper() + '_'
- TEMPLATE_CONSTANTS += [
- ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
- 'Generic (non-compiled) %s wheel package' % desc),
- ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
- 'Generic (non-compiled) %s wheel package with lowercase name' % desc),
- ]
+ TEMPLATE_CONSTANTS.update({
+ 'SOURCE_%sWHL' % name_infix: ('%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
+ 'Generic (non-compiled) %s wheel package' % desc),
+ 'SOURCELOWER_%sWHL' % name_infix: ('%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
+ 'Generic (non-compiled) %s wheel package with lowercase name' % desc),
+ })
# TODO derived config templates
# versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) )
-def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None):
+def template_constant_dict(config, ignore=None, toolchain=None):
"""Create a dict for templating the values in the easyconfigs.
- - config is a dict with the structure of EasyConfig._config
+ - config -- Dict with the structure of EasyConfig._config
+ - ignore -- List of template names to ignore
"""
- if skip_lower is not None:
- _log.deprecated("Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0')
-
- # TODO find better name
- # ignore
if ignore is None:
ignore = []
# make dict
@@ -208,6 +299,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
template_values['arch'] = platform.uname()[4]
+ # set 'rpath' template based on 'rpath' configuration option, using empty string as fallback
+ template_values['rpath_enabled'] = 'true' if build_option('rpath') else 'false'
+
# set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
template_values['sysroot'] = build_option('sysroot') or ''
@@ -220,10 +314,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
continue
# check if this template name is already handled
- if template_values.get(name[0]) is not None:
+ if template_values.get(name) is not None:
continue
- if name[0].startswith('toolchain_'):
+ if name.startswith('toolchain_'):
tc = config.get('toolchain')
if tc is not None:
template_values['toolchain_name'] = tc.get('name', None)
@@ -231,7 +325,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
# only go through this once
ignore.extend(['toolchain_name', 'toolchain_version'])
- elif name[0].startswith('version_'):
+ elif name.startswith('version_'):
# parse major and minor version numbers
version = config['version']
if version is not None:
@@ -250,87 +344,85 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
# only go through this once
ignore.extend(['version_major', 'version_minor', 'version_major_minor'])
- elif name[0].endswith('letter'):
+ elif name.endswith('letter'):
# parse first letters
- if name[0].startswith('name'):
+ if name.startswith('name'):
softname = config['name']
if softname is not None:
template_values['nameletter'] = softname[0]
- elif name[0] == 'module_name':
+ elif name == 'module_name':
template_values['module_name'] = getattr(config, 'short_mod_name', None)
else:
raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
# step 2: define *ver and *shortver templates
- if TEMPLATE_SOFTWARE_VERSIONS:
-
- name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS}
- deps = config.get('dependencies', [])
-
- # also consider build dependencies for *ver and *shortver templates;
- # we need to be a bit careful here, because for iterative installations
- # (when multi_deps is used for example) the builddependencies value may be a list of lists
-
- # first, determine if we have an EasyConfig instance
- # (indirectly by checking for 'iterating' and 'iterate_options' attributes,
- # because we can't import the EasyConfig class here without introducing
- # a cyclic import...);
- # we need to know to determine whether we're iterating over a list of build dependencies
- is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options')
- if is_easyconfig:
- # if we're iterating over different lists of build dependencies,
- # only consider build dependencies when we're actually in iterative mode!
- if 'builddependencies' in config.iterate_options:
- if config.iterating:
- build_deps = config.get('builddependencies')
- else:
- build_deps = None
- else:
+ name_to_prefix = {name.lower(): prefix for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items()}
+ deps = config.get('dependencies', [])
+
+ # also consider build dependencies for *ver and *shortver templates;
+ # we need to be a bit careful here, because for iterative installations
+ # (when multi_deps is used for example) the builddependencies value may be a list of lists
+
+ # first, determine if we have an EasyConfig instance
+ # (indirectly by checking for 'iterating' and 'iterate_options' attributes,
+ # because we can't import the EasyConfig class here without introducing
+ # a cyclic import...);
+ # we need to know to determine whether we're iterating over a list of build dependencies
+ is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options')
+ if is_easyconfig:
+ # if we're iterating over different lists of build dependencies,
+ # only consider build dependencies when we're actually in iterative mode!
+ if 'builddependencies' in config.iterate_options:
+ if config.iterating:
build_deps = config.get('builddependencies')
+ else:
+ build_deps = None
+ else:
+ build_deps = config.get('builddependencies')
+ if build_deps:
+ # Don't use += to avoid changing original list
+ deps = deps + build_deps
+ # include all toolchain deps (e.g. CUDAcore component in fosscuda);
+ # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
+ if config._toolchain is not None and config._toolchain.tcdeps:
+ # If we didn't create a new list above do it here
if build_deps:
- # Don't use += to avoid changing original list
- deps = deps + build_deps
- # include all toolchain deps (e.g. CUDAcore component in fosscuda);
- # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
- if config._toolchain is not None and config._toolchain.tcdeps:
- # If we didn't create a new list above do it here
- if build_deps:
- deps.extend(config._toolchain.tcdeps)
- else:
- deps = deps + config._toolchain.tcdeps
-
- for dep in deps:
- if isinstance(dep, dict):
- dep_name, dep_version = dep['name'], dep['version']
-
- # take into account dependencies marked as external modules,
- # where name/version may have to be harvested from metadata available for that external module
- if dep.get('external_module', False):
- metadata = dep.get('external_module_metadata', {})
- if dep_name is None:
- # name is a list in metadata, just take first value (if any)
- dep_name = metadata.get('name', [None])[0]
- if dep_version is None:
- # version is a list in metadata, just take first value (if any)
- dep_version = metadata.get('version', [None])[0]
-
- elif isinstance(dep, (list, tuple)):
- dep_name, dep_version = dep[0], dep[1]
+ deps.extend(config._toolchain.tcdeps)
else:
- raise EasyBuildError("Unexpected type for dependency: %s", dep)
-
- if isinstance(dep_name, string_type) and dep_version:
- pref = name_to_prefix.get(dep_name.lower())
- if pref:
- dep_version = pick_dep_version(dep_version)
- template_values['%sver' % pref] = dep_version
- dep_version_parts = dep_version.split('.')
- template_values['%smajver' % pref] = dep_version_parts[0]
- if len(dep_version_parts) > 1:
- template_values['%sminver' % pref] = dep_version_parts[1]
- template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
+ deps = deps + config._toolchain.tcdeps
+
+ for dep in deps:
+ if isinstance(dep, dict):
+ dep_name, dep_version = dep['name'], dep['version']
+
+ # take into account dependencies marked as external modules,
+ # where name/version may have to be harvested from metadata available for that external module
+ if dep.get('external_module', False):
+ metadata = dep.get('external_module_metadata', {})
+ if dep_name is None:
+ # name is a list in metadata, just take first value (if any)
+ dep_name = metadata.get('name', [None])[0]
+ if dep_version is None:
+ # version is a list in metadata, just take first value (if any)
+ dep_version = metadata.get('version', [None])[0]
+
+ elif isinstance(dep, (list, tuple)):
+ dep_name, dep_version = dep[0], dep[1]
+ else:
+ raise EasyBuildError("Unexpected type for dependency: %s", dep)
+
+ if isinstance(dep_name, str) and dep_version:
+ pref = name_to_prefix.get(dep_name.lower())
+ if pref:
+ dep_version = pick_dep_version(dep_version)
+ template_values['%sver' % pref] = dep_version
+ dep_version_parts = dep_version.split('.')
+ template_values['%smajver' % pref] = dep_version_parts[0]
+ if len(dep_version_parts) > 1:
+ template_values['%sminver' % pref] = dep_version_parts[1]
+ template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
# step 3: add remaining from config
for name in TEMPLATE_NAMES_CONFIG:
@@ -378,14 +470,17 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc)
template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc)
template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc)
+ int_values = [cc.replace('.', '') for cc in cuda_cc]
+ template_values['cuda_int_comma_sep'] = ','.join(int_values)
+ template_values['cuda_int_space_sep'] = ' '.join(int_values)
+ template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc]
template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
unknown_names = []
for key in template_values:
- dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC)
- if not (key in common_template_names or key in dynamic_template_names):
+ if not (key in common_template_names or key in TEMPLATE_NAMES_DYNAMIC):
unknown_names.append(key)
if unknown_names:
raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names))
@@ -435,15 +530,15 @@ def template_documentation():
# step 1: add TEMPLATE_NAMES_EASYCONFIG
doc.append('Template names/values derived from easyconfig instance')
- for name in TEMPLATE_NAMES_EASYCONFIG:
- doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYCONFIG.items():
+ doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc))
# step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS
doc.append("Template names/values for (short) software versions")
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name))
- doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name))
- doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name))
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, prefix, name))
+ doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, prefix, name))
+ doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, prefix, name))
# step 3: add remaining self._config
doc.append('Template names/values as set in easyconfig')
@@ -459,11 +554,16 @@ def template_documentation():
# step 5. self.template_values can/should be updated from outside easyconfig
# (eg the run_setp code in EasyBlock)
doc.append('Template values set outside EasyBlock runstep')
- for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items():
+ doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc))
doc.append('Template constants that can be used in easyconfigs')
- for cst in TEMPLATE_CONSTANTS:
- doc.append('%s%s: %s (%s)' % (indent_l1, cst[0], cst[2], cst[1]))
+ for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items():
+ doc.append('%s%s: %s (%s)' % (indent_l1, name, cur_doc, value))
return "\n".join(doc)
+
+
+# Add template constants to export list
+globals().update({name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()})
+__all__ = list(TEMPLATE_CONSTANTS.keys())
diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index cb92b36b76..3d85b931ea 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -52,14 +52,13 @@
from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig
from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class
from easybuild.framework.easyconfig.easyconfig import process_easyconfig
-from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
-from easybuild.tools.filetools import read_file, resolve_path, which, write_file
+from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, get_cwd, find_easyconfigs, is_patch_file
+from easybuild.tools.filetools import locate_files, read_file, resolve_path, which, write_file
from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
@@ -404,13 +403,19 @@ def parse_easyconfigs(paths, validate=True):
"""
easyconfigs = []
generated_ecs = False
+ parsed_paths = []
for (path, generated) in paths:
+ # Avoid processing the same file multiple times
path = os.path.abspath(path)
+ if any(os.path.samefile(path, p) for p in parsed_paths):
+ continue
+ parsed_paths.append(path)
+
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
- raise EasyBuildError("Can't find path %s", path)
+ raise EasyBuildError("Can't find path %s", path, exit_code=EasyBuildExit.MISSING_EASYCONFIG)
try:
ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
@@ -427,7 +432,7 @@ def parse_easyconfigs(paths, validate=True):
return easyconfigs, generated_ecs
-def stats_to_str(stats, isyeb=False):
+def stats_to_str(stats):
"""
Pretty print build statistics to string.
"""
@@ -437,13 +442,7 @@ def stats_to_str(stats, isyeb=False):
txt = "{\n"
pref = " "
for key in sorted(stats):
- if isyeb:
- val = stats[key]
- if isinstance(val, tuple):
- val = list(val)
- key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(val)
- else:
- key, val = quote_str(key), quote_str(stats[key])
+ key, val = quote_str(key), quote_str(stats[key])
txt += "%s%s: %s,\n" % (pref, key, val)
txt += "}"
return txt
@@ -758,7 +757,7 @@ def avail_easyblocks():
"""Return a list of all available easyblocks."""
module_regexp = re.compile(r"^([^_].*)\.py$")
- class_regex = re.compile(r"^class ([^(]*)\(", re.M)
+ class_regex = re.compile(r"^class ([^(:]*)\(", re.M)
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
@@ -768,33 +767,43 @@ def avail_easyblocks():
__import__(pkg)
# determine paths for this package
- paths = sys.modules[pkg].__path__
+ paths = [path for path in sys.modules[pkg].__path__ if os.path.exists(path)]
# import all modules in these paths
for path in paths:
- if os.path.exists(path):
- for fn in os.listdir(path):
- res = module_regexp.match(fn)
- if res:
- easyblock_mod_name = '%s.%s' % (pkg, res.group(1))
-
- if easyblock_mod_name not in easyblocks:
- __import__(easyblock_mod_name)
- easyblock_loc = os.path.join(path, fn)
-
- class_names = class_regex.findall(read_file(easyblock_loc))
- if len(class_names) == 1:
- easyblock_class = class_names[0]
- elif class_names:
- raise EasyBuildError("Found multiple class names for easyblock %s: %s",
- easyblock_loc, class_names)
- else:
- raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
-
- easyblocks[easyblock_mod_name] = {'class': easyblock_class, 'loc': easyblock_loc}
+ for fn in os.listdir(path):
+ res = module_regexp.match(fn)
+ if not res:
+ continue
+ easyblock_mod_name = res.group(1)
+ easyblock_full_mod_name = '%s.%s' % (pkg, easyblock_mod_name)
+
+ if easyblock_full_mod_name in easyblocks:
+ _log.debug("%s already imported from %s, ignoring %s",
+ easyblock_full_mod_name, easyblocks[easyblock_full_mod_name]['loc'], path)
+ else:
+ __import__(easyblock_full_mod_name)
+ easyblock_loc = os.path.join(path, fn)
+
+ class_names = class_regex.findall(read_file(easyblock_loc))
+ if len(class_names) > 1:
+ if pkg.endswith('.generic'):
+ # In generic easyblocks we have e.g. ConfigureMake in configuremake.py
+ sw_specific_class_names = [name for name in class_names
+ if name.lower() == easyblock_mod_name.lower()]
else:
- _log.debug("%s already imported from %s, ignoring %s",
- easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
+ # If there is exactly one software specific easyblock we use that
+ sw_specific_class_names = [name for name in class_names
+ if name.startswith(EASYBLOCK_CLASS_PREFIX)]
+ if len(sw_specific_class_names) == 1:
+ class_names = sw_specific_class_names
+ if len(class_names) == 1:
+ easyblocks[easyblock_full_mod_name] = {'class': class_names[0], 'loc': easyblock_loc}
+ elif class_names:
+ raise EasyBuildError("Found multiple class names for easyblock %s: %s",
+ easyblock_loc, class_names)
+ else:
+ raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
return easyblocks
@@ -807,14 +816,13 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
target_path, paths = None, []
- # if only one argument is specified, use current directory as target directory
if len(orig_paths) == 1:
- target_path = os.getcwd()
+ # if only one argument is specified, use current directory as target directory
+ target_path = get_cwd()
paths = orig_paths[:]
-
- # if multiple arguments are specified, assume that last argument is target location,
- # and remove that from list of paths to copy
elif orig_paths:
+ # if multiple arguments are specified, assume that last argument is target location,
+ # and remove that from list of paths to copy
target_path = orig_paths[-1]
paths = orig_paths[:-1]
@@ -832,7 +840,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir))
# assume that files need to be copied to current working directory for now
- target_path = os.getcwd()
+ target_path = get_cwd()
if orig_paths:
last_path = orig_paths[-1]
@@ -869,7 +877,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
commit_paths = fetch_files_from_commit(from_commit, path=tmpdir)
# assume that files need to be copied to current working directory for now
- target_path = os.getcwd()
+ target_path = get_cwd()
if orig_paths:
last_path = orig_paths[-1]
diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py
index fb01f3a83e..2c34c83a74 100644
--- a/easybuild/framework/easyconfig/tweak.py
+++ b/easybuild/framework/easyconfig/tweak.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,6 +36,7 @@
* Fotis Georgatos (Uni.Lu, NTUA)
* Alan O'Cais (Juelich Supercomputing Centre)
* Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
"""
import copy
import functools
@@ -60,7 +61,6 @@
from easybuild.tools.config import build_option
from easybuild.tools.filetools import read_file, write_file
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig, search_easyconfigs
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES
@@ -83,8 +83,9 @@ def ec_filename_for(path):
return fn
-def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
- """Tweak list of easyconfigs according to provided build specifications."""
+def tweak(easyconfigs, build_specs, modtool, targetdirs=None, return_map=False):
+ """Tweak list of easyconfigs according to provided build specifications.
+ If return_map=True, also returns tweaked to original file mapping"""
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
@@ -93,6 +94,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
modifying_toolchains_or_deps = False
src_to_dst_tc_mapping = {}
+ tweak_map = {}
revert_to_regex = False
if 'update_deps' in build_specs:
@@ -224,13 +226,18 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
if modifying_toolchains_or_deps:
if tc_name in src_to_dst_tc_mapping:
# Note pruned_build_specs are not passed down for dependencies
- map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
- targetdir=tweaked_ecs_deps_path,
- update_dep_versions=update_dependencies,
- ignore_versionsuffixes=ignore_versionsuffixes)
+ new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
+ targetdir=tweaked_ecs_deps_path,
+ update_dep_versions=update_dependencies,
+ ignore_versionsuffixes=ignore_versionsuffixes)
else:
- tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
+ new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
+
+ if new_ec_file:
+ tweak_map[new_ec_file] = orig_ec['spec']
+ if return_map:
+ return tweaked_easyconfigs, tweak_map
return tweaked_easyconfigs
@@ -348,7 +355,7 @@ def __repr__(self):
'"None"': 'None',
}
for (key, val) in tweaks.items():
- if isinstance(val, string_type) and val in special_values:
+ if isinstance(val, str) and val in special_values:
str_val = val
val = special_values[val]
else:
@@ -994,10 +1001,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
if 'version' in update_build_specs:
# take into account that version in exts_list may have to be updated as well
- if 'exts_list' in parsed_ec and parsed_ec['exts_list']:
+ if 'exts_list' in parsed_ec and parsed_ec.get_ref('exts_list'):
_log.warning("Found 'exts_list' in %s, will only update extension version of %s (if applicable)",
ec_spec, parsed_ec['name'])
- for idx, extension in enumerate(parsed_ec['exts_list']):
+ for idx, extension in enumerate(parsed_ec.get_ref('exts_list')):
if isinstance(extension, tuple) and extension[0] == parsed_ec['name']:
ext_as_list = list(extension)
# in the extension tuple the version is the second element
@@ -1207,7 +1214,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
tc_candidate = fetch_parameters_from_easyconfig(read_file(path), ['toolchain'])[0]
if isinstance(tc_candidate, dict) and tc_candidate['name'] == SYSTEM_TOOLCHAIN_NAME:
cand_paths_filtered += [path]
- if isinstance(tc_candidate, string_type) and tc_candidate == TC_CONSTANT_SYSTEM:
+ if isinstance(tc_candidate, str) and tc_candidate == TC_CONSTANT_SYSTEM:
cand_paths_filtered += [path]
cand_paths = cand_paths_filtered
@@ -1237,7 +1244,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
(highest_version_ignoring_versionsuffix is not None and highest_version is not None and
LooseVersion(highest_version_ignoring_versionsuffix) > LooseVersion(highest_version))
- exclude_alternate_versionsuffixes = False
+ exclude_alternative_versionsuffixes = False
if ignored_versionsuffix_greater:
if ignore_versionsuffixes:
highest_version = highest_version_ignoring_versionsuffix
@@ -1248,11 +1255,11 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
dep['name'], versionsuffix, [d['path'] for d in potential_version_mappings if
d['version'] == highest_version_ignoring_versionsuffix])
# exclude candidates with a different versionsuffix
- exclude_alternate_versionsuffixes = True
+ exclude_alternative_versionsuffixes = True
else:
# If the other version suffixes are not greater, then just ignore them
- exclude_alternate_versionsuffixes = True
- if exclude_alternate_versionsuffixes:
+ exclude_alternative_versionsuffixes = True
+ if exclude_alternative_versionsuffixes:
potential_version_mappings = [d for d in potential_version_mappings if d['versionsuffix'] == versionsuffix]
if highest_versions_only and highest_version is not None:
diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py
index 42be1b99e4..78f021768d 100644
--- a/easybuild/framework/easyconfig/types.py
+++ b/easybuild/framework/easyconfig/types.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,13 +31,11 @@
* Caroline De Brouwer (Ghent University)
* Kenneth Hoste (Ghent University)
"""
-from distutils.util import strtobool
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
_log = fancylogger.getLogger('easyconfig.types', fname=False)
@@ -272,7 +270,7 @@ def to_toolchain_dict(spec):
:param spec: a comma-separated string with two or three values, or a 2/3-element list of strings, or a dict
"""
# check if spec is a string or a list of two values; else, it can not be converted
- if isinstance(spec, string_type):
+ if isinstance(spec, str):
spec = spec.split(',')
if isinstance(spec, (list, tuple)):
@@ -281,7 +279,14 @@ def to_toolchain_dict(spec):
res = {'name': spec[0].strip(), 'version': spec[1].strip()}
# 3-element list
elif len(spec) == 3:
- res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())}
+ hidden = spec[2].strip().lower()
+ if hidden in {'yes', 'true', 't', 'y', '1', 'on'}:
+ hidden = True
+ elif hidden in {'no', 'false', 'f', 'n', '0', 'off'}:
+ hidden = False
+ else:
+ raise EasyBuildError("Invalid truth value %s", hidden)
+ res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': hidden}
else:
raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec)
@@ -314,11 +319,11 @@ def to_list_of_strings(value):
res = None
# if value is already of correct type, we don't need to change anything
- if isinstance(value, list) and all(isinstance(s, string_type) for s in value):
+ if isinstance(value, list) and all(isinstance(s, str) for s in value):
res = value
- elif isinstance(value, string_type):
+ elif isinstance(value, str):
res = [value]
- elif isinstance(value, tuple) and all(isinstance(s, string_type) for s in value):
+ elif isinstance(value, tuple) and all(isinstance(s, str) for s in value):
res = list(value)
else:
raise EasyBuildError("Don't know how to convert provided value to a list of strings: %s", value)
@@ -341,7 +346,7 @@ def to_list_of_strings_and_tuples(spec):
raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec))
for elem in spec:
- if isinstance(elem, (string_type, tuple)):
+ if isinstance(elem, (str, tuple)):
str_tup_list.append(elem)
elif isinstance(elem, list):
str_tup_list.append(tuple(elem))
@@ -366,7 +371,7 @@ def to_list_of_strings_and_tuples_and_dicts(spec):
raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec))
for elem in spec:
- if isinstance(elem, (string_type, tuple, dict)):
+ if isinstance(elem, (str, tuple, dict)):
str_tup_list.append(elem)
elif isinstance(elem, list):
str_tup_list.append(tuple(elem))
@@ -392,17 +397,17 @@ def to_sanity_check_paths_entry(spec):
raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec))
for elem in spec:
- if isinstance(elem, (string_type, tuple)):
+ if isinstance(elem, (str, tuple)):
result.append(elem)
elif isinstance(elem, list):
result.append(tuple(elem))
elif isinstance(elem, dict):
for key, value in elem.items():
- if not isinstance(key, string_type):
+ if not isinstance(key, str):
raise EasyBuildError("Expected keys to be of type string, got %s (%s)", key, type(key))
elif isinstance(value, list):
elem[key] = tuple(value)
- elif not isinstance(value, (string_type, tuple)):
+ elif not isinstance(value, (str, tuple)):
raise EasyBuildError("Expected elements to be of type string, tuple or list, got %s (%s)",
value, type(value))
result.append(elem)
@@ -505,33 +510,51 @@ def to_dependencies(dep_list):
return [to_dependency(dep) for dep in dep_list]
-def to_checksums(checksums):
- """Ensure correct element types for list of checksums: convert list elements to tuples."""
- res = []
- for checksum in checksums:
- # each list entry can be:
- # * None (indicates no checksum)
- # * a string (MD5 or SHA256 checksum)
- # * a tuple with 2 elements: checksum type + checksum value
- # * a list of checksums (i.e. multiple checksums for a single file)
- # * a dict (filename to checksum mapping)
- if isinstance(checksum, string_type):
- res.append(checksum)
- elif isinstance(checksum, (list, tuple)):
- # 2 elements + only string/int values => a checksum tuple
- if len(checksum) == 2 and all(isinstance(x, (string_type, int)) for x in checksum):
- res.append(tuple(checksum))
+def _to_checksum(checksum, list_level=0, allow_dict=True):
+ """Ensure the correct element type for each checksum in the checksum list"""
+ # each entry can be:
+ # * None (indicates no checksum)
+ # * a string (SHA256 checksum)
+ # * a list or tuple with 2 elements: checksum type + checksum value
+ # * a list or tuple of checksums (i.e. multiple checksums for a single file)
+ # * a dict (filename to checksum mapping)
+ if checksum is None or isinstance(checksum, str):
+ return checksum
+ elif isinstance(checksum, (list, tuple)):
+ if len(checksum) == 2 and isinstance(checksum[0], str) and isinstance(checksum[1], (str, int)):
+ # 2 elements so either:
+ # - a checksum tuple (2nd element string or int)
+ # - 2 alternative checksums (tuple)
+ # - 2 checksums that must each match (list)
+ # --> Convert to tuple only if we can exclude the 3rd case
+ if not isinstance(checksum[1], str) or list_level > 0:
+ return tuple(checksum)
else:
- res.append(to_checksums(checksum))
- elif isinstance(checksum, dict):
- validated_dict = {}
- for key, value in checksum.items():
- validated_dict[key] = to_checksums(value)
- res.append(validated_dict)
- else:
- res.append(checksum)
+ return checksum
+ elif list_level < 2:
+ # Alternative checksums or multiple checksums for a single file
+ # Allowed to nest (at most) 2 times, e.g. [[[type, value]]] == [[(type, value)]]
+ # None is not allowed here
+ if any(x is None for x in checksum):
+ raise ValueError('Unexpected None in ' + str(checksum))
+ if isinstance(checksum, tuple) or list_level > 0:
+ # When we already are in a tuple no further recursion is allowed -> set list_level very high
+ return tuple(_to_checksum(x, list_level=99, allow_dict=allow_dict) for x in checksum)
+ else:
+ return list(_to_checksum(x, list_level=list_level+1, allow_dict=allow_dict) for x in checksum)
+ elif isinstance(checksum, dict) and allow_dict:
+ return {key: _to_checksum(value, allow_dict=False) for key, value in checksum.items()}
- return res
+ # Not returned -> Wrong type/format
+ raise ValueError('Unexpected type of "%s": %s' % (type(checksum), str(checksum)))
+
+
+def to_checksums(checksums):
+ """Ensure correct element types for list of checksums: convert list elements to tuples."""
+ try:
+ return [_to_checksum(checksum) for checksum in checksums]
+ except ValueError as e:
+ raise EasyBuildError('Invalid checksums: %s\n\tError: %s', checksums, e)
def ensure_iterable_license_specs(specs):
@@ -543,9 +566,9 @@ def ensure_iterable_license_specs(specs):
"""
if specs is None:
license_specs = [None]
- elif isinstance(specs, string_type):
+ elif isinstance(specs, str):
license_specs = [specs]
- elif isinstance(specs, (list, tuple)) and all(isinstance(x, string_type) for x in specs):
+ elif isinstance(specs, (list, tuple)) and all(isinstance(x, str) for x in specs):
license_specs = list(specs)
else:
msg = "Unsupported type %s for easyconfig parameter 'license_file'! " % type(specs)
@@ -608,37 +631,57 @@ def ensure_iterable_license_specs(specs):
}))
# checksums is a list of checksums, one entry per file (source/patch)
# each entry can be:
+# None
# a single checksum value (string)
# a single checksum value of a specified type (2-tuple, 1st element is checksum type, 2nd element is checksum)
# a list of checksums (of different types, perhaps different formats), which should *all* be valid
-# a dictionary with a mapping from filename to checksum value
-CHECKSUM_LIST = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT]}))
-CHECKSUMS = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT, CHECKSUM_LIST]}))
+# a tuple of checksums (of different types, perhaps different formats), where one should be valid
+# a dictionary with a mapping from filename to checksum (None, value, type&value, alternatives)
+
+# Type & value, value may be an int for type "size"
+# This is a bit too permissive as it allows the first element to be an int and doesn't restrict the number of elements
+CHECKSUM_AND_TYPE = (tuple, as_hashable({'elem_types': [str, int]}))
+CHECKSUM_LIST = (list, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE]}))
+CHECKSUM_TUPLE = (tuple, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE]}))
+CHECKSUM_DICT = (dict, as_hashable(
+ {
+ 'elem_types': [type(None), str, CHECKSUM_AND_TYPE, CHECKSUM_TUPLE, CHECKSUM_LIST],
+ 'key_types': [str],
+ }
+))
+# At the top-level we allow tuples/lists containing a dict
+CHECKSUM_LIST_W_DICT = (list, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE, CHECKSUM_DICT]}))
+CHECKSUM_TUPLE_W_DICT = (tuple, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE, CHECKSUM_DICT]}))
-CHECKABLE_TYPES = [CHECKSUM_LIST, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
+CHECKSUMS = (list, as_hashable({'elem_types': [type(None), str, CHECKSUM_AND_TYPE,
+ CHECKSUM_LIST_W_DICT, CHECKSUM_TUPLE_W_DICT, CHECKSUM_DICT]}))
+
+CHECKABLE_TYPES = [CHECKSUM_AND_TYPE, CHECKSUM_LIST, CHECKSUM_TUPLE,
+ CHECKSUM_LIST_W_DICT, CHECKSUM_TUPLE_W_DICT, CHECKSUM_DICT, CHECKSUMS,
+ DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
SANITY_CHECK_PATHS_DICT, SANITY_CHECK_PATHS_ENTRY, STRING_DICT, STRING_OR_TUPLE_LIST,
STRING_OR_TUPLE_DICT, STRING_OR_TUPLE_OR_DICT_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
# easy types, that can be verified with isinstance
-EASY_TYPES = [string_type, bool, dict, int, list, str, tuple]
+EASY_TYPES = [str, bool, dict, int, list, str, tuple, type(None)]
# type checking is skipped for easyconfig parameters names not listed in PARAMETER_TYPES
PARAMETER_TYPES = {
'checksums': CHECKSUMS,
'docurls': LIST_OF_STRINGS,
- 'name': string_type,
+ 'name': str,
'osdependencies': STRING_OR_TUPLE_LIST,
'patches': STRING_OR_TUPLE_OR_DICT_LIST,
'sanity_check_paths': SANITY_CHECK_PATHS_DICT,
'toolchain': TOOLCHAIN_DICT,
- 'version': string_type,
+ 'version': str,
}
# add all dependency types as dependencies
for dep in DEPENDENCY_PARAMETERS:
PARAMETER_TYPES[dep] = DEPENDENCIES
TYPE_CONVERSION_FUNCTIONS = {
- string_type: str,
+ str: str,
float: float,
int: int,
str: str,
diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py
index 4a0ec15cdd..471b5ac5db 100644
--- a/easybuild/framework/easystack.py
+++ b/easybuild/framework/easystack.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2024 Ghent University
+# Copyright 2020-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,7 +37,6 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import only_if_module_is_available
try:
import yaml
@@ -53,7 +52,7 @@ def check_value(value, context):
Check whether specified value obtained from a YAML file in specified context represents is valid.
The value must be a string (not a float or an int).
"""
- if not isinstance(value, string_type):
+ if not isinstance(value, str):
error_msg = '\n'.join([
"Value %(value)s (of type %(type)s) obtained for %(context)s is not valid!",
"Make sure to wrap the value in single quotes (like '%(value)s') to avoid that it is interpreted "
@@ -189,9 +188,6 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa
@only_if_module_is_available('yaml', pkgname='PyYAML')
def parse_easystack(filepath):
"""Parses through easystack file, returns what EC are to be installed together with their options."""
- log_msg = "Support for easybuild-ing from multiple easyconfigs based on "
- log_msg += "information obtained from provided file (easystack) with build specifications."
- _log.experimental(log_msg)
_log.info("Building from easystack: '%s'" % filepath)
# class instance which contains all info about planned build
diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 56f242ffce..51a40bbe15 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,10 +40,9 @@
from easybuild.framework.easyconfig.easyconfig import resolve_template
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
-from easybuild.tools.build_log import EasyBuildError, raise_nosupport
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, raise_nosupport
from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import check_async_cmd, run_cmd
-from easybuild.tools.py2vs3 import string_type
+from easybuild.tools.run import run_shell_cmd
def resolve_exts_filter_template(exts_filter, ext):
@@ -54,7 +53,7 @@ def resolve_exts_filter_template(exts_filter, ext):
:return: (cmd, input) as a tuple of strings
"""
- if isinstance(exts_filter, string_type) or len(exts_filter) != 2:
+ if isinstance(exts_filter, str) or len(exts_filter) != 2:
raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")')
cmd, cmdinput = exts_filter
@@ -117,7 +116,7 @@ def __init__(self, mself, ext, extra_params=None):
# Add install/builddir templates with values from master.
for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None))
+ self.cfg.template_values[key] = str(getattr(self.master, key, None))
# We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong).
# It should be specified for the extension specifically, or be empty (so it is auto-derived).
@@ -129,7 +128,15 @@ def __init__(self, mself, ext, extra_params=None):
self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values)
self.src_extract_cmd = self.ext.get('extract_cmd', None)
self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values)
- self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values)
+ # Some options may not be resolvable yet
+ self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})),
+ self.cfg.template_values,
+ expect_resolved=False)
+ if 'parallel' in self.options:
+ # Replace value and issue better warning for easyconfig parameters,
+ # as opposed to warnings meant for easyblocks
+ self.log.deprecated("Easyconfig parameter 'parallel' is deprecated, use 'max_parallel' instead.", '6.0')
+ self.options['max_parallel'] = self.options.pop('parallel')
if extra_params:
self.cfg.extend_params(extra_params, overwrite=False)
@@ -147,16 +154,17 @@ def __init__(self, mself, ext, extra_params=None):
self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
key, name, version, value)
+ # If parallelism has been set already take potentially new limitation into account
+ if self.cfg.is_parallel_set:
+ max_par = self.cfg['max_parallel']
+ if max_par is not None and max_par < self.cfg.parallel:
+ self.cfg.parallel = max_par
+
self.sanity_check_fail_msgs = []
self.sanity_check_module_loaded = False
self.fake_mod_data = None
- self.async_cmd_info = None
- self.async_cmd_output = None
- self.async_cmd_check_cnt = None
- # initial read size should be relatively small,
- # to avoid hanging for a long time until desired output is available in async_cmd_check
- self.async_cmd_read_size = 1024
+ self.async_cmd_task = None
@property
def name(self):
@@ -173,18 +181,39 @@ def version(self):
return self.ext.get('version', None)
def prerun(self):
+ """
+ [DEPRECATED][6.0] Stuff to do before installing a extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.pre_install_extension()
+
+ def pre_install_extension(self):
"""
Stuff to do before installing a extension.
"""
pass
def run(self, *args, **kwargs):
+ """
+ [DEPRECATED][6.0] Actual installation of an extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.install_extension(*args, **kwargs)
+
+ def install_extension(self, *args, **kwargs):
"""
Actual installation of an extension.
"""
pass
def run_async(self, *args, **kwargs):
+ """
+ [DEPRECATED][6.0] Asynchronous installation of an extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.install_extension_async(*args, **kwargs)
+
+ def install_extension_async(self, *args, **kwargs):
"""
Asynchronous installation of an extension.
"""
@@ -192,47 +221,58 @@ def run_async(self, *args, **kwargs):
def postrun(self):
"""
- Stuff to do after installing a extension.
+ [DEPRECATED][6.0] Stuff to do after installing a extension.
"""
- self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', []))
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.post_install_extension()
- def async_cmd_start(self, cmd, inp=None):
+ def post_install_extension(self):
"""
- Start installation asynchronously using specified command.
+ Stuff to do after installing a extension.
"""
- self.async_cmd_output = ''
- self.async_cmd_check_cnt = 0
- self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True)
+ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', []))
- def async_cmd_check(self):
+ def install_extension_substep(self, substep, *args, **kwargs):
"""
- Check progress of installation command that was started asynchronously.
-
- :return: True if command completed, False otherwise
+ Carry out extension installation substep allowing use of deprecated
+ methods on those extensions using an older EasyBlock
"""
- if self.async_cmd_info is None:
- raise EasyBuildError("No installation command running asynchronously for %s", self.name)
- elif self.async_cmd_info is False:
- self.log.info("No asynchronous command was started for extension %s", self.name)
- return True
+ substeps_mapping = {
+ 'pre_install_extension': 'prerun',
+ 'install_extension': 'run',
+ 'install_extension_async': 'run_async',
+ 'post_install_extension': 'postrun',
+ }
+
+ deprecated_substep = substeps_mapping.get(substep)
+ if deprecated_substep is None:
+ raise EasyBuildError("Unknown extension installation substep: %s", substep)
+
+ try:
+ substep_method = getattr(self, deprecated_substep)
+ except AttributeError:
+ log_msg = f"EasyBlock does not implement deprecated method '{deprecated_substep}' "
+ log_msg += f"for installation substep {substep}"
+ self.log.debug(log_msg)
+ substep_method = getattr(self, substep)
else:
- self.log.debug("Checking on installation of extension %s...", self.name)
- # use small read size, to avoid waiting for a long time until sufficient output is produced
- res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size)
- self.async_cmd_output += res['output']
- if res['done']:
- self.log.info("Installation of extension %s completed!", self.name)
- self.async_cmd_info = None
+ # Qualified method name contains class defining the method (PEP 3155)
+ substep_method_name = substep_method.__qualname__
+ self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}")
+
+ base_method_name = f"Extension.{deprecated_substep}"
+ if substep_method_name == base_method_name:
+ # No custom method in child Easyblock, deprecated method is defined by base Extension class
+ # Switch to non-deprecated substep method
+ substep_method = getattr(self, substep)
else:
- self.async_cmd_check_cnt += 1
- self.log.debug("Installation of extension %s still running (checked %d times)",
- self.name, self.async_cmd_check_cnt)
- # increase read size after sufficient checks,
- # to avoid that installation hangs due to output buffer filling up...
- if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2):
- self.async_cmd_read_size *= 2
+ # Custom deprecated method used by child Easyblock
+ self.log.deprecated(
+ f"{substep_method_name}() is deprecated, use {substep}() instead.",
+ "6.0",
+ )
- return res['done']
+ return substep_method(*args, **kwargs)
@property
def required_deps(self):
@@ -274,15 +314,14 @@ def sanity_check_step(self):
self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name)
elif exts_filter:
cmd, stdin = resolve_exts_filter_template(exts_filter, self)
- # set log_ok to False so we can catch the error instead of run_cmd
- (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin)
+ cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin)
- if ec:
+ if cmd_res.exit_code != EasyBuildExit.SUCCESS:
if stdin:
fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin)
else:
fail_msg = 'command "%s" failed' % cmd
- fail_msg += "; output:\n%s" % output.strip()
+ fail_msg += "; output:\n%s" % cmd_res.output.strip()
self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg)
res = (False, fail_msg)
# keep track of all reasons of failure
diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py
index efd7c349f7..52d787528d 100644
--- a/easybuild/framework/extensioneasyblock.py
+++ b/easybuild/framework/extensioneasyblock.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of the University of Ghent (http://ugent.be/hpc).
@@ -89,6 +89,7 @@ def __init__(self, *args, **kwargs):
self.installdir = self.master.installdir
self.modules_tool = self.master.modules_tool
self.module_generator = self.master.module_generator
+ self.module_load_environment = self.master.module_load_environment
self.robot_path = self.master.robot_path
self.is_extension = True
self.unpack_options = None
@@ -135,7 +136,7 @@ def _set_start_dir(self):
self.log.warning(warn_msg)
print_warning(warn_msg, silent=build_option('silent'))
- def run(self, unpack_src=False):
+ def install_extension(self, unpack_src=False):
"""Common operations for extensions: unpacking sources, patching, ..."""
# unpack file if desired
diff --git a/easybuild/main.py b/easybuild/main.py
index b6adc08559..fd25e27dc1 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,6 +37,7 @@
* Ward Poelmans (Ghent University)
* Fotis Georgatos (Uni.Lu, NTUA)
* Maxime Boissonneault (Compute Canada)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
"""
import copy
import os
@@ -70,7 +71,7 @@
from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr
from easybuild.tools.github import new_pr_from_branch
from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr
-from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL
+from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, CRASH, FAIL
from easybuild.tools.hooks import load_hooks, run_hook
from easybuild.tools.modules import modules_tool
from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color
@@ -79,7 +80,6 @@
from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
from easybuild.tools.package.utilities import check_pkg_support
from easybuild.tools.parallelbuild import submit_jobs
-from easybuild.tools.py2vs3 import python2_is_deprecated
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.systemtools import check_easybuild_deps
from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
@@ -134,10 +134,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
ec_res = {}
try:
- (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
+ (ec_res['success'], app_log, err_msg, err_code) = build_and_install_one(ec, init_env)
ec_res['log_file'] = app_log
if not ec_res['success']:
- ec_res['err'] = EasyBuildError(err)
+ ec_res['err'] = EasyBuildError(err_msg, exit_code=err_code)
except Exception as err:
# purposely catch all exceptions
ec_res['success'] = False
@@ -152,11 +152,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
# keep track of success/total count
if ec_res['success']:
- test_msg = "Successfully built %s" % ec['spec']
+ test_msg = "Successfully installed %s" % ec['spec']
else:
- test_msg = "Build of %s failed" % ec['spec']
+ test_msg = "Installation of %s failed" % os.path.basename(ec['spec'])
if 'err' in ec_res:
- test_msg += " (err: %s)" % ec_res['err']
+ test_msg += ": %s" % ec_res['err']
# dump test report next to log file
test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
@@ -172,10 +172,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
if not ec_res['success'] and exit_on_failure:
- if 'traceback' in ec_res:
- raise EasyBuildError(ec_res['traceback'])
+ if not isinstance(ec_res['err'], EasyBuildError):
+ raise ec_res['err']
else:
- raise EasyBuildError(test_msg)
+ raise EasyBuildError(test_msg, exit_code=err_code)
res.append((ec, ec_res))
@@ -431,7 +431,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
- easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
+ easyconfigs, tweak_map = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths, return_map=True)
+ else:
+ tweak_map = None
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
@@ -553,7 +555,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
# submit build as job(s), clean up and exit
if options.job:
- submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
+ submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing, tweak_map=tweak_map)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
return True
@@ -613,9 +615,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
options, orig_paths = eb_go.options, eb_go.args
- if 'python2' not in build_option('silence_deprecation_warnings'):
- python2_is_deprecated()
-
global _log
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
from_pr_list, tweaked_ecs_paths) = cfg_settings
@@ -778,20 +777,28 @@ def prepare_main(args=None, logfile=None, testing=None):
return init_session_state, eb_go, cfg_settings
-if __name__ == "__main__":
+def main_with_hooks(args=None):
# take into account that EasyBuildError may be raised when parsing the EasyBuild configuration
try:
- init_session_state, eb_go, cfg_settings = prepare_main()
+ init_session_state, eb_go, cfg_settings = prepare_main(args=args)
except EasyBuildError as err:
- print_error(err.msg)
+ print_error(err.msg, exit_code=err.exit_code)
hooks = load_hooks(eb_go.options.hooks)
try:
- main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
+ main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
except EasyBuildError as err:
run_hook(FAIL, hooks, args=[err])
- print_error(err.msg)
+ print_error(err.msg, exit_on_error=True, exit_code=err.exit_code)
except KeyboardInterrupt as err:
run_hook(CANCEL, hooks, args=[err])
print_error("Cancelled by user: %s" % err)
+ except Exception as err:
+ run_hook(CRASH, hooks, args=[err])
+ sys.stderr.write("EasyBuild crashed! Please consider reporting a bug, this should not happen...\n\n")
+ raise
+
+
+if __name__ == "__main__":
+ main_with_hooks()
diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py
deleted file mode 100755
index ef8c281162..0000000000
--- a/easybuild/scripts/bootstrap_eb.py
+++ /dev/null
@@ -1,1172 +0,0 @@
-#!/usr/bin/env python
-##
-# Copyright 2013-2024 Ghent University
-#
-# This file is part of EasyBuild,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/easybuilders/easybuild
-#
-# EasyBuild is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation v2.
-#
-# EasyBuild is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with EasyBuild. If not, see .
-##
-
-"""
-Bootstrap script for EasyBuild
-
-Installs distribute with included (patched) distribute_setup.py script to obtain easy_install,
-and then performs a staged install of EasyBuild:
- * stage 0: install setuptools (which provides easy_install), unless already available
- * stage 1: install EasyBuild with easy_install to a temporary directory
- * stage 2: install EasyBuild with EasyBuild from stage 1 to specified install directory
-
-Authors: Kenneth Hoste (UGent), Stijn Deweirdt (UGent), Ward Poelmans (UGent)
-License: GPLv2
-
-inspired by https://bitbucket.org/pdubroy/pip/raw/tip/getpip.py
-(via http://dubroy.com/blog/so-you-want-to-install-a-python-package/)
-"""
-
-import codecs
-import copy
-import glob
-import os
-import re
-import shutil
-import site
-import sys
-import tempfile
-import traceback
-from distutils.version import LooseVersion
-from hashlib import md5
-from platform import python_version
-
-IS_PY3 = sys.version_info[0] == 3
-
-if not IS_PY3:
- import urllib2 as std_urllib
-else:
- import urllib.request as std_urllib
-
-
-EB_BOOTSTRAP_VERSION = '20210715.01'
-
-# argparse preferrred, optparse deprecated >=2.7
-HAVE_ARGPARSE = False
-try:
- import argparse
- HAVE_ARGPARSE = True
-except ImportError:
- import optparse
-
-PYPI_SOURCE_URL = 'https://pypi.python.org/packages/source'
-
-VSC_BASE = 'vsc-base'
-VSC_INSTALL = 'vsc-install'
-# Python 3 is not supported by the vsc-* packages
-EASYBUILD_PACKAGES = (([] if IS_PY3 else [VSC_INSTALL, VSC_BASE]) +
- ['easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs'])
-
-STAGE1_SUBDIR = 'eb_stage1'
-
-# the EasyBuild bootstrap script is deprecated, and will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined
-EASYBUILD_BOOTSTRAP_DEPRECATED = os.environ.pop('EASYBUILD_BOOTSTRAP_DEPRECATED', None)
-
-# set print_debug to True for detailed progress info
-print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False)
-
-# install with --force in stage2?
-forced_install = os.environ.pop('EASYBUILD_BOOTSTRAP_FORCED', False)
-
-# don't add user site directory to sys.path (equivalent to python -s), see https://www.python.org/dev/peps/pep-0370/
-os.environ['PYTHONNOUSERSITE'] = '1'
-site.ENABLE_USER_SITE = False
-
-# clean PYTHONPATH to avoid finding readily installed stuff
-os.environ['PYTHONPATH'] = ''
-
-EASYBUILD_BOOTSTRAP_SOURCEPATH = os.environ.pop('EASYBUILD_BOOTSTRAP_SOURCEPATH', None)
-EASYBUILD_BOOTSTRAP_SKIP_STAGE0 = os.environ.pop('EASYBUILD_BOOTSTRAP_SKIP_STAGE0', False)
-EASYBUILD_BOOTSTRAP_FORCE_VERSION = os.environ.pop('EASYBUILD_BOOTSTRAP_FORCE_VERSION', None)
-
-# keep track of original environment (after clearing PYTHONPATH)
-orig_os_environ = copy.deepcopy(os.environ)
-
-# If the modules tool is specified, use it
-easybuild_modules_tool = os.environ.get('EASYBUILD_MODULES_TOOL', None)
-easybuild_module_syntax = os.environ.get('EASYBUILD_MODULE_SYNTAX', None)
-
-# If modules subdir specifications are defined, use them
-easybuild_installpath_modules = os.environ.get('EASYBUILD_INSTALLPATH_MODULES', None)
-easybuild_subdir_modules = os.environ.get('EASYBUILD_SUBDIR_MODULES', 'modules')
-easybuild_suffix_modules_path = os.environ.get('EASYBUILD_SUFFIX_MODULES_PATH', 'all')
-
-
-#
-# Utility functions
-#
-def debug(msg):
- """Print debug message."""
-
- if print_debug:
- print("[[DEBUG]] " + msg)
-
-
-def info(msg):
- """Print info message."""
-
- print("[[INFO]] " + msg)
-
-
-def error(msg, exit=True):
- """Print error message and exit."""
-
- print("[[ERROR]] " + msg)
- sys.exit(1)
-
-
-def mock_stdout_stderr():
- """Mock stdout/stderr channels"""
- try:
- from cStringIO import StringIO
- except ImportError:
- from io import StringIO
- orig_stdout, orig_stderr = sys.stdout, sys.stderr
- sys.stdout.flush()
- sys.stdout = StringIO()
- sys.stderr.flush()
- sys.stderr = StringIO()
-
- return orig_stdout, orig_stderr
-
-
-def restore_stdout_stderr(orig_stdout, orig_stderr):
- """Restore stdout/stderr channels after mocking"""
- # collect output
- sys.stdout.flush()
- stdout = sys.stdout.getvalue()
- sys.stderr.flush()
- stderr = sys.stderr.getvalue()
-
- # restore original stdout/stderr
- sys.stdout = orig_stdout
- sys.stderr = orig_stderr
-
- return stdout, stderr
-
-
-def det_lib_path(libdir):
- """Determine relative path of Python library dir."""
- if libdir is None:
- libdir = 'lib'
- pyver = '.'.join([str(x) for x in sys.version_info[:2]])
- return os.path.join(libdir, 'python%s' % pyver, 'site-packages')
-
-
-def det_modules_path(install_path):
- """Determine modules path."""
- if easybuild_installpath_modules is not None:
- modules_path = os.path.join(easybuild_installpath_modules, easybuild_suffix_modules_path)
- else:
- modules_path = os.path.join(install_path, easybuild_subdir_modules, easybuild_suffix_modules_path)
-
- return modules_path
-
-
-def find_egg_dir_for(path, pkg):
- """Find full path of egg dir for given package."""
-
- res = None
-
- for libdir in ['lib', 'lib64']:
- full_libpath = os.path.join(path, det_lib_path(libdir))
- eggdir_regex = re.compile('%s-[0-9a-z.]+-py[0-9.]+.egg' % pkg.replace('-', '_'))
- subdirs = (os.path.exists(full_libpath) and sorted(os.listdir(full_libpath))) or []
- for subdir in subdirs:
- if eggdir_regex.match(subdir):
- eggdir = os.path.join(full_libpath, subdir)
- if res is None:
- debug("Found egg dir for %s at %s" % (pkg, eggdir))
- res = eggdir
- else:
- debug("Found another egg dir for %s at %s (ignoring it)" % (pkg, eggdir))
-
- # no egg dir found
- if res is None:
- debug("Failed to determine egg dir path for %s in %s (subdirs: %s)" % (pkg, path, subdirs))
-
- return res
-
-
-def prep(path):
- """Prepare for installing a Python package in the specified path."""
-
- debug("Preparing for path %s" % path)
-
- # restore original environment first
- os.environ = copy.deepcopy(orig_os_environ)
- debug("os.environ['PYTHONPATH'] after reset: %s" % os.environ['PYTHONPATH'])
-
- # update PATH
- os.environ['PATH'] = os.pathsep.join([os.path.join(path, 'bin')] +
- [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0])
- debug("$PATH: %s" % os.environ['PATH'])
-
- # update actual Python search path
- sys.path.insert(0, path)
-
- # make sure directory exists (this is required by setuptools)
- # usually it's 'lib', but can be 'lib64' as well
- for libdir in ['lib', 'lib64']:
- full_libpath = os.path.join(path, det_lib_path(libdir))
- if not os.path.exists(full_libpath):
- os.makedirs(full_libpath)
- # PYTHONPATH needs to be set as well, otherwise setuptools will fail
- pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0]
- os.environ['PYTHONPATH'] = os.pathsep.join([full_libpath] + pythonpaths)
-
- debug("$PYTHONPATH: %s" % os.environ['PYTHONPATH'])
-
- os.environ['EASYBUILD_MODULES_TOOL'] = easybuild_modules_tool
- debug("$EASYBUILD_MODULES_TOOL set to %s" % os.environ['EASYBUILD_MODULES_TOOL'])
-
- if easybuild_module_syntax:
- # if module syntax is specified, use it
- os.environ['EASYBUILD_MODULE_SYNTAX'] = easybuild_module_syntax
- debug("Using specified module syntax: %s" % os.environ['EASYBUILD_MODULE_SYNTAX'])
- elif easybuild_modules_tool != 'Lmod':
- # Lua is the default module syntax, but that requires Lmod
- # if Lmod is not being used, use Tcl module syntax
- os.environ['EASYBUILD_MODULE_SYNTAX'] = 'Tcl'
- debug("$EASYBUILD_MODULE_SYNTAX set to %s" % os.environ['EASYBUILD_MODULE_SYNTAX'])
-
-
-def check_module_command(tmpdir):
- """Check which module command is available, and prepare for using it."""
- global easybuild_modules_tool
-
- if easybuild_modules_tool is not None:
- info("Using modules tool specified by $EASYBUILD_MODULES_TOOL: %s" % easybuild_modules_tool)
- return easybuild_modules_tool
-
- def check_cmd_help(modcmd):
- """Check 'help' output for specified command."""
- modcmd_re = re.compile(r'module\s.*command')
- cmd = "%s python help" % modcmd
- os.system("%s > %s 2>&1" % (cmd, out))
- txt = open(out, 'r').read()
- debug("Output from %s: %s" % (cmd, txt))
- return modcmd_re.search(txt)
-
- def is_modulecmd_tcl_modulestcl():
- """Determine if modulecmd.tcl is EnvironmentModulesTcl."""
- modcmd_re = re.compile('Modules Release Tcl')
- cmd = "modulecmd.tcl python --version"
- os.system("%s > %s 2>&1" % (cmd, out))
- txt = open(out, 'r').read()
- debug("Output from %s: %s" % (cmd, txt))
- return modcmd_re.search(txt)
-
- # order matters, which is why we don't use a dict
- known_module_commands = [
- ('lmod', 'Lmod'),
- ('modulecmd.tcl', 'EnvironmentModules'),
- ('modulecmd', 'EnvironmentModulesC'),
- ]
- out = os.path.join(tmpdir, 'module_command.out')
- modtool = None
- for modcmd, modtool in known_module_commands:
- if check_cmd_help(modcmd):
- # distinguish between EnvironmentModulesTcl and EnvironmentModules
- if modcmd == 'modulecmd.tcl' and is_modulecmd_tcl_modulestcl():
- modtool = 'EnvironmentModulesTcl'
- easybuild_modules_tool = modtool
- info("Found module command '%s' (%s), so using it." % (modcmd, modtool))
- break
- elif modcmd == 'lmod':
- # check value of $LMOD_CMD as fallback
- modcmd = os.environ.get('LMOD_CMD')
- if modcmd and check_cmd_help(modcmd):
- easybuild_modules_tool = modtool
- info("Found module command '%s' via $LMOD_CMD (%s), so using it." % (modcmd, modtool))
- break
- elif modtool == 'EnvironmentModules':
- # check value of $MODULESHOME as fallback
- moduleshome = os.environ.get('MODULESHOME', 'MODULESHOME_NOT_DEFINED')
- modcmd = os.path.join(moduleshome, 'libexec', 'modulecmd.tcl')
- if os.path.exists(modcmd) and check_cmd_help(modcmd):
- easybuild_modules_tool = modtool
- info("Found module command '%s' via $MODULESHOME (%s), so using it." % (modcmd, modtool))
- break
-
- if easybuild_modules_tool is None:
- mod_cmds = [m for (m, _) in known_module_commands]
- msg = [
- "Could not find any module command, make sure one available in your $PATH.",
- "Known module commands are checked in order, and include: %s" % ', '.join(mod_cmds),
- "Check the output of 'type module' to determine the location of the module command you are using.",
- ]
- error('\n'.join(msg))
-
- return modtool
-
-
-def check_setuptools():
- """Check whether a suitable setuptools installation is already available."""
-
- debug("Checking whether suitable setuptools installation is available...")
- res = None
-
- _, outfile = tempfile.mkstemp()
-
- # note: we need to be very careful here, because switching to a different setuptools installation (e.g. in stage0)
- # after the setuptools module was imported is very tricky...
- # So, we'll check things by running commands through os.system rather than importing setuptools directly.
- cmd_tmpl = "%s -c '%%s' > %s 2>&1" % (sys.executable, outfile)
-
- # check setuptools version
- try:
- os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)")
- setuptools_ver = LooseVersion(open(outfile).read().strip())
- debug("Found setuptools version %s" % setuptools_ver)
-
- min_setuptools_ver = '0.6c11'
- if setuptools_ver < LooseVersion(min_setuptools_ver):
- debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver))
- res = False
- except Exception as err:
- debug("Failed to check setuptools version: %s" % err)
- res = False
-
- os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)")
- out = open(outfile).read().strip()
- debug("Location of setuptools' easy_install module: %s" % out)
- if 'setuptools/command/easy_install' not in out:
- debug("Module 'setuptools.command.easy_install not found")
- res = False
-
- if res is None:
- os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)")
- setuptools_loc = open(outfile).read().strip()
- res = os.path.dirname(os.path.dirname(setuptools_loc))
- debug("Location of setuptools installation: %s" % res)
-
- try:
- os.remove(outfile)
- except Exception:
- pass
-
- return res
-
-
-def run_easy_install(args):
- """Run easy_install with specified list of arguments"""
- import setuptools
- debug("Active setuptools installation: %s" % setuptools.__file__)
- from setuptools.command import easy_install
-
- orig_stdout, orig_stderr = mock_stdout_stderr()
- try:
- easy_install.main(args)
- easy_install_stdout, easy_install_stderr = restore_stdout_stderr(orig_stdout, orig_stderr)
- except (Exception, SystemExit) as err:
- easy_install_stdout, easy_install_stderr = restore_stdout_stderr(orig_stdout, orig_stderr)
- error("Running 'easy_install %s' failed: %s\n%s" % (' '.join(args), err, traceback.format_exc()))
-
- debug("stdout for 'easy_install %s':\n%s" % (' '.join(args), easy_install_stdout))
- debug("stderr for 'easy_install %s':\n%s" % (' '.join(args), easy_install_stderr))
-
-
-def check_easy_install_cmd():
- """Try to make sure available 'easy_install' command matches active 'setuptools' installation."""
-
- debug("Checking whether available 'easy_install' command matches active 'setuptools' installation...")
-
- _, outfile = tempfile.mkstemp()
-
- import setuptools
- debug("Location of active setuptools installation: %s" % setuptools.__file__)
-
- easy_install_regex = re.compile('^(setuptools|distribute) %s' % setuptools.__version__)
- debug("Pattern for 'easy_install --version': %s" % easy_install_regex.pattern)
-
- pythonpath = os.getenv('PYTHONPATH', '')
- cmd = "PYTHONPATH='%s' %s -m easy_install --version" % (pythonpath, sys.executable)
- os.system("%s > %s 2>&1" % (cmd, outfile))
- outtxt = open(outfile).read().strip()
- debug("Output of '%s':\n%s" % (cmd, outtxt))
- res = bool(easy_install_regex.match(outtxt))
- debug("Result: %s" % res)
- if res:
- debug("Found right 'easy_install' command")
- return
-
- error("Failed to find right 'easy_install' command!")
-
-
-#
-# Stage functions
-#
-def stage0(tmpdir):
- """STAGE 0: Prepare and install distribute via included (patched) distribute_setup.py script."""
-
- print('\n')
- info("+++ STAGE 0: installing distribute via included (patched) distribute_setup.py...\n")
-
- txt = DISTRIBUTE_SETUP_PY
- if not print_debug:
- # silence distribute_setup.py by redirecting output to /dev/null
- txt = re.sub(r'([^\n]*)(return subprocess.call\(args)(\) == 0)',
- r"\1f = open(os.devnull, 'w'); \2, stdout=f, stderr=f\3",
- txt)
- # silence distribute_setup.py completely by setting high log level threshold
- txt = re.sub(r'([^\n]*)(# extracting the tarball[^\n]*)', r'\1log.set_verbosity(1000)\n\1\2', txt)
-
- # write distribute_setup.py to file (with correct header)
- distribute_setup = os.path.join(tmpdir, 'distribute_setup.py')
- f = open(distribute_setup, "w")
- f.write(txt)
- f.close()
-
- # create expected directories, set Python search path
- debug("preparing environment...")
- prep(tmpdir)
-
- import distribute_setup
- debug("distribute_setup.__file__: %s" % distribute_setup.__file__)
-
- # install easy_install to temporary directory
- from distribute_setup import main as distribute_setup_main
- orig_sys_argv = sys.argv[:] # make a copy
- sys.argv.append('--prefix=%s' % tmpdir)
- # We download a custom version of distribute: it uses a newer version of markerlib to avoid a bug (#1099)
- # It's is the source of distribute 0.6.49 with the file _markerlib/markers.py replaced by the 0.6 version of
- # markerlib which can be found at https://pypi.python.org/pypi/markerlib/0.6.0
- sys.argv.append('--download-base=https://easybuilders.github.io/easybuild/files/')
- distribute_setup_main(version="0.6.49-patched1")
- sys.argv = orig_sys_argv
-
- # sanity check
- if os.path.exists(os.path.join(tmpdir, 'bin', 'easy_install')):
- debug("easy_install sanity check OK")
- else:
- error("Installing distribute which should deliver easy_install failed?")
-
- # prepend distribute egg dir to sys.path, so we know which setuptools we're using
- distribute_egg_dir = find_egg_dir_for(tmpdir, 'distribute')
- if distribute_egg_dir is None:
- error("Failed to determine egg dir path for distribute_egg_dir in %s" % tmpdir)
- else:
- sys.path.insert(0, distribute_egg_dir)
-
- # make sure we're getting the setuptools we expect
- import setuptools
- from setuptools.command import easy_install
-
- for mod, path in [('setuptools', setuptools.__file__), ('easy_install', easy_install.__file__)]:
- if tmpdir not in path:
- error("Found another %s module than expected: %s" % (mod, path))
- else:
- debug("Found %s in expected path, good!" % mod)
-
- info("Installed setuptools version %s (%s)" % (setuptools.__version__, setuptools.__file__))
-
- return distribute_egg_dir
-
-
-def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion):
- """STAGE 1: temporary install EasyBuild using distribute's easy_install."""
-
- print('\n')
- info("+++ STAGE 1: installing EasyBuild in temporary dir with easy_install...\n")
-
- # determine locations of source tarballs, if sources path is specified
- source_tarballs = {}
- if sourcepath is not None:
- info("Fetching sources from %s..." % sourcepath)
- for pkg in EASYBUILD_PACKAGES:
- pkg_tarball_glob = os.path.join(sourcepath, '%s*.tar.gz' % pkg)
- pkg_tarball_paths = glob.glob(pkg_tarball_glob)
- if len(pkg_tarball_paths) > 1:
- error("Multiple tarballs found for %s: %s" % (pkg, pkg_tarball_paths))
- elif len(pkg_tarball_paths) == 0:
- if pkg not in [VSC_BASE, VSC_INSTALL]:
- # vsc-base package is not strictly required
- # it's only a dependency since EasyBuild v2.0;
- # with EasyBuild v2.0, it will be pulled in from PyPI when installing easybuild-framework;
- # vsc-install is an optional dependency, only required to run unit tests
- error("Missing source tarball: %s" % pkg_tarball_glob)
- else:
- info("Found %s for %s package" % (pkg_tarball_paths[0], pkg))
- source_tarballs.update({pkg: pkg_tarball_paths[0]})
-
- if print_debug:
- debug("$ easy_install --help")
- run_easy_install(['--help'])
-
- # prepare install dir
- targetdir_stage1 = os.path.join(tmpdir, STAGE1_SUBDIR)
- prep(targetdir_stage1) # set PATH, Python search path
-
- # install latest EasyBuild with easy_install from PyPi
- cmd = [
- '--upgrade', # make sure the latest version is pulled from PyPi
- '--prefix=%s' % targetdir_stage1,
- ]
-
- post_vsc_base = []
- if source_tarballs:
- # install provided source tarballs (order matters)
- cmd.extend([source_tarballs[pkg] for pkg in EASYBUILD_PACKAGES if pkg in source_tarballs])
- # add vsc-base again at the end, to avoid that the one available on the system is used instead
- if VSC_BASE in source_tarballs:
- cmd.append(source_tarballs[VSC_BASE])
- else:
- # install meta-package easybuild from PyPI
- if forcedversion:
- cmd.append('easybuild==%s' % forcedversion)
- elif IS_PY3:
- cmd.append('easybuild>=4.0') # Python 3 support added in EasyBuild 4
- else:
- cmd.append('easybuild')
-
- if not IS_PY3:
- # install vsc-base again at the end, to avoid that the one available on the system is used instead
- post_vsc_base = cmd[:]
- post_vsc_base[-1] = VSC_BASE + '<2.9.0'
-
- if not print_debug:
- cmd.insert(0, '--quiet')
-
- # There is no support for Python3 in the older vsc-* packages and EasyBuild 4 includes working versions of vsc-*
- if not IS_PY3:
- # install vsc-install version prior to 0.11.4, where mock was introduced as a dependency
- # workaround for problem reported in https://github.com/easybuilders/easybuild-framework/issues/2712
- # also stick to vsc-base < 2.9.0 to avoid requiring 'future' Python package as dependency
- for pkg in [VSC_INSTALL + '<0.11.4', VSC_BASE + '<2.9.0']:
- precmd = cmd[:-1] + [pkg]
- info("running pre-install command 'easy_install %s'" % (' '.join(precmd)))
- run_easy_install(precmd)
-
- info("installing EasyBuild with 'easy_install %s'\n" % (' '.join(cmd)))
- syntax_error_note = '\n'.join([
- "Note: a 'SyntaxError' may be reported for the easybuild/tools/py2vs3/py%s.py module." % ('3', '2')[IS_PY3],
- "You can safely ignore this message, it will not affect the functionality of the EasyBuild installation.",
- '',
- ])
- info(syntax_error_note)
- run_easy_install(cmd)
-
- if post_vsc_base:
- info("running post install command 'easy_install %s'" % (' '.join(post_vsc_base)))
- run_easy_install(post_vsc_base)
-
- pkg_egg_dir = find_egg_dir_for(targetdir_stage1, VSC_BASE)
- if pkg_egg_dir is None:
- # if vsc-base available on system is the same version as the one being installed,
- # the .egg directory may not get installed...
- # in that case, try to have it *copied* by also including --always-copy;
- # using --always-copy should be used as a last resort, since it can result in all kinds of problems
- info(".egg dir for vsc-base not found, trying again with --always-copy...")
- post_vsc_base.insert(0, '--always-copy')
- info("running post install command 'easy_install %s'" % (' '.join(post_vsc_base)))
- run_easy_install(post_vsc_base)
-
- # clear the Python search path, we only want the individual eggs dirs to be in the PYTHONPATH (see below)
- # this is needed to avoid easy-install.pth controlling what Python packages are actually used
- if distribute_egg_dir is not None:
- os.environ['PYTHONPATH'] = distribute_egg_dir
- else:
- del os.environ['PYTHONPATH']
-
- # template string to inject in template easyconfig
- templates = {}
-
- for pkg in EASYBUILD_PACKAGES:
- templates.update({pkg: ''})
-
- pkg_egg_dir = find_egg_dir_for(targetdir_stage1, pkg)
- if pkg_egg_dir is None:
- if pkg in [VSC_BASE, VSC_INSTALL]:
- # vsc-base is optional in older EasyBuild versions
- continue
-
- # prepend EasyBuild egg dirs to Python search path, so we know which EasyBuild we're using
- sys.path.insert(0, pkg_egg_dir)
- pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0]
- os.environ['PYTHONPATH'] = os.pathsep.join([pkg_egg_dir] + pythonpaths)
- debug("$PYTHONPATH: %s" % os.environ['PYTHONPATH'])
-
- if source_tarballs:
- if pkg in source_tarballs:
- templates.update({pkg: "'%s'," % os.path.basename(source_tarballs[pkg])})
- else:
- # determine per-package versions based on egg dirs, to use them in easyconfig template
- version_regex = re.compile('%s-([0-9a-z.-]*)-py[0-9.]*.egg' % pkg.replace('-', '_'))
- pkg_egg_dirname = os.path.basename(pkg_egg_dir)
- res = version_regex.search(pkg_egg_dirname)
- if res is not None:
- pkg_version = res.group(1)
- debug("Found version for easybuild-%s: %s" % (pkg, pkg_version))
- templates.update({pkg: "'%s-%s.tar.gz'," % (pkg, pkg_version)})
- else:
- tup = (pkg, pkg_egg_dirname, version_regex.pattern)
- error("Failed to determine version for easybuild-%s package from %s with %s" % tup)
-
- # figure out EasyBuild version via eb command line
- # note: EasyBuild uses some magic to determine the EasyBuild version based on the versions of the individual pkgs
- ver_regex = {'ver': '[0-9.]*[a-z0-9]*'}
- pattern = r"This is EasyBuild (?P%(ver)s) \(framework: %(ver)s, easyblocks: %(ver)s\)" % ver_regex
- version_re = re.compile(pattern)
- version_out_file = os.path.join(tmpdir, 'eb_version.out')
- eb_version_cmd = 'from easybuild.tools.version import this_is_easybuild; print(this_is_easybuild())'
- cmd = "%s -c '%s' > %s 2>&1" % (sys.executable, eb_version_cmd, version_out_file)
- debug("Determining EasyBuild version using command '%s'" % cmd)
- os.system(cmd)
- txt = open(version_out_file, "r").read()
- res = version_re.search(txt)
- if res:
- eb_version = res.group(1)
- debug("installing EasyBuild v%s" % eb_version)
- else:
- error("Stage 1 failed, could not determine EasyBuild version (txt: %s)." % txt)
-
- templates.update({'version': eb_version})
-
- # clear PYTHONPATH before we go to stage2
- # PYTHONPATH doesn't need to (and shouldn't) include the stage1 egg dirs
- os.environ['PYTHONPATH'] = ''
-
- # make sure we're getting the expected EasyBuild packages
- import easybuild.framework
- import easybuild.easyblocks
- pkgs_to_check = [easybuild.framework, easybuild.easyblocks]
- # vsc is part of EasyBuild 4
- if LooseVersion(eb_version) < LooseVersion('4'):
- import vsc.utils.fancylogger
- pkgs_to_check.append(vsc.utils.fancylogger)
-
- for pkg in pkgs_to_check:
- if tmpdir not in pkg.__file__:
- error("Found another %s than expected: %s" % (pkg.__name__, pkg.__file__))
- else:
- debug("Found %s in expected path, good!" % pkg.__name__)
-
- debug("templates: %s" % templates)
- return templates
-
-
-def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
- """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
-
- print('\n')
- info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
-
- preinstallopts = ''
-
- eb_looseversion = LooseVersion(templates['version'])
-
- # setuptools is no longer required for EasyBuild v4.0 & newer, so skip the setuptools stuff in that case
- if eb_looseversion < LooseVersion('4.0') and distribute_egg_dir is not None:
- # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
- # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
- # this is required for the easy_install from stage 0 to work
- preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
-
- # ensure that (latest) setuptools is installed as well alongside EasyBuild,
- # since it is a required runtime dependency for recent vsc-base and EasyBuild versions
- # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
- preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
-
- # vsc-install is no longer required for EasyBuild v4.0, so skip pre-installed vsc-install in that case
- if eb_looseversion < LooseVersion('4.0'):
- # vsc-install is a runtime dependency for the EasyBuild unit test suite,
- # and is easily picked up from stage1 rather than being actually installed, so force it
- vsc_install = "'%s<0.11.4'" % VSC_INSTALL
- if sourcepath:
- vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
- if len(vsc_install_tarball_paths) == 1:
- vsc_install = vsc_install_tarball_paths[0]
- preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
-
- templates.update({
- 'preinstallopts': preinstallopts,
- })
-
- # determine PyPI URLs for individual packages
- pkg_urls = []
- for pkg in EASYBUILD_PACKAGES:
-
- # vsc-base and vsc-install are not dependencies anymore for EasyBuild v4.0,
- # so skip them here for recent EasyBuild versions
- if eb_looseversion >= LooseVersion('4.0') and pkg in [VSC_INSTALL, VSC_BASE]:
- continue
-
- # format of pkg entries in templates: "'',"
- pkg_filename = templates[pkg][1:-2]
-
- # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
- # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
-
- # determine download URL via PyPI's 'simple' API
- pkg_simple = None
- try:
- pkg_simple = std_urllib.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
- except (std_urllib.URLError, std_urllib.HTTPError) as err:
- # failing to figure out the package download URl may be OK when source tarballs are provided
- if sourcepath:
- info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
- else:
- raise err
-
- if pkg_simple:
- if IS_PY3:
- pkg_simple = pkg_simple.decode('utf-8')
- pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
- res = pkg_url_part_regex.search(pkg_simple)
- if res:
- pkg_url = 'https://pypi.python.org/' + res.group(1)
- pkg_urls.append(pkg_url)
- elif sourcepath:
- info("Ignoring failure to determine source URL for '%s' (source tarballs are provided)" % pkg_filename)
- else:
- error_msg = "Failed to determine PyPI package URL for %s using pattern '%s': %s\n"
- error(error_msg % (pkg, pkg_url_part_regex.pattern, pkg_simple))
-
- # vsc-base and vsc-install are no longer required for EasyBuild v4.0.0,
- # so only include them in 'sources' for older versions
- sources_tmpl = "%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
- if eb_looseversion < LooseVersion('4.0'):
- sources_tmpl = "%(vsc-install)s%(vsc-base)s" + sources_tmpl
-
- templates.update({
- 'source_urls': '\n'.join(["'%s'," % x for x in pkg_urls]),
- 'sources': sources_tmpl % templates,
- 'pythonpath': distribute_egg_dir,
- })
-
- # create easyconfig file
- ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
- handle = open(ebfile, 'w')
- ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
- handle.write(ebfile_txt)
- handle.close()
- debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
-
- # set command line arguments for eb
- eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
- if print_debug:
- eb_args.extend(['--debug', '--logtostdout'])
- if forced_install:
- info("Performing FORCED installation, as requested...")
- eb_args.append('--force')
-
- # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
- # and set build and install path explicitely
- if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
- os.environ['EASYBUILD_PREFIX'] = tmpdir
- os.environ['EASYBUILD_BUILDPATH'] = tmpdir
- if install_path is not None:
- os.environ['EASYBUILD_INSTALLPATH'] = install_path
- else:
- # only for v1.3 and up
- eb_args.append('--prefix=%s' % tmpdir)
- eb_args.append('--buildpath=%s' % tmpdir)
- if install_path is not None:
- eb_args.append('--installpath=%s' % install_path)
- if sourcepath is not None:
- eb_args.append('--sourcepath=%s' % sourcepath)
-
- # make sure EasyBuild can find EasyBuild-*.eb easyconfig file when it needs to;
- # (for example when HierarchicalMNS is used as module naming scheme,
- # see https://github.com/easybuilders/easybuild-framework/issues/2393)
- eb_args.append('--robot-paths=%s:' % tmpdir)
-
- # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
- if install_path is not None:
- modules_path = det_modules_path(install_path)
- if not os.path.exists(modules_path):
- os.makedirs(modules_path)
- debug("Created path %s" % modules_path)
-
- debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
- sys.argv = eb_args
-
- # location to 'eb' command (from stage 1) may be expected to be included in $PATH
- # it usually is there after stage1, unless 'prep' is called again with another location
- # (only when stage 0 is not skipped)
- # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
- curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
- os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
- debug("$PATH: %s" % os.environ['PATH'])
-
- # install EasyBuild with EasyBuild
- from easybuild.main import main as easybuild_main
- easybuild_main()
-
- if print_debug:
- os.environ['EASYBUILD_DEBUG'] = '1'
-
- # make sure the EasyBuild module was actually installed
- # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
- # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
- from easybuild.tools.config import build_option, install_path, get_module_syntax
- from easybuild.framework.easyconfig.easyconfig import ActiveMNS
- eb_spec = {
- 'name': 'EasyBuild',
- 'hidden': False,
- 'toolchain': {'name': 'dummy', 'version': 'dummy'},
- 'version': templates['version'],
- 'versionprefix': '',
- 'versionsuffix': '',
- 'moduleclass': 'tools',
- }
-
- mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
- debug("EasyBuild module should have been installed to %s" % mod_path)
-
- eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
- debug("EasyBuild module name: %s" % eb_mod_name)
-
- eb_mod_path = os.path.join(mod_path, eb_mod_name)
- if get_module_syntax() == 'Lua':
- eb_mod_path += '.lua'
-
- if os.path.exists(eb_mod_path):
- info("EasyBuild module installed: %s" % eb_mod_path)
- else:
- error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
-
-
-def main():
- """Main script: bootstrap EasyBuild in stages."""
-
- self_txt = open(__file__).read()
- if IS_PY3:
- self_txt = self_txt.encode('utf-8')
-
- url = 'https://docs.easybuild.io/en/latest/Installation.html'
- info("Use of the EasyBuild boostrap script is DEPRECATED (since June 2021).")
- info("It is strongly recommended to use one of the installation methods outlined at %s instead!\n" % url)
- if not EASYBUILD_BOOTSTRAP_DEPRECATED:
- error("The EasyBuild bootstrap script will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined.")
- else:
- msg = "You have opted to continue with the EasyBuild bootstrap script by defining "
- msg += "$EASYBUILD_BOOTSTRAP_DEPRECATED. Good luck!\n"
- info(msg)
-
- info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest()))
- info("Found Python %s\n" % '; '.join(sys.version.split('\n')))
-
- # disallow running as root, since stage 2 will fail
- if os.getuid() == 0:
- error("Don't run the EasyBuild bootstrap script as root, "
- "since stage 2 (installing EasyBuild with EasyBuild) will fail.")
-
- # general option/argument parser
- if HAVE_ARGPARSE:
- bs_argparser = argparse.ArgumentParser()
- bs_argparser.add_argument("prefix", help="Installation prefix directory",
- type=str)
- bs_args = bs_argparser.parse_args()
-
- # prefix specification
- install_path = os.path.abspath(bs_args.prefix)
- else:
- bs_argparser = optparse.OptionParser(usage="usage: %prog [options] prefix")
- (bs_opts, bs_args) = bs_argparser.parse_args()
-
- # poor method, but should prefer argparse module for better pos arg support.
- if len(bs_args) < 1:
- error("Too few arguments\n" + bs_argparser.get_usage())
- elif len(bs_args) > 1:
- error("Too many arguments\n" + bs_argparser.get_usage())
-
- # prefix specification
- install_path = os.path.abspath(str(bs_args[0]))
-
- info("Installation prefix %s" % install_path)
-
- sourcepath = EASYBUILD_BOOTSTRAP_SOURCEPATH
- if sourcepath is not None:
- info("Fetching sources from %s..." % sourcepath)
-
- forcedversion = EASYBUILD_BOOTSTRAP_FORCE_VERSION
- if forcedversion:
- info("Forcing specified version %s..." % forcedversion)
- if IS_PY3 and LooseVersion(forcedversion) < LooseVersion('4'):
- error('Python 3 support is only available with EasyBuild 4.x but you are trying to install EasyBuild %s'
- % forcedversion)
-
- # create temporary dir for temporary installations
- tmpdir = tempfile.mkdtemp()
- debug("Going to use %s as temporary directory" % tmpdir)
- os.chdir(tmpdir)
-
- # check whether a module command is available, we need that
- modtool = check_module_command(tmpdir)
-
- # clean sys.path, remove paths that may contain EasyBuild packages or stuff installed with easy_install
- orig_sys_path = sys.path[:]
- sys.path = []
- for path in orig_sys_path:
- include_path = True
- # exclude path if it's potentially an EasyBuild/VSC package, providing the 'easybuild'/'vsc' namespace, resp.
- if any(os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']):
- include_path = False
- # exclude any .egg paths
- if path.endswith('.egg'):
- include_path = False
- # exclude any path that contains an easy-install.pth file
- if os.path.exists(os.path.join(path, 'easy-install.pth')):
- include_path = False
-
- if include_path:
- sys.path.append(path)
- else:
- debug("Excluding %s from sys.path" % path)
-
- debug("sys.path after cleaning: %s" % sys.path)
-
- # install EasyBuild in stages
-
- # STAGE 0: install distribute, which delivers easy_install
- distribute_egg_dir = None
- if EASYBUILD_BOOTSTRAP_SKIP_STAGE0:
- info("Skipping stage 0, using local distribute/setuptools providing easy_install")
- else:
- setuptools_loc = check_setuptools()
- if setuptools_loc:
- info("Suitable setuptools installation already found, skipping stage 0...")
- sys.path.insert(0, setuptools_loc)
- else:
- info("No suitable setuptools installation found, proceeding with stage 0...")
- distribute_egg_dir = stage0(tmpdir)
-
- # STAGE 1: install EasyBuild using easy_install to tmp dir
- templates = stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion)
-
- # add location to easy_install provided through stage0 to $PATH
- # this must be done *after* stage1, since $PATH is reset during stage1
- if distribute_egg_dir:
- prep(tmpdir)
-
- # make sure the active 'easy_install' is the right one (i.e. it matches the active setuptools installation)
- check_easy_install_cmd()
-
- # STAGE 2: install EasyBuild using EasyBuild (to final target installation dir)
- stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath)
-
- # clean up the mess
- debug("Cleaning up %s..." % tmpdir)
- shutil.rmtree(tmpdir)
-
- print('')
- info('Bootstrapping EasyBuild completed!\n')
-
- if install_path is not None:
- info('EasyBuild v%s was installed to %s, so make sure your $MODULEPATH includes %s' %
- (templates['version'], install_path, det_modules_path(install_path)))
- else:
- info('EasyBuild v%s was installed to configured install path, make sure your $MODULEPATH is set correctly.' %
- templates['version'])
- info('(default config => add "$HOME/.local/easybuild/modules/all" in $MODULEPATH)')
-
- print('')
- info("Run 'module load EasyBuild', and run 'eb --help' to get help on using EasyBuild.")
- info("Set $EASYBUILD_MODULES_TOOL to '%s' to use the same modules tool as was used now." % modtool)
- print('')
- info("By default, EasyBuild will install software to $HOME/.local/easybuild.")
- info("To install software with EasyBuild to %s, set $EASYBUILD_INSTALLPATH accordingly." % install_path)
- info("See http://easybuild.readthedocs.org/en/latest/Configuration.html for details on configuring EasyBuild.")
-
-
-# template easyconfig file for EasyBuild
-EASYBUILD_EASYCONFIG_TEMPLATE = """
-easyblock = 'EB_EasyBuildMeta'
-
-name = 'EasyBuild'
-version = '%(version)s'
-
-homepage = 'http://easybuilders.github.com/easybuild/'
-description = \"\"\"EasyBuild is a software build and installation framework
-written in Python that allows you to install software in a structured,
-repeatable and robust way.\"\"\"
-
-toolchain = {'name': 'dummy', 'version': 'dummy'}
-
-source_urls = [%(source_urls)s]
-sources = [%(sources)s]
-
-# EasyBuild is a (set of) Python packages, so it depends on Python
-# usually, we want to use the system Python, so no actual Python dependency is listed
-allow_system_deps = [('Python', SYS_PYTHON_VERSION)]
-
-preinstallopts = "%(preinstallopts)s"
-
-sanity_check_paths = {
- 'files': ['bin/eb'],
- 'dirs': ['lib'],
-}
-
-moduleclass = 'tools'
-"""
-
-# check Python version
-loose_pyver = LooseVersion(python_version())
-min_pyver2 = LooseVersion('2.6')
-min_pyver3 = LooseVersion('3.5')
-if loose_pyver < min_pyver2 or (loose_pyver >= LooseVersion('3') and loose_pyver < min_pyver3):
- sys.stderr.write("ERROR: Incompatible Python version: %s (should be Python 2 >= %s or Python 3 >= %s)\n"
- % (python_version(), min_pyver2, min_pyver3))
- sys.exit(1)
-
-# distribute_setup.py script (https://pypi.python.org/pypi/distribute)
-#
-# A compressed copy of a patched distribute_setup.py (version 0.6.49), generated like so:
-# >>> import base64
-# >>> import zlib
-# >>> base64.b64encode(zlib.compress(open("distribute_setup.py").read()))
-# compressed copy below is for setuptools 0.6c11, after applying patch:
-#
-# --- distribute_setup.py.orig 2013-07-05 03:50:13.000000000 +0200
-# +++ distribute_setup.py 2015-11-27 12:20:12.040032041 +0100
-# @@ -528,6 +528,8 @@
-# log.warn("--user requires Python 2.6 or later")
-# raise SystemExit(1)
-# install_args.append('--user')
-# + if options.prefix_install:
-# + install_args.append('--prefix=%s' % options.prefix_install)
-# return install_args
-#
-# def _parse_args():
-# @@ -539,6 +541,8 @@
-# '--user', dest='user_install', action='store_true', default=False,
-# help='install in user site package (requires Python 2.6 or later)')
-# parser.add_option(
-# + '--prefix', dest='prefix_install', metavar="PATH", help='install in prefix')
-# + parser.add_option(
-# '--download-base', dest='download_base', metavar="URL",
-# default=DEFAULT_URL,
-# help='alternative URL from where to download the distribute package')
-# @@ -549,7 +553,7 @@
-# def main(version=DEFAULT_VERSION):
-# """Install or upgrade setuptools and EasyInstall"""
-# options = _parse_args()
-# - tarball = download_setuptools(download_base=options.download_base)
-# + tarball = download_setuptools(version=version, download_base=options.download_base)
-# return _install(tarball, _build_install_args(options))
-#
-# if __name__ == '__main__':
-#
-DISTRIBUTE_SETUP_PY = """
-eJztPGtz2ziS3/UrcHK5SGVlxs7Mze6lTlOVmTizrs0mqdjZ/ZC4ZIiEJI75Gj6saH/9dTcAAiAh
-2bmZ/XBV592JJaLRaPS7G6BP/qPat9uymEyn05/Ksm3amlcsSeF3uupawdKiaXmW8TYFoMnVmu3L
-ju140bK2ZF0jWCParmrLMmsAFkdrVvH4nm9E0MjBqNrP2a9d0wJAnHWJYO02bSbrNEP08AWQ8FzA
-qrWI27Les13ablnazhkvEsaThCbgggjblhUr13Iljf/ly8mEwc+6LnOL+iWNszSvyrpFapeGWoJ3
-H4Wz0Q5r8VsHZDHOmkrE6TqN2YOoG2AG0mCmzvEzQCXlrshKnkzytK7Les7KmrjEC8azVtQFB55q
-ILPjOS0aA1RSsqZkqz1ruqrK9mmxmeCmeVXVZVWnOL2sUBjEj7u74Q7u7qLJ5AbZRfyNaWHEKFjd
-wecGtxLXaUXbU9IlKqtNzRNbnhEqxUQxr2z0p2bbtWnWf9v3A22aC/15XeS8jbf9kMgrpKf/zmv7
-K+yo4nUjJpNegLhGoyWXlZvJpK33L42QmxSVUw5/ur78uLy+urmciK+xgJ1d0fNLlICc0kOwBXtX
-FsLCpvfRrYDDsWgaqUmJWLOltI1lnCfhM15vmpmcgj/4FZCFsP9IfBVx1/JVJuYz9ica6uFqYGdd
-WOijGBgeEja2WLDzyUGiT8AOQDYgORBywtYgJEkQexF994cSecJ+68oWdA0fd7koWmD9GpYvQFUN
-GDxCTBV4AyAmR/IDgPnuRWCW1GQhQoHbnLljCk8A/wPbh/HxsMW2YHraTAN2ioAjOAUzHFKb/mwo
-INbBB7ViczuUTtlETcXBKEP49GH5z1dXN3M2YBp7Zsvs9eWbV5/e3iz/cfnx+ur9O1hveh79EH3/
-X9N+6NPHt/h427bVy+fPq32VRlJUUVlvnisf2TxvwI/F4nny3Lit59PJ9eXNpw8379+/vV6+efW3
-y9eDheKLi+nEBvrwt1+WV+/evMfx6fTL5O+i5Qlv+dk/pLd6yS6i88k7cLMvLQuf9KOnzeS6y3MO
-VsG+ws/kr2UuziqgkL5PXnVAeW1/PhM5TzP55G0ai6JRoK+F9C+EFx8AQSDBQzuaTCakxsoVheAa
-VvB7rn3TEtm+CGczbRTiK8SomLSVwoGEp8E2r8ClAg+0v4ny+wQ/g2fHcfAj0Y7XRRhcGiSgFKdN
-MFeTJWCZJctdAohAHzaijXeJwtC7DYICY97CnNCeSlCciJBOLiorUehtGZil2ofaszM5irOyERiO
-jIVuSkUsbroPHD0AOBi5dSCq4u02+hXgFWFzfJiBelm0fj6/nY03IrGYAcOwd+WO7cr63uaYhrao
-VELDsDXGcdUPste9sgcz278UZet4tEBHeFguUMjh4zNbOQYexqx3DToMmQMstwOXxnZ1CR+Trtaq
-Y+c2kUXHAIsQBC3QM7McXDhYBeOr8kEMJ6Fypm1cQhjVrvuFA6D8jXy4TgtY3adPUv0Mbhlyozpv
-ayF6bVOGs+pSABebTQj/zVlvP225RLD/N5v/A2ZDMqT4WzCQogfRTz2EZTsIawQh5W04dciMzn7D
-f1cYb1Bt6NEZfoN/agvRH6Kkcn9S77Xu1aLpstZVLSBEAisnoOUBBtW0DQ0bGmqeQk599Z6SpTD4
-ueyyhGYRH6W1bjZoncpGEtiSSrpDlb/P+zR8ueKN0Nu2Hici43u1KjJ6qCQKPjCB++y0Oav2p0kE
-/0fOelMb+DllhgpMM9QXiIHrEjTM8/DidvZ09iijAYL7vVglwoH9H6J18HOAS0brHnFHBEhZJJIP
-HljUbXg+Z0b+Kh93CzWrKlnpMlWnDEsqCBZkOFLeg6JI7XgxSNoGHFhYiZuXG3IHCzSArvYwYXHx
-n3MQznLN78Xipu5E73tzTg6g6aCS3FE6TzrKV02ZoRkjLyZmCUvTAAR/hzb3drxZSi4J9LVBdb9Z
-gklRHtlQWo38zcuky0SDpd2XfjeB4coQcOynnS+WaJz1Jg7ECbs27YDz6M8rquAhwoqapVR6diml
-0yzEgaoWa1Hj92EcBR/ZtGmLzKHkvdjbbk8JNHImjcjFH4fWSFXz4dSw4ccFkjmdjWZKl+U8VoVa
-6CLt6QKK3pXtm7IrEr81ufNU7v1zWayzNG49dVLFmwF/lfVvecPbtnYJAU+0TKy0ylt34Wylov7C
-i4bcnsiIM9L7mnK1hzhUybo8/V3u+LB2HhC1YcmPiykUvmq9mYew4T6OystdXIAhyoIxJocdzsBl
-D0VnG+6Y+zgdcg1IM6NdnULpOoKY3lDwpI0ldh/K6teFPy5OoUoFY0NR8weo0ah8/VJMx/jQBmNe
-BC1mqsqbAubdFltH1HNTrSL4VHcFBvCIfcgECMiLT7eTOMvLGkmNMe/WhK7Tumnn4P0AjXd6AIj3
-2quzs0/WvoLIA/+l+FKEP3d1Datke4mYndYzQO6EWBFhmYCp21iZpdDSNnzhjoms8VgI6CAbGJ3l
-Qj8PXPEtTsHkh5p95f2hDsa32MOTwvQhm/Hqtc9//W6zfZRMH4mjXPOYw1rGtYC4MvRXS9wcWWCf
-xlJKcCQT+r15Af14kgOVEyhHPJ1OX5u+b2+vlLv03WUwvqyMqSClyKk4n7aynVvwXMiAcKdov4O0
-m7LfFfaoH3iWOti16RVdvoIQ3G452XLvFggX9oSp7KBQ2zenQQ1oimB3Dj/uGHa2QnAS8VavLopE
-pibYuOcseB7MInYneXKHKzrVFzgYUQudpvelsl5FJDKw3xEH++lqD9R2j8sioQZ4xTGZWok1+hvs
-dcdtxzPThqf9tVjLtpEWw78lKXPCEQm1q7MsXVEgEnj2oQ4h6gwL24lljt5waaF44Zvbbv61RHXA
-hp9TgERYCG/+hR5QSZ/gYa5dD6AgIRhqLDLH5g8CWHqgzNGgKoOvY0QH+1INdctcB7WJRDt7iTx/
-9VCmqNUVmm7Sk2NazKOo3heIvfWQk2+mc9yS67MlUYpL4Wj4hH0UPHlOwZVhhIEEGEhnKzC5+zke
-u+xQCTEcSt8CmhyXdQ3ugqxvgAx2S1qt7SXFAzBIdBFeq7D+wVYsZgh1DPrAk0FSJdlIREtmzdl0
-N8xJAUjlBYjMjI2cppIErOXJMYAA3SQZwAN+T7yDRW140y8nCcNeMrIEJWNdboPDbsB9rcqv4bor
-YnRnygvSsD1OXfU5e/bsfjc7UnrIMx9TBKrp2jJea9dyLZ8fS5qHsJg3l1niS5iR2oe0lB3C0emK
-/UOp+vDhcKUI1wFJj5/3izwBRw8LmA7Pq/D4jepDrESfkNqYCW84ADw5px/OOyxDpTxaIxzRP6rO
-apkx3ccZ5BWBN63zQ9pab+ut1nQia4neIURDmIPDAB9QtCbo/5OiG4/vu4r8xVrm1qKQW4LYrkPS
-WjsBxDRTEQJcKDi8pUILEGvbg6wd6wQ2jScsNEmGdSfg/RDJXnPV02985ULYnW7FDqMmZtYHhMYq
-IdITINHEqLEkN2E25uwVLHAX+MKocniap/4kce3zUKT0E0s8sJ7tdqyR3mvhJQTNN680f7eAFHFH
-hKRpGXFNUlCInY76+BQieBC9f/s6Om3wkBQP4CP8Z9S3/4joZODErIn6xXKHGqNq4GPhjF9D36gi
-Xz80tOblAyTiYHRL+0glrDIeiy2YkdDHEIMMIW2wkTwGc4n/VNxDjHVOayCnU5uw5h5W0DXWNI5n
-w0xXX0Cxuvx+UoBqdZ8hUr9DnDu322nPsOF7hoXHMJDoxUd+eAV6cm+zZe2WXoYDpsFN1YF9ScNe
-d8CAyUAJ/l4+oAqITMgT/rJrMZtGp7Tje2OyuoIaZoIWc+ZMWo4ifiBPNd/igwkSyvCA6KUDK63J
-iUyjSdLNqhlz5jn+VocWip+jmGF4AXbAhx5uzlxqvKp0Aph3LBNt0DBU+96nD/nZq5niN2paaPdf
-54POrTqdwYst+NHinsZwTB4KxjlMHR4SKJBBHW27mhGWca7g00rwyfLUBdiqiVWcOHAGElh94sRq
-QASDnbju/JCnGTr3Q3C9q+drSNT7KwdIgOKKFQDX1LRWzaj+0KE3EsN8LIQAA554gkfPc6BiWa5+
-Dfvz6lmkTyoqKDahopOR8bEGhuPbFOHfMsdxuTa9oJSeOu0xN+wRt+aO7lmMXRCxav9ATAKjU4Eq
-fPKhlyw2zUZVKLfsSJ25NcYPwgJfHK0LD909mUvavsHtDUgZeUDNy7EL9PoeCX7E8wx9+M8ofxnI
-R7Pc3KnPVXp/2SdZKqW/LB7SuizQ2AZ5vV10Yzm8hfKOVaLO06bRVw9lGX2KHdD7tKqApumRbYzp
-U7md14kTwOE8T4qr7RO7w+Ky9CSCCcEwLxqws91aUjUcVI8fS1MdMqxG4FBnHt/fY2Y+8HdPcieT
-mVuz9BHYZJcnKq5BOIzvUatT/OarFQ7aiszmL3/55QwlifEMpCo/f4OtpKNs/GjQPxSgnhrkjZ5a
-gf5A4QLE5/c939xHR1kxe8TLPMq5Yxb9h5mWxQqnhFLa4q+i1KC5kUTt2D68jkLrT7JdezC2es4g
-FWbqSS/x2Dj9GuqoYwJdH3EpTKnzbV5vHl4O/JFJxtVtUQ34WX+AkJ2Ir1YMh1rr4uVtn9XR8Fzf
-KhVFl4uay/umdo8IQeV9bZn/nZ3VsFt55UZuY1guwDYARwSr1m2D/XTU4wUGNYnH04Bqy0oJh2ZW
-WdqGAa6zCGafzwbnkBYTNAPtxRSu4WGYpAoLVUmF93A5A+WUl2R/lPzx98occpvPBIncHdP5rbQa
-KoG9XSPqgE5RzO1vdSyKHWtvb8q3Sj/bYx1SK0fqOlT465jT4al1xqrv//oM2HvPguRwpBd3wnYi
-AKuKgQRP0mHl26CS1l2KpBSSKY0QOQZ2+yh43Pahrzs0Gbd4UZfckA3jDVnhCDNlmLxrIswi3TNu
-F+NHedCN6UlEbwcMqqdaUKjH8QW5bVdnnVTnZl+JEb+wGWudlb36cPXvJNhQZ4rU4TKgnq5q2hcJ
-7Tdt7JJJtg38gjphiWiFurCJ8RzvIPRBEfU2GWbY/fnjYkhepIeG2m2/AWQVZXJp0HXdrPGsonhB
-pynSN6J6km9MvxKxVV0+pIlI5MtA6VrB268dNVZrJFF3PfO5Oc3RjVa7HnJDiY8yvxQUNmDofEgw
-IJKHQL5bzmPfJl+AAFq2UB0drqCczAxsVMqxsW+Kqi15NhGJIlFBhC4kHthUcSYvkhrhOXSgtR/u
-8nmlqicr0gCHz996mj7WXePDJB1KojykHM+pFInhoZQUdc7tlnxDWnowLz3SWx/wxp037q7butGf
-WaJZ08G69JB0yG15/0IMResWBL9XnmN6ISblFRAprEOBjHdFvDWZnXkyDJ4f1YizcYx2siyll/Qk
-BPUk9FtcEqpKK5bz+h5vMpSM0/EzNxNW3UYVT3lygS8NBWex+6YBJBJ4rSgTZ6o1cSa+0nt8kECf
-5byAGJ4Etz2SF4eQ1CIuaw2pXqSihPPld7eYsEgKwJX4Bl5YoVQPn9/2fRBsF9rJ7Gf31aVbSK70
-tInGQXeevG+l9SKxbto3IlvLE4PFNII6Pxd4F6NZYLQyR13q7QA6W1cQ8uCW7lXUIEIpL/way2tb
-+nK83p+5I4LeHt+pLHcFVvJ5meDblzKy4BkHAdg9CXMIqpGkoN7U6QNFSpqI3eEGgv6uDd5HBaRr
-QYSM3g0AQhUH4CNMVjsK0MfLJJhnRESOr7bS/Ru8tSp061PjwbMFZR8QnVZ7thGtwhXO3EspKvGL
-y2pvf4eqD2oLdc9SMlS+I6EBFOPNZUybA6APt33GocUyyjT0AEb9bD3p6xxYiTwtxCoF4pTpalh1
-/gce70QT5tCj7gg1fC1QqmJwU8JARryCcjcJ1Rquk9R0LYhbEf5zFBDv6GGv6Pvv/yLTojjNQX5U
-nwGB538+P7eyv2wdKdlrpFL7+3Tlo0B3IFiD7Ldptu3bbmay/2bhizn73mIRGhnOF3UIGC7miOfF
-zFuXxHlFMBG2JCRgZM7lhnxDpKHE7AeoJfWh79RlhOhe7BdaAyO8QQH6izQHSEAww0ScsMkr75pB
-kBTS5RnA1Ztwb7aouyQPzBKHzBtonTXuEEnnn94mipa4y6DRdQSScrwF4oyMFd7ZGLCjc9UnAMZb
-2NkRQN18tUz2f3GVWKooTs6A+xkU3BfjYpsuag8Ked/VD0K2TFabEJRwqnwLvhOKl8jE4EUz+827
-UL2KbkIA/f6pfwvHvE4M3jRQby/rmMX6bk2pX8/qizFVHrve0VpZejXFDEVFhBW/ps/xUgeM8YdD
-GdpUdg/0XeuGfdDvXf+AIZreZxhcz5KX4q/3TSvyS4ytF1bSZBGunZpuUMyGm5CFhGcbfiQSfCHP
-Vfw4nL6FjabvB4P1SnkOBPkBR2S4ludaEMwKYTW1GgecENFFSfU+f/SeoAhNrbyNBIp4kiwlDlNS
-a57g3dmmXQS2POEhp2tDi6BpsbvYgrchyDXvMtUBMNdztyKrFjoDQzdC8qQ/GqBUi4XHpDsLnkKt
-6uBpel22B5gmtfyB14vph1c3f4W0aUSVbgE+YS19z/AMr272SzoXOu0VP318OzXs0FzyXmWWVOk/
-T4E5Gl7wpTxDXdQtzS1Hv52qHSilmOtEVO3IVjCdl5cgC5VC9T6CY1N4U4B0E1tltaqRtuYc/PyB
-i9tGe6+O/V0LCkGXvNkrKK2++u9qLFyTkO2sp7xSt/Bfil9os3SeOlY5fvv9mLcFj5zSNUqsRZfU
-7lwukTHLpfpLDH2GT+yCCf8D2cp1xw==
-"""
-if IS_PY3:
- DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii')
-DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib")
-
-# run main function as body of script
-main()
diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py
index ec1213e4b3..2063b8dfb1 100755
--- a/easybuild/scripts/clean_gists.py
+++ b/easybuild/scripts/clean_gists.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2014-2024 Ward Poelmans
+# Copyright 2014-2025 Ward Poelmans
#
# https://github.com/easybuilders/easybuild
#
@@ -29,6 +29,7 @@
import re
+from urllib.request import HTTPError, URLError
from easybuild.base import fancylogger
from easybuild.base.generaloption import simple_option
@@ -37,7 +38,6 @@
from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO
from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token
from easybuild.tools.options import EasyBuildOptions
-from easybuild.tools.py2vs3 import HTTPError, URLError
HTTP_DELETE_OK = 204
diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py
index c3607e47c7..6fc6141f00 100755
--- a/easybuild/scripts/findPythonDeps.py
+++ b/easybuild/scripts/findPythonDeps.py
@@ -1,4 +1,36 @@
#!/usr/bin/env python
+##
+# Copyright 2021-2025 Alexander Grund
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Find Python dependencies for a given Python package after loading dependencies specified in an EasyConfig.
+This is intended for writing or updating PythonBundle EasyConfigs:
+ 1. Create a EasyConfig with at least 'Python' as a dependency.
+ When updating to a new toolchain it is a good idea to reduce the dependencies to a minimum
+ as e.g. the new "Python" module might have different packages included.
+ 2. Run this script
+ 3. For each dependency found by this script search existing EasyConfigs for ones providing that Python package.
+ E.g many are contained in Python-bundle-PyPI. Some can be updated from an earlier toolchain.
+ 4. Add those EasyConfigs as dependencies to your new EasyConfig.
+ 5. Rerun this script so it takes the newly provided packages into account.
+ You can do steps 3-5 iteratively adding EasyConfig-dependencies one-by-one.
+ 6. Finally you copy the packages found by this script as "exts_list" into the new EasyConfig.
+ You usually want the list printed as "in install order", the format is already suitable to be copied as-is.
+"""
import argparse
import json
@@ -8,12 +40,13 @@
import subprocess
import sys
import tempfile
+import textwrap
from contextlib import contextmanager
from pprint import pprint
try:
import pkg_resources
except ImportError as e:
- print('pkg_resources could not be imported: %s\nYou might need to install setuptools!' % e)
+ print(f'pkg_resources could not be imported: {e}\nYou might need to install setuptools!')
sys.exit(1)
try:
@@ -22,6 +55,7 @@
_canonicalize_regex = re.compile(r"[-_.]+")
def canonicalize_name(name):
+ """Fallback if the import doesn't work with same behavior."""
return _canonicalize_regex.sub("-", name).lower()
@@ -36,19 +70,19 @@ def temporary_directory(*args, **kwargs):
def extract_pkg_name(package_spec):
- return re.split('<|>|=|~', args.package, 1)[0]
+ """Get the package name from a specification such as 'package>=3.42'"""
+ return re.split('<|>|=|~', package_spec, 1)[0]
-def can_run(cmd, argument):
+def can_run(cmd, *arguments):
"""Check if the given cmd and argument can be run successfully"""
- with open(os.devnull, 'w') as FNULL:
- try:
- return subprocess.call([cmd, argument], stdout=FNULL, stderr=subprocess.STDOUT) == 0
- except (subprocess.CalledProcessError, OSError):
- return False
+ try:
+ return subprocess.call([cmd, *arguments], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0
+ except (subprocess.CalledProcessError, OSError):
+ return False
-def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
+def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
"""Run the command and return the return code and output"""
extra_args = kwargs or {}
if sys.version_info[0] >= 3:
@@ -59,14 +93,14 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
if p.returncode != 0:
if err:
err = "\nSTDERR:\n" + err
- raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err))
+ raise RuntimeError(f'Failed to {action_desc}: {out}{err}')
return out
def run_in_venv(cmd, venv_path, action_desc):
"""Run the given command in the virtualenv at the given path"""
- cmd = 'source %s/bin/activate && %s' % (venv_path, cmd)
- return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash')
+ cmd = f'source {venv_path}/bin/activate && {cmd}'
+ return run_shell_cmd(cmd, action_desc, shell=True, executable='/bin/bash')
def get_dep_tree(package_spec, verbose):
@@ -78,43 +112,59 @@ def get_dep_tree(package_spec, verbose):
venv_dir = os.path.join(tmp_dir, 'venv')
if verbose:
print('Creating virtualenv at ' + venv_dir)
- run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv')
+ run_shell_cmd(
+ [sys.executable, '-m', 'venv', '--system-site-packages', venv_dir], action_desc='create virtualenv'
+ )
if verbose:
print('Updating pip in virtualenv')
run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip')
if verbose:
- print('Installing %s into virtualenv' % package_spec)
- out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec)
- print('%s installed: %s' % (package_spec, out))
+ print(f'Installing {package_spec} into virtualenv')
+ out = run_in_venv(f'pip install "{package_spec}"', venv_dir, action_desc='install ' + package_spec)
+ print(f'{package_spec} installed: {out}')
# install pipdeptree, figure out dependency tree for installed package
run_in_venv('pip install pipdeptree', venv_dir, action_desc='install pipdeptree')
- dep_tree = run_in_venv('pipdeptree -j -p "%s"' % package_name,
+ dep_tree = run_in_venv(f'pipdeptree -j -p "{package_name}"',
venv_dir, action_desc='collect dependencies')
return json.loads(dep_tree)
def find_deps(pkgs, dep_tree):
"""Recursively resolve dependencies of the given package(s) and return them"""
+ MAX_PACKAGES = 1000
res = []
- for orig_pkg in pkgs:
- pkg = canonicalize_name(orig_pkg)
- matching_entries = [entry for entry in dep_tree
- if pkg in (entry['package']['package_name'], entry['package']['key'])]
- if not matching_entries:
+ next_pkgs = set(pkgs)
+ # Don't check any package multiple times to avoid infinite recursion
+ seen_pkgs = set()
+ count = 0
+ while next_pkgs:
+ cur_pkgs = next_pkgs - seen_pkgs
+ seen_pkgs.update(cur_pkgs)
+ next_pkgs = set()
+ for orig_pkg in cur_pkgs:
+ count += 1
+ if count > MAX_PACKAGES:
+ raise RuntimeError(f"Aborting after checking {MAX_PACKAGES} packages. Possibly cycle detected!")
+ pkg = canonicalize_name(orig_pkg)
matching_entries = [entry for entry in dep_tree
- if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
- if not matching_entries:
- raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree))
- if len(matching_entries) > 1:
- raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree))
- entry = matching_entries[0]
- res.append((entry['package']['package_name'], entry['package']['installed_version']))
- deps = (dep['package_name'] for dep in entry['dependencies'])
- res.extend(find_deps(deps, dep_tree))
+ if pkg in (entry['package']['package_name'], entry['package']['key'])]
+ if not matching_entries:
+ matching_entries = [entry for entry in dep_tree
+ if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
+ if not matching_entries:
+ raise RuntimeError(f"Found no installed package for '{pkg}' in {dep_tree}")
+ if len(matching_entries) > 1:
+ raise RuntimeError(f"Found multiple installed packages for '{pkg}' in {dep_tree}")
+ entry = matching_entries[0]
+ res.append(entry['package'])
+ # Add dependencies to list of packages to check next
+ # Could call this function recursively but that might exceed the max recursion depth
+ next_pkgs.update(dep['package_name'] for dep in entry['dependencies'])
return res
def print_deps(package, verbose):
+ """Print dependencies of the given package that are not installed yet in a format usable as 'exts_list'"""
if verbose:
print('Getting dep tree of ' + package)
dep_tree = get_dep_tree(package, verbose)
@@ -131,13 +181,18 @@ def print_deps(package, verbose):
res = []
handled = set()
for dep in reversed(deps):
- if dep not in handled:
- handled.add(dep)
- if dep[0] in installed_modules:
+ # Tuple as we need it for exts_list
+ dep_entry = (dep['package_name'], dep['installed_version'])
+ if dep_entry not in handled:
+ handled.add(dep_entry)
+ # Need to check for key and package_name as naming is not consistent. E.g.:
+ # "PyQt5-sip": 'key': 'pyqt5-sip', 'package_name': 'PyQt5-sip'
+ # "jupyter-core": 'key': 'jupyter-core', 'package_name': 'jupyter_core'
+ if dep['key'] in installed_modules or dep['package_name'] in installed_modules:
if verbose:
- print("Skipping installed module '%s'" % dep[0])
+ print(f"Skipping installed module '{dep['package_name']}'")
else:
- res.append(dep)
+ res.append(dep_entry)
print("List of dependencies in (likely) install order:")
pprint(res, indent=4)
@@ -145,69 +200,73 @@ def print_deps(package, verbose):
pprint(sorted(res), indent=4)
-examples = [
- 'Example usage with EasyBuild (after installing dependency modules):',
- '\t' + sys.argv[0] + ' --ec TensorFlow-2.3.4.eb tensorflow==2.3.4',
- 'Which is the same as:',
- '\t' + ' && '.join(['eb TensorFlow-2.3.4.eb --dump-env',
- 'source TensorFlow-2.3.4.env',
- sys.argv[0] + ' tensorflow==2.3.4',
- ]),
-]
-parser = argparse.ArgumentParser(
- description='Find dependencies of Python packages by installing it in a temporary virtualenv. ',
- epilog='\n'.join(examples),
- formatter_class=argparse.RawDescriptionHelpFormatter
-)
-parser.add_argument('package', metavar='python-pkg-spec',
- help='Python package spec, e.g. tensorflow==2.3.4')
-parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. '
- 'You need to have dependency modules installed already!')
-parser.add_argument('--verbose', help='Verbose output', action='store_true')
-args = parser.parse_args()
-
-if args.ec:
- if not can_run('eb', '--version'):
- print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!')
- sys.exit(1)
- if args.verbose:
- print('Checking with EasyBuild for missing dependencies')
- missing_dep_out = run_cmd(['eb', args.ec, '--missing'],
- capture_stderr=False,
- action_desc='Get missing dependencies'
- )
- excluded_dep = '(%s)' % os.path.basename(args.ec)
- missing_deps = [dep for dep in missing_dep_out.split('\n')
- if dep.startswith('*') and excluded_dep not in dep
- ]
- if missing_deps:
- print('You need to install all modules on which %s depends first!' % args.ec)
- print('\n\t'.join(['Missing:'] + missing_deps))
- sys.exit(1)
-
- # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
- ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
- with temporary_directory() as tmp_dir:
- old_dir = os.getcwd()
- os.chdir(tmp_dir)
- if args.verbose:
- print('Running EasyBuild to get build environment')
- run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
- os.chdir(old_dir)
+def main():
+ """Entrypoint of the script"""
+ examples = textwrap.dedent(f"""
+ Example usage with EasyBuild (after installing dependency modules):
+ {sys.argv[0]} --ec TensorFlow-2.3.4.eb tensorflow==2.3.4
+ Which is the same as:
+ eb TensorFlow-2.3.4.eb --dump-env && source TensorFlow-2.3.4.env && {sys.argv[0]} tensorflow==2.3.4
+ Using the '--ec' parameter is recommended as the latter requires manually updating the .env file
+ after each change to the EasyConfig.
+ """)
+ parser = argparse.ArgumentParser(
+ description='Find dependencies of Python packages by installing it in a temporary virtualenv. ',
+ epilog='\n'.join(examples),
+ formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ parser.add_argument('package', metavar='python-pkg-spec',
+ help='Python package spec, e.g. tensorflow==2.3.4')
+ parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. '
+ 'You need to have dependency modules installed already!')
+ parser.add_argument('--verbose', help='Verbose output', action='store_true')
+ args = parser.parse_args()
- cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package)
+ if args.ec:
+ if not can_run('eb', '--version'):
+ print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!')
+ sys.exit(1)
if args.verbose:
- cmd += ' --verbose'
- print('Restarting script in new build environment')
-
- out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash')
- print(out)
-else:
- if not can_run('virtualenv', '--version'):
- print('Virtualenv not found or executable. ' +
- 'Make sure it is installed (e.g. in the currently loaded Python module)!')
- sys.exit(1)
- if 'PIP_PREFIX' in os.environ:
- print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.")
- del os.environ['PIP_PREFIX']
- print_deps(args.package, args.verbose)
+ print('Checking with EasyBuild for missing dependencies')
+ missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'],
+ capture_stderr=False,
+ action_desc='Get missing dependencies')
+ excluded_dep = f'({os.path.basename(args.ec)})'
+ missing_deps = [dep for dep in missing_dep_out.split('\n')
+ if dep.startswith('*') and excluded_dep not in dep
+ ]
+ if missing_deps:
+ print(f'You need to install all modules on which {args.ec} depends first!')
+ print('\n\t'.join(['Missing:'] + missing_deps))
+ sys.exit(1)
+
+ # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
+ ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
+ with temporary_directory() as tmp_dir:
+ old_dir = os.getcwd()
+ os.chdir(tmp_dir)
+ if args.verbose:
+ print('Running EasyBuild to get build environment')
+ run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
+ os.chdir(old_dir)
+
+ cmd = f"source {tmp_dir}/*.env && python {sys.argv[0]} '{args.package}'"
+ if args.verbose:
+ cmd += ' --verbose'
+ print('Restarting script in new build environment')
+
+ out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash')
+ print(out)
+ else:
+ if not can_run(sys.executable, '-m', 'venv', '-h'):
+ print("'venv' module not found. This should be available in Python 3.3+.")
+ sys.exit(1)
+ if 'PIP_PREFIX' in os.environ:
+ print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.")
+ del os.environ['PIP_PREFIX']
+ os.environ['PYTHONNOUSERSITE'] = '1'
+ print_deps(args.package, args.verbose)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh
index 66b7e34bd8..918e9b2ca5 100755
--- a/easybuild/scripts/findUpdatedEcs.sh
+++ b/easybuild/scripts/findUpdatedEcs.sh
@@ -8,7 +8,7 @@ YELLOW='\033[0;33m'
NC='\033[0m'
function printError {
- echo -e "${RED}$@${NC}"
+ echo -e "${RED}$*${NC}"
}
verbose=0
@@ -23,11 +23,11 @@ function checkModule {
moduleStr="$moduleName/$moduleVersion"
printVerbose "Processing $moduleStr"
ec_glob=( "$moduleFolder/easybuild/"*.eb )
- if [[ ! -e "${ec_glob[@]}" ]]; then
+ if [[ ! -e "${ec_glob[0]}" ]]; then
printError "=== Did not find installed EC for $moduleStr"
return
fi
- ec_installed="$ec_glob"
+ ec_installed="${ec_glob[0]}"
ec_filename=$(basename "$ec_installed")
# Try with most likely location first for speed
first_letter=${ec_filename:0:1}
@@ -67,15 +67,15 @@ if path=$(which eb 2>/dev/null); then
fi
function usage {
- echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
+ echo "Usage: $(basename "$0") [--verbose] [--short] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
echo
echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed."
echo "Can either check the currently loaded modules or all modules installed in a specific location"
echo
echo "--verbose Verbose status output while checking"
- echo "--loaded Check only currently loaded modules"
echo "--short Only show filename of changed ECs"
echo "--diff Show diff of changed module files"
+ echo "--loaded Check only currently loaded modules"
echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries"
echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}"
exit 0
@@ -115,6 +115,7 @@ done
if [ -z "$easyconfigFolder" ]; then
printError "Folder to easyconfigs not given!" && exit 1
fi
+
if [ -z "$modulesFolder" ]; then
if (( checkLoadedModules == 0 )); then
printError "Need either --modules or --loaded to specify what to check!" && exit 1
@@ -123,13 +124,18 @@ elif (( checkLoadedModules == 1 )); then
printError "Cannot specify --modules and --loaded!" && exit 1
fi
+if (( showDiff == 1 && short == 1 )); then
+ printError "Cannot specify --diff and --short" && exit 1
+fi
+
if [ -d "$easyconfigFolder/easybuild/easyconfigs" ]; then
easyconfigFolder="$easyconfigFolder/easybuild/easyconfigs"
fi
if (( checkLoadedModules == 1 )); then
- for varname in $(compgen -A variable | grep '^EBROOT'); do
- checkModule "${!varname}"
+ IFS=$'\n' read -r -d '' -a unique_module_paths <<< "$(for varname in $(compgen -A variable | grep '^EBROOT'); do echo "${!varname}"; done | sort -u )" || true
+ for module in "${unique_module_paths[@]}"; do
+ checkModule "$module"
done
else
for module in "$modulesFolder"/*/*/easybuild; do
diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py
index a14f18caf8..bd56b88981 100755
--- a/easybuild/scripts/fix_docs.py
+++ b/easybuild/scripts/fix_docs.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py
index f6958572fa..bd5906e110 100755
--- a/easybuild/scripts/mk_tmpl_easyblock_for.py
+++ b/easybuild/scripts/mk_tmpl_easyblock_for.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,7 @@
import sys
from optparse import OptionParser
-from easybuild.tools.filetools import encode_class_name
+from easybuild.tools.filetools import encode_class_name, EASYBLOCK_CLASS_PREFIX
# parse options
parser = OptionParser()
@@ -83,8 +83,9 @@
# determine parent easyblock class
parent_import = "from easybuild.framework.easyblock import EasyBlock"
if not options.parent == "EasyBlock":
- if options.parent.startswith('EB_'):
- ebmod = options.parent[3:].lower() # FIXME: here we should actually decode the encoded class name
+ if options.parent.startswith(EASYBLOCK_CLASS_PREFIX):
+ # FIXME: here we should actually decode the encoded class name
+ ebmod = options.parent[len(EASYBLOCK_CLASS_PREFIX):].lower()
else:
ebmod = "generic.%s" % options.parent.lower()
parent_import = "from easybuild.easyblocks.%s import %s" % (ebmod, options.parent)
@@ -122,7 +123,7 @@
import easybuild.tools.toolchain as toolchain
%(parent_import)s
from easybuild.framework.easyconfig import CUSTOM, MANDATORY
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
class %(class_name)s(%(parent)s):
@@ -150,7 +151,7 @@ def configure_step(self):
env.setvar('CUSTOM_ENV_VAR', 'foo')
cmd = "configure command"
- run_cmd(cmd, log_all=True, simple=True, log_ok=True)
+ run_shell_cmd(cmd)
# complete configuration with configure_method of parent
super(%(class_name)s, self).configure_step()
@@ -165,22 +166,22 @@ def build_step(self):
comp_fam = comp_map[self.toolchain.comp_family()]
# enable parallel build
- par = self.cfg['parallel']
+ par = self.cfg.parallel
cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam)
- run_cmd(cmd, log_all=True, simple=True, log_ok=True)
+ run_shell_cmd(cmd)
def test_step(self):
\"\"\"Custom built-in test procedure for %(name)s.\"\"\"
if self.cfg['runtest']:
cmd = "test-command"
- run_cmd(cmd, simple=True, log_all=True, log_ok=True)
+ run_shell_cmd(cmd)
def install_step(self):
\"\"\"Custom install procedure for %(name)s.\"\"\"
cmd = "install command"
- run_cmd(cmd, log_all=True, simple=True, log_ok=True)
+ run_shell_cmd(cmd)
def sanity_check_step(self):
\"\"\"Custom sanity check for %(name)s.\"\"\"
diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py
index b477aa63d8..611c6a414c 100755
--- a/easybuild/scripts/rpath_args.py
+++ b/easybuild/scripts/rpath_args.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in
index 44a5aac43a..558a494581 100644
--- a/easybuild/scripts/rpath_wrapper_template.sh.in
+++ b/easybuild/scripts/rpath_wrapper_template.sh.in
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py
index 2f468f2812..da63a3c7c2 100644
--- a/easybuild/toolchains/__init__.py
+++ b/easybuild/toolchains/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py
index 9e25c6ec57..9a31fa2a61 100644
--- a/easybuild/toolchains/cgmpich.py
+++ b/easybuild/toolchains/cgmpich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py
index 5629ee5be3..e6fa8e31e9 100644
--- a/easybuild/toolchains/cgmpolf.py
+++ b/easybuild/toolchains/cgmpolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py
index 76e306b12d..1c108b4709 100644
--- a/easybuild/toolchains/cgmvapich2.py
+++ b/easybuild/toolchains/cgmvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py
index e68aa90fe1..a61507caff 100644
--- a/easybuild/toolchains/cgmvolf.py
+++ b/easybuild/toolchains/cgmvolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py
index 9b9267bf78..f7c4500831 100644
--- a/easybuild/toolchains/cgompi.py
+++ b/easybuild/toolchains/cgompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py
index 8e2ea8503e..a2a9b23e7e 100644
--- a/easybuild/toolchains/cgoolf.py
+++ b/easybuild/toolchains/cgoolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py
index c1e09e68ef..b22a78d883 100644
--- a/easybuild/toolchains/clanggcc.py
+++ b/easybuild/toolchains/clanggcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py
index 9c1ba469c5..37b7514bba 100644
--- a/easybuild/toolchains/compiler/__init__.py
+++ b/easybuild/toolchains/compiler/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py
index 8b7b16e8e3..4477454913 100644
--- a/easybuild/toolchains/compiler/clang.py
+++ b/easybuild/toolchains/compiler/clang.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
@@ -34,7 +34,6 @@
"""
import easybuild.tools.systemtools as systemtools
-from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.toolchain.compiler import Compiler
@@ -56,10 +55,10 @@ class Clang(Compiler):
'basic-block-vectorize': (False, "Basic block vectorization"),
}
COMPILER_UNIQUE_OPTION_MAP = {
- 'unroll': 'funroll-loops',
- 'loop-vectorize': ['fvectorize'],
- 'basic-block-vectorize': ['fslp-vectorize'],
- 'optarch': 'march=native',
+ 'unroll': '-funroll-loops',
+ 'loop-vectorize': ['-fvectorize'],
+ 'basic-block-vectorize': ['-fslp-vectorize'],
+ 'optarch': '-march=native',
# Clang's options do not map well onto these precision modes. The flags enable and disable certain classes of
# optimizations.
#
@@ -81,38 +80,31 @@ class Clang(Compiler):
#
# 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details
# flags for strict and precise for robustness against future changes.
- 'strict': ['fno-fast-math'],
- 'precise': ['fno-unsafe-math-optimizations'],
+ 'strict': ['-fno-fast-math'],
+ 'precise': ['-fno-unsafe-math-optimizations'],
'defaultprec': [],
- 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'],
- 'veryloose': ['ffast-math'],
- 'vectorize': {False: 'fno-vectorize', True: 'fvectorize'},
+ 'loose': ['-ffast-math', '-fno-unsafe-math-optimizations'],
+ 'veryloose': ['-ffast-math'],
+ 'vectorize': {False: '-fno-vectorize', True: '-fvectorize'},
}
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
- (systemtools.POWER, systemtools.POWER): 'mcpu=native', # no support for march=native on POWER
- (systemtools.POWER, systemtools.POWER_LE): 'mcpu=native', # no support for march=native on POWER
- (systemtools.X86_64, systemtools.AMD): 'march=native',
- (systemtools.X86_64, systemtools.INTEL): 'march=native',
+ (systemtools.POWER, systemtools.POWER): '-mcpu=native', # no support for march=native on POWER
+ (systemtools.POWER, systemtools.POWER_LE): '-mcpu=native', # no support for march=native on POWER
+ (systemtools.X86_64, systemtools.AMD): '-march=native',
+ (systemtools.X86_64, systemtools.INTEL): '-march=native',
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
- (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
- (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic',
- (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic',
+ (systemtools.RISCV64, systemtools.RISCV): '-march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
+ (systemtools.X86_64, systemtools.AMD): '-march=x86-64 -mtune=generic',
+ (systemtools.X86_64, systemtools.INTEL): '-march=x86-64 -mtune=generic',
}
COMPILER_CC = 'clang'
COMPILER_CXX = 'clang++'
- COMPILER_C_UNIQUE_FLAGS = []
+ COMPILER_C_UNIQUE_OPTIONS = []
LIB_MULTITHREAD = ['pthread']
LIB_MATH = ['m']
-
- def _set_compiler_vars(self):
- """Set compiler variables."""
- super(Clang, self)._set_compiler_vars()
-
- if self.options.get('32bit', None):
- raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit Clang in EasyBuild")
diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py
index 73b6a103e2..bcb0197b93 100644
--- a/easybuild/toolchains/compiler/craype.py
+++ b/easybuild/toolchains/compiler/craype.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -65,7 +65,7 @@ class CrayPECompiler(Compiler):
COMPILER_UNIQUE_OPTS = {
'dynamic': (True, "Generate dynamically linked executable"),
- 'mpich-mt': (False, "Directs the driver to link in an alternate version of the Cray-MPICH library which \
+ 'mpich-mt': (False, "Directs the driver to link in an alternative version of the Cray-MPICH library which \
provides fine-grained multi-threading support to applications that perform \
MPI operations within threaded regions."),
'optarch': (False, "Enable architecture optimizations"),
@@ -76,8 +76,8 @@ class CrayPECompiler(Compiler):
# handle shared and dynamic always via $CRAYPE_LINK_TYPE environment variable, don't pass flags to wrapper
'shared': '',
'dynamic': '',
- 'verbose': 'craype-verbose',
- 'mpich-mt': 'craympich-mt',
+ 'verbose': '-craype-verbose',
+ 'mpich-mt': '-craympich-mt',
}
COMPILER_CC = 'cc'
@@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs):
"""Constructor."""
super(CrayPECompiler, self).__init__(*args, **kwargs)
# 'register' additional toolchain options that correspond to a compiler flag
- self.COMPILER_FLAGS.extend(['dynamic', 'mpich-mt'])
+ self.COMPILER_OPTIONS.extend(['dynamic', 'mpich-mt'])
# use name of PrgEnv module as name of module that provides compiler
self.COMPILER_MODULE_NAME = ['PrgEnv-%s' % self.PRGENV_MODULE_NAME_SUFFIX]
@@ -139,7 +139,7 @@ class CrayPEGCC(CrayPECompiler):
def __init__(self, *args, **kwargs):
"""CrayPEGCC constructor."""
super(CrayPEGCC, self).__init__(*args, **kwargs)
- for precflag in self.COMPILER_PREC_FLAGS:
+ for precflag in self.COMPILER_PREC_OPTIONS:
self.COMPILER_UNIQUE_OPTION_MAP[precflag] = Gcc.COMPILER_UNIQUE_OPTION_MAP[precflag]
@@ -151,7 +151,7 @@ class CrayPEIntel(CrayPECompiler):
def __init__(self, *args, **kwargs):
"""CrayPEIntel constructor."""
super(CrayPEIntel, self).__init__(*args, **kwargs)
- for precflag in self.COMPILER_PREC_FLAGS:
+ for precflag in self.COMPILER_PREC_OPTIONS:
self.COMPILER_UNIQUE_OPTION_MAP[precflag] = IntelIccIfort.COMPILER_UNIQUE_OPTION_MAP[precflag]
@@ -163,8 +163,8 @@ class CrayPEPGI(CrayPECompiler):
def __init__(self, *args, **kwargs):
"""CrayPEPGI constructor."""
super(CrayPEPGI, self).__init__(*args, **kwargs)
- self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = 'mp'
- for precflag in self.COMPILER_PREC_FLAGS:
+ self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = '-mp'
+ for precflag in self.COMPILER_PREC_OPTIONS:
self.COMPILER_UNIQUE_OPTION_MAP[precflag] = Pgi.COMPILER_UNIQUE_OPTION_MAP[precflag]
@@ -176,6 +176,6 @@ class CrayPECray(CrayPECompiler):
def __init__(self, *args, **kwargs):
"""CrayPEIntel constructor."""
super(CrayPECray, self).__init__(*args, **kwargs)
- self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = 'homp'
- for precflag in self.COMPILER_PREC_FLAGS:
+ self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = '-homp'
+ for precflag in self.COMPILER_PREC_OPTIONS:
self.COMPILER_UNIQUE_OPTION_MAP[precflag] = []
diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py
index 3fd3d705f1..24e8c038bb 100644
--- a/easybuild/toolchains/compiler/cuda.py
+++ b/easybuild/toolchains/compiler/cuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -60,8 +60,8 @@ class Cuda(Compiler):
# always C++ compiler command, even for C!
COMPILER_CUDA_UNIQUE_OPTION_MAP = {
- '_opt_CUDA_CC': 'ccbin="%(CXX_base)s"',
- '_opt_CUDA_CXX': 'ccbin="%(CXX_base)s"',
+ '_opt_CUDA_CC': '-ccbin="%(CXX_base)s"',
+ '_opt_CUDA_CXX': '-ccbin="%(CXX_base)s"',
}
COMPILER_CUDA_CC = 'nvcc'
@@ -90,14 +90,14 @@ def _set_compiler_flags(self):
# note: using $LIBS will yield the use of -lcudart in Xlinker, which is silly, but fine
cuda_flags = [
- 'Xcompiler="%s"' % str(self.variables['CXXFLAGS']),
- 'Xlinker="%s %s"' % (str(self.variables['LDFLAGS']), str(self.variables['LIBS'])),
+ '-Xcompiler="%s"' % str(self.variables['CXXFLAGS']),
+ '-Xlinker="%s %s"' % (str(self.variables['LDFLAGS']), str(self.variables['LIBS'])),
]
self.variables.nextend('CUDA_CFLAGS', cuda_flags)
self.variables.nextend('CUDA_CXXFLAGS', cuda_flags)
# add gencode compiler flags to list of flags for compiler variables
for gencode_val in self.options.get('cuda_gencode', []):
- gencode_option = 'gencode %s' % gencode_val
+ gencode_option = '-gencode %s' % gencode_val
self.variables.nappend('CUDA_CFLAGS', gencode_option)
self.variables.nappend('CUDA_CXXFLAGS', gencode_option)
diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py
index 437ec34048..c49d36fc33 100644
--- a/easybuild/toolchains/compiler/fujitsu.py
+++ b/easybuild/toolchains/compiler/fujitsu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -58,23 +58,23 @@ class FujitsuCompiler(Compiler):
COMPILER_FC = 'frt'
COMPILER_UNIQUE_OPTION_MAP = {
- DEFAULT_OPT_LEVEL: 'O2',
- 'lowopt': 'O1',
- 'noopt': 'O0',
- 'opt': 'Kfast', # -O3 -Keval,fast_matmul,fp_contract,fp_relaxed,fz,ilfunc,mfunc,omitfp,simd_packed_promotion
+ DEFAULT_OPT_LEVEL: '-O2',
+ 'lowopt': '-O1',
+ 'noopt': '-O0',
+ 'opt': '-Kfast', # -O3 -Keval,fast_matmul,fp_contract,fp_relaxed,fz,ilfunc,mfunc,omitfp,simd_packed_promotion
'optarch': '', # Fujitsu compiler by default generates code for the arch it is running on
- 'openmp': 'Kopenmp',
- 'unroll': 'funroll-loops',
+ 'openmp': '-Kopenmp',
+ 'unroll': '-funroll-loops',
# apparently the -Kfp_precision flag doesn't work in clang mode, will need to look into these later
# also at strict vs precise and loose vs veryloose
- 'strict': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'],
- 'precise': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'],
+ 'strict': ['-Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['-Kfp_precision'],
+ 'precise': ['-Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['-Kfp_precision'],
'defaultprec': [],
- 'loose': ['Kfp_relaxed'],
- 'veryloose': ['Kfp_relaxed'],
+ 'loose': ['-Kfp_relaxed'],
+ 'veryloose': ['-Kfp_relaxed'],
# apparently the -K[NO]SVE flags don't work in clang mode
# SVE is enabled by default, -Knosimd seems to disable it
- 'vectorize': {False: 'Knosimd', True: ''},
+ 'vectorize': {False: '-Knosimd', True: ''},
}
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
@@ -109,8 +109,8 @@ def _set_compiler_vars(self):
super(FujitsuCompiler, self)._set_compiler_vars()
# enable clang compatibility mode
- self.variables.nappend('CFLAGS', ['Nclang'])
- self.variables.nappend('CXXFLAGS', ['Nclang'])
+ self.variables.nappend('CFLAGS', ['-Nclang'])
+ self.variables.nappend('CXXFLAGS', ['-Nclang'])
# also add fujitsu module library path to LDFLAGS
libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')
diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py
index e9748f3bda..20a969e102 100644
--- a/easybuild/toolchains/compiler/gcc.py
+++ b/easybuild/toolchains/compiler/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -55,26 +55,26 @@ class Gcc(Compiler):
'lto': (False, "Enable Link Time Optimization"),
}
COMPILER_UNIQUE_OPTION_MAP = {
- 'i8': 'fdefault-integer-8',
- 'r8': ['fdefault-real-8', 'fdefault-double-8'],
- 'unroll': 'funroll-loops',
- 'f2c': 'ff2c',
- 'loop': ['ftree-switch-conversion', 'floop-interchange', 'floop-strip-mine', 'floop-block'],
- 'lto': 'flto',
- 'ieee': ['mieee-fp', 'fno-trapping-math'],
- 'strict': ['mieee-fp', 'mno-recip'],
- 'precise': ['mno-recip'],
- 'defaultprec': ['fno-math-errno'],
- 'loose': ['fno-math-errno', 'mrecip', 'mno-ieee-fp'],
- 'veryloose': ['fno-math-errno', 'mrecip=all', 'mno-ieee-fp'],
- 'vectorize': {False: 'fno-tree-vectorize', True: 'ftree-vectorize'},
- DEFAULT_OPT_LEVEL: ['O2', 'ftree-vectorize'],
+ 'i8': '-fdefault-integer-8',
+ 'r8': ['-fdefault-real-8', '-fdefault-double-8'],
+ 'unroll': '-funroll-loops',
+ 'f2c': '-ff2c',
+ 'loop': ['-ftree-switch-conversion', '-floop-interchange', '-floop-strip-mine', '-floop-block'],
+ 'lto': '-flto',
+ 'ieee': ['-mieee-fp', '-fno-trapping-math'],
+ 'strict': ['-mieee-fp', '-mno-recip'],
+ 'precise': ['-mno-recip'],
+ 'defaultprec': ['-fno-math-errno'],
+ 'loose': ['-fno-math-errno', '-mrecip', '-mno-ieee-fp'],
+ 'veryloose': ['-fno-math-errno', '-mrecip=all', '-mno-ieee-fp'],
+ 'vectorize': {False: '-fno-tree-vectorize', True: '-ftree-vectorize'},
+ DEFAULT_OPT_LEVEL: ['-O2', '-ftree-vectorize'],
}
# gcc on aarch64 does not support -mno-recip, -mieee-fp, -mfno-math-errno...
# https://gcc.gnu.org/onlinedocs/gcc/AArch64-Options.html
if systemtools.get_cpu_architecture() == systemtools.AARCH64:
- no_recip_alternative = ['mno-low-precision-recip-sqrt', 'mno-low-precision-sqrt', 'mno-low-precision-div']
+ no_recip_alternative = ['-mno-low-precision-recip-sqrt', '-mno-low-precision-sqrt', '-mno-low-precision-div']
COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative
COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative
@@ -84,38 +84,38 @@ class Gcc(Compiler):
if systemtools.get_cpu_family() == systemtools.RISCV:
COMPILER_UNIQUE_OPTION_MAP['strict'] = []
COMPILER_UNIQUE_OPTION_MAP['precise'] = []
- COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
- COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno']
+ COMPILER_UNIQUE_OPTION_MAP['loose'] = ['-fno-math-errno']
+ COMPILER_UNIQUE_OPTION_MAP['veryloose'] = ['-fno-math-errno']
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
- (systemtools.AARCH32, systemtools.ARM): 'mcpu=native', # implies -march=native and -mtune=native
- (systemtools.AARCH64, systemtools.ARM): 'mcpu=native', # since GCC 6; implies -march=native and -mtune=native
+ (systemtools.AARCH32, systemtools.ARM): '-mcpu=native', # implies -march=native and -mtune=native
+ (systemtools.AARCH64, systemtools.ARM): '-mcpu=native', # since GCC 6; implies -march=native and -mtune=native
# no support for -march on POWER; implies -mtune=native
- (systemtools.POWER, systemtools.POWER): 'mcpu=native',
- (systemtools.POWER, systemtools.POWER_LE): 'mcpu=native',
- (systemtools.X86_64, systemtools.AMD): 'march=native', # implies -mtune=native
- (systemtools.X86_64, systemtools.INTEL): 'march=native', # implies -mtune=native
+ (systemtools.POWER, systemtools.POWER): '-mcpu=native',
+ (systemtools.POWER, systemtools.POWER_LE): '-mcpu=native',
+ (systemtools.X86_64, systemtools.AMD): '-march=native', # implies -mtune=native
+ (systemtools.X86_64, systemtools.INTEL): '-march=native', # implies -mtune=native
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
- (systemtools.AARCH32, systemtools.ARM): 'mcpu=generic-armv7', # implies -march=armv7 and -mtune=generic-armv7
- (systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic
- (systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER
- (systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER
- (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
- (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic',
- (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic',
+ (systemtools.AARCH32, systemtools.ARM): '-mcpu=generic-armv7', # implies -march=armv7 and -mtune=generic-armv7
+ (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic', # implies -march=armv8-a and -mtune=generic
+ (systemtools.POWER, systemtools.POWER): '-mcpu=powerpc64', # no support for -march on POWER
+ (systemtools.POWER, systemtools.POWER_LE): '-mcpu=powerpc64le', # no support for -march on POWER
+ (systemtools.RISCV64, systemtools.RISCV): '-march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
+ (systemtools.X86_64, systemtools.AMD): '-march=x86-64 -mtune=generic',
+ (systemtools.X86_64, systemtools.INTEL): '-march=x86-64 -mtune=generic',
}
COMPILER_CC = 'gcc'
COMPILER_CXX = 'g++'
- COMPILER_C_UNIQUE_FLAGS = []
+ COMPILER_C_UNIQUE_OPTIONS = []
COMPILER_F77 = 'gfortran'
COMPILER_F90 = 'gfortran'
COMPILER_FC = 'gfortran'
- COMPILER_F_UNIQUE_FLAGS = ['f2c']
+ COMPILER_F_UNIQUE_OPTIONS = ['f2c']
LIB_MULTITHREAD = ['pthread']
LIB_MATH = ['m']
@@ -123,9 +123,6 @@ class Gcc(Compiler):
def _set_compiler_vars(self):
super(Gcc, self)._set_compiler_vars()
- if self.options.get('32bit', None):
- raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit GCC in EasyBuild")
-
# to get rid of lots of problems with libgfortranbegin
# or remove the system gcc-gfortran
# also used in eg LIBBLAS variable
@@ -189,8 +186,8 @@ def _guess_aarch64_default_optarch(self):
break
if core_types:
# On big.LITTLE setups, sort core types to have big core (higher model number) first.
- # Example: 'mcpu=cortex-a72.cortex-a53' for "ARM Cortex-A53 + Cortex-A72"
- default_optarch = 'mcpu=%s' % '.'.join([ct[1] for ct in sorted(core_types, reverse=True)])
+ # Example: '-mcpu=cortex-a72.cortex-a53' for "ARM Cortex-A53 + Cortex-A72"
+ default_optarch = '-mcpu=%s' % '.'.join([ct[1] for ct in sorted(core_types, reverse=True)])
self.log.debug("Using architecture-specific compiler optimization flag '%s'", default_optarch)
return default_optarch
diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py
index c33b5ee1a2..637b056a3b 100644
--- a/easybuild/toolchains/compiler/intel_compilers.py
+++ b/easybuild/toolchains/compiler/intel_compilers.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -106,22 +106,22 @@ def set_variables(self):
if oneapi:
# fp-model source is not supported by icx but is equivalent to precise
- self.options.options_map['defaultprec'] = ['fp-speculation=safe', 'fp-model precise']
+ self.options.options_map['defaultprec'] = ['-fp-speculation=safe', '-fp-model precise']
if LooseVersion(comp_ver) >= LooseVersion('2022'):
- self.options.options_map['defaultprec'].insert(0, 'ftz')
+ self.options.options_map['defaultprec'].insert(0, '-ftz')
# icx doesn't like -fp-model fast=1; fp-model fast is equivalent
- self.options.options_map['loose'] = ['fp-model fast']
+ self.options.options_map['loose'] = ['-fp-model fast']
# fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'"
- self.options.options_map['veryloose'] = ['fp-model fast']
+ self.options.options_map['veryloose'] = ['-fp-model fast']
# recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers
# https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html
- self.options.options_map['openmp'] = ['qopenmp']
+ self.options.options_map['openmp'] = ['-qopenmp']
# -xSSE2 is not supported by Intel oneAPI compilers,
# so use -march=x86-64 -mtune=generic when using optarch=GENERIC
self.COMPILER_GENERIC_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic',
- (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic',
+ (systemtools.X86_64, systemtools.AMD): '-march=x86-64 -mtune=generic',
+ (systemtools.X86_64, systemtools.INTEL): '-march=x86-64 -mtune=generic',
}
# skip IntelIccIfort.set_variables (no longer relevant for recent versions)
diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py
index 01015a0d83..434250fce0 100644
--- a/easybuild/toolchains/compiler/inteliccifort.py
+++ b/easybuild/toolchains/compiler/inteliccifort.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -57,40 +57,40 @@ class IntelIccIfort(Compiler):
}
COMPILER_UNIQUE_OPTION_MAP = {
- 'i8': 'i8',
- 'r8': 'r8',
- 'optarch': 'xHost',
- 'ieee': 'fltconsistency',
- 'strict': ['fp-speculation=strict', 'fp-model strict'],
- 'precise': ['fp-model precise'],
- 'defaultprec': ['ftz', 'fp-speculation=safe', 'fp-model source'],
- 'loose': ['fp-model fast=1'],
- 'veryloose': ['fp-model fast=2'],
- 'vectorize': {False: 'no-vec', True: 'vec'},
- 'intel-static': 'static-intel',
- 'no-icc': 'no-icc',
- 'error-unknown-option': 'we10006', # error at warning #10006: ignoring unknown option
+ 'i8': '-i8',
+ 'r8': '-r8',
+ 'optarch': '-xHost',
+ 'ieee': '-fltconsistency',
+ 'strict': ['-fp-speculation=strict', '-fp-model strict'],
+ 'precise': ['-fp-model precise'],
+ 'defaultprec': ['-ftz', '-fp-speculation=safe', '-fp-model source'],
+ 'loose': ['-fp-model fast=1'],
+ 'veryloose': ['-fp-model fast=2'],
+ 'vectorize': {False: '-no-vec', True: '-vec'},
+ 'intel-static': '-static-intel',
+ 'no-icc': '-no-icc',
+ 'error-unknown-option': '-we10006', # error at warning #10006: ignoring unknown option
}
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'xHost',
- (systemtools.X86_64, systemtools.INTEL): 'xHost',
+ (systemtools.X86_64, systemtools.AMD): '-xHost',
+ (systemtools.X86_64, systemtools.INTEL): '-xHost',
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'xSSE2',
- (systemtools.X86_64, systemtools.INTEL): 'xSSE2',
+ (systemtools.X86_64, systemtools.AMD): '-xSSE2',
+ (systemtools.X86_64, systemtools.INTEL): '-xSSE2',
}
COMPILER_CC = 'icc'
COMPILER_CXX = 'icpc'
- COMPILER_C_UNIQUE_FLAGS = ['intel-static', 'no-icc']
+ COMPILER_C_UNIQUE_OPTIONS = ['intel-static', 'no-icc']
COMPILER_F77 = 'ifort'
COMPILER_F90 = 'ifort'
COMPILER_FC = 'ifort'
- COMPILER_F_UNIQUE_FLAGS = ['intel-static']
+ COMPILER_F_UNIQUE_OPTIONS = ['intel-static']
LINKER_TOGGLE_STATIC_DYNAMIC = {
'static': '-Bstatic',
@@ -124,20 +124,17 @@ def _set_compiler_vars(self):
if LooseVersion(icc_version) < LooseVersion('2011'):
self.LIB_MULTITHREAD.insert(1, "guide")
- libpaths = ['intel64']
- if self.options.get('32bit', None):
- libpaths.append('ia32')
- libpaths = ['lib/%s' % x for x in libpaths]
+ libpath = 'lib/intel64'
if LooseVersion(icc_version) > LooseVersion('2011.4') and LooseVersion(icc_version) < LooseVersion('2013_sp1'):
- libpaths = ['compiler/%s' % x for x in libpaths]
+ libpath = 'compiler/%s' % libpath
- self.variables.append_subdirs("LDFLAGS", icc_root, subdirs=libpaths)
+ self.variables.append_subdirs("LDFLAGS", icc_root, subdirs=[libpath])
def set_variables(self):
"""Set the variables."""
# -fopenmp is not supported in old versions (11.x)
icc_version, _ = self.get_software_version(self.COMPILER_MODULE_NAME)[0:2]
if LooseVersion(icc_version) < LooseVersion('12'):
- self.options.options_map['openmp'] = 'openmp'
+ self.options.options_map['openmp'] = '-openmp'
super(IntelIccIfort, self).set_variables()
diff --git a/easybuild/toolchains/compiler/nvhpc.py b/easybuild/toolchains/compiler/nvhpc.py
index 2011137716..e32d70de8b 100644
--- a/easybuild/toolchains/compiler/nvhpc.py
+++ b/easybuild/toolchains/compiler/nvhpc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Bart Oldeman
+# Copyright 2015-2025 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
@@ -60,28 +60,28 @@ class NVHPC(Compiler):
# http://www.pgroup.com/products/freepgi/freepgi_ref/ch02.html#Mfprelaxed
# http://www.pgroup.com/products/freepgi/freepgi_ref/ch02.html#Mfpapprox
COMPILER_UNIQUE_OPTION_MAP = {
- 'i8': 'i8',
- 'r8': 'r8',
+ 'i8': '-i8',
+ 'r8': '-r8',
'optarch': '', # PGI by default generates code for the arch it is running on!
- 'openmp': 'mp',
- 'ieee': 'Kieee',
- 'strict': ['Mnoflushz', 'Kieee'],
- 'precise': ['Mnoflushz'],
- 'defaultprec': ['Mflushz'],
- 'loose': ['Mfprelaxed'],
- 'veryloose': ['Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', 'Mfpapprox'],
- 'vectorize': {False: 'Mnovect', True: 'Mvect'},
+ 'openmp': '-mp',
+ 'ieee': '-Kieee',
+ 'strict': ['-Mnoflushz', '-Kieee'],
+ 'precise': ['-Mnoflushz'],
+ 'defaultprec': ['-Mflushz'],
+ 'loose': ['-Mfprelaxed'],
+ 'veryloose': ['-Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', 'Mfpapprox'],
+ 'vectorize': {False: '-Mnovect', True: '-Mvect'},
}
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'tp=host',
- (systemtools.X86_64, systemtools.INTEL): 'tp=host',
+ (systemtools.X86_64, systemtools.AMD): '-tp=host',
+ (systemtools.X86_64, systemtools.INTEL): '-tp=host',
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'tp=px',
- (systemtools.X86_64, systemtools.INTEL): 'tp=px',
+ (systemtools.X86_64, systemtools.AMD): '-tp=px',
+ (systemtools.X86_64, systemtools.INTEL): '-tp=px',
}
COMPILER_CC = 'nvc'
diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py
index 0f55614926..416004b1e2 100644
--- a/easybuild/toolchains/compiler/pgi.py
+++ b/easybuild/toolchains/compiler/pgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Bart Oldeman
+# Copyright 2015-2025 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
@@ -56,17 +56,17 @@ class Pgi(Compiler):
# http://www.pgroup.com/products/freepgi/freepgi_ref/ch02.html#Mfprelaxed
# http://www.pgroup.com/products/freepgi/freepgi_ref/ch02.html#Mfpapprox
COMPILER_UNIQUE_OPTION_MAP = {
- 'i8': 'i8',
- 'r8': 'r8',
+ 'i8': '-i8',
+ 'r8': '-r8',
'optarch': '', # PGI by default generates code for the arch it is running on!
- 'openmp': 'mp',
- 'ieee': 'Kieee',
- 'strict': ['Mnoflushz', 'Kieee'],
- 'precise': ['Mnoflushz'],
- 'defaultprec': ['Mflushz'],
- 'loose': ['Mfprelaxed'],
- 'veryloose': ['Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', 'Mfpapprox'],
- 'vectorize': {False: 'Mnovect', True: 'Mvect'},
+ 'openmp': '-mp',
+ 'ieee': '-Kieee',
+ 'strict': ['-Mnoflushz', '-Kieee'],
+ 'precise': ['-Mnoflushz'],
+ 'defaultprec': ['-Mflushz'],
+ 'loose': ['-Mfprelaxed'],
+ 'veryloose': ['-Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', '-Mfpapprox'],
+ 'vectorize': {False: '-Mnovect', True: '-Mvect'},
}
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
@@ -76,8 +76,8 @@ class Pgi(Compiler):
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
- (systemtools.X86_64, systemtools.AMD): 'tp=x64',
- (systemtools.X86_64, systemtools.INTEL): 'tp=x64',
+ (systemtools.X86_64, systemtools.AMD): '-tp=x64',
+ (systemtools.X86_64, systemtools.INTEL): '-tp=x64',
}
COMPILER_CC = 'pgcc'
diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py
index d1fd06d4d9..2a7f4164f5 100644
--- a/easybuild/toolchains/compiler/systemcompiler.py
+++ b/easybuild/toolchains/compiler/systemcompiler.py
@@ -1,5 +1,5 @@
##
-# Copyright 2019-2024 Ghent University
+# Copyright 2019-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,3 +40,12 @@ class SystemCompiler(Compiler):
"""System compiler"""
COMPILER_MODULE_NAME = []
COMPILER_FAMILY = TC_CONSTANT_SYSTEM
+
+ # The system compiler does not currently support even the shared options
+ # (changing this would require updating set_minimal_build_env() of the toolchain class)
+ COMPILER_UNIQUE_OPTS = None
+ # only keep the rpath toolchainopt since we want to be able to disable it for
+ # sanity checks in binary-only installations
+ COMPILER_SHARED_OPTS = {k: Compiler.COMPILER_SHARED_OPTS[k] for k in ('rpath',)}
+ COMPILER_UNIQUE_OPTION_MAP = None
+ COMPILER_SHARED_OPTION_MAP = None
diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py
index 13b92e55d2..7c93c10c33 100644
--- a/easybuild/toolchains/craycce.py
+++ b/easybuild/toolchains/craycce.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py
index 8d198362bd..5ea033c113 100644
--- a/easybuild/toolchains/craygnu.py
+++ b/easybuild/toolchains/craygnu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py
index 5997eb63d6..c99aeaa8b0 100644
--- a/easybuild/toolchains/crayintel.py
+++ b/easybuild/toolchains/crayintel.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py
index 49004775a4..c11f474c24 100644
--- a/easybuild/toolchains/craypgi.py
+++ b/easybuild/toolchains/craypgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py
index bc99335c48..c0d4e056ce 100644
--- a/easybuild/toolchains/fcc.py
+++ b/easybuild/toolchains/fcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,41 +40,10 @@ class FCC(FujitsuCompiler):
OPTIONAL = False
# override in order to add an exception for the Fujitsu lang/tcsds module
- def _add_dependency_variables(self, names=None, cpp=None, ld=None):
- """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
- names should be a list of strings containing the name of the dependency
+ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
"""
- cpp_paths = ['include']
- ld_paths = ['lib']
- if not self.options.get('32bit', None):
- ld_paths.insert(0, 'lib64')
-
- if cpp is not None:
- for p in cpp:
- if p not in cpp_paths:
- cpp_paths.append(p)
- if ld is not None:
- for p in ld:
- if p not in ld_paths:
- ld_paths.append(p)
-
- if not names:
- deps = self.dependencies
- else:
- deps = [{'name': name} for name in names if name is not None]
-
- # collect software install prefixes for dependencies
- roots = []
- for dep in deps:
- if dep.get('external_module', False):
- # for software names provided via external modules, install prefix may be unknown
- names = dep['external_module_metadata'].get('name', [])
- roots.extend([root for root in self.get_software_root(names) if root is not None])
- else:
- roots.extend(self.get_software_root(dep['name']))
-
- for root in roots:
- # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
- if 'tcsds' not in root:
- self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
- self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+ Append prepocessor paths for given dependency root directory
+ """
+ # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
+ if "tcsds" not in dep_root:
+ super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs)
diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py
index 71a49f7913..d0a2293b9e 100644
--- a/easybuild/toolchains/ffmpi.py
+++ b/easybuild/toolchains/ffmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py
index afcf0211dd..c566907d41 100644
--- a/easybuild/toolchains/fft/__init__.py
+++ b/easybuild/toolchains/fft/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py
index 9cb4c4e851..cb0d3f2cef 100644
--- a/easybuild/toolchains/fft/fftw.py
+++ b/easybuild/toolchains/fft/fftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py
index 4de1769528..d9e74aae5f 100644
--- a/easybuild/toolchains/fft/fujitsufftw.py
+++ b/easybuild/toolchains/fft/fujitsufftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py
index 7882278a91..5541d2b939 100644
--- a/easybuild/toolchains/fft/intelfftw.py
+++ b/easybuild/toolchains/fft/intelfftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py
index 031daa4c9c..942aa8a2f5 100644
--- a/easybuild/toolchains/foss.py
+++ b/easybuild/toolchains/foss.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py
index 9465515d36..45dbe6cce2 100644
--- a/easybuild/toolchains/fosscuda.py
+++ b/easybuild/toolchains/fosscuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py
index 707255e03c..3a65c6ae50 100644
--- a/easybuild/toolchains/fujitsu.py
+++ b/easybuild/toolchains/fujitsu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py
index 71091e795c..0b6b2b96f9 100644
--- a/easybuild/toolchains/gcc.py
+++ b/easybuild/toolchains/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py
index ce4703e89c..f3ff61dae7 100644
--- a/easybuild/toolchains/gcccore.py
+++ b/easybuild/toolchains/gcccore.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py
index 47ab250824..cc2194c853 100644
--- a/easybuild/toolchains/gcccuda.py
+++ b/easybuild/toolchains/gcccuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py
index d41f5b7d06..72274b641f 100644
--- a/easybuild/toolchains/gfbf.py
+++ b/easybuild/toolchains/gfbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py
index 1d7b9704f6..7e69b3e788 100644
--- a/easybuild/toolchains/gimkl.py
+++ b/easybuild/toolchains/gimkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py
index 76e6bd638a..1f5ed9ba0e 100644
--- a/easybuild/toolchains/gimpi.py
+++ b/easybuild/toolchains/gimpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py
index dde1d6c8f8..3005b97919 100644
--- a/easybuild/toolchains/gimpic.py
+++ b/easybuild/toolchains/gimpic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py
index 121c81121b..7cbaacb130 100644
--- a/easybuild/toolchains/giolf.py
+++ b/easybuild/toolchains/giolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py
index df798626aa..588d4922bb 100644
--- a/easybuild/toolchains/giolfc.py
+++ b/easybuild/toolchains/giolfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py
index ee95c3f7f3..9eda2321f8 100644
--- a/easybuild/toolchains/gmacml.py
+++ b/easybuild/toolchains/gmacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py
index 67e33e1e62..521edab090 100644
--- a/easybuild/toolchains/gmkl.py
+++ b/easybuild/toolchains/gmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py
index bead896d5e..182d7f4220 100644
--- a/easybuild/toolchains/gmklc.py
+++ b/easybuild/toolchains/gmklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py
index 30d10a90f7..cd584ae4e1 100644
--- a/easybuild/toolchains/gmpflf.py
+++ b/easybuild/toolchains/gmpflf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py
index 66d2b29123..1b16206eba 100644
--- a/easybuild/toolchains/gmpich.py
+++ b/easybuild/toolchains/gmpich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py
index f734d3cb56..e54cc84260 100644
--- a/easybuild/toolchains/gmpich2.py
+++ b/easybuild/toolchains/gmpich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py
index a39baf934d..c5a658d888 100644
--- a/easybuild/toolchains/gmpit.py
+++ b/easybuild/toolchains/gmpit.py
@@ -1,5 +1,5 @@
##
-# Copyright 2022-2024 Ghent University
+# Copyright 2022-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py
index c5a8be97d9..e64d1c4e05 100644
--- a/easybuild/toolchains/gmpolf.py
+++ b/easybuild/toolchains/gmpolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py
index 26aa6ad73e..98fef55752 100644
--- a/easybuild/toolchains/gmvapich2.py
+++ b/easybuild/toolchains/gmvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py
index 613d8a6ed3..23f859fd86 100644
--- a/easybuild/toolchains/gmvolf.py
+++ b/easybuild/toolchains/gmvolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py
index 84b06af535..0968a5237d 100644
--- a/easybuild/toolchains/gnu.py
+++ b/easybuild/toolchains/gnu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py
index db1280907b..97bb0b07fd 100644
--- a/easybuild/toolchains/goalf.py
+++ b/easybuild/toolchains/goalf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py
index 9df26ff706..36801ddba7 100644
--- a/easybuild/toolchains/gobff.py
+++ b/easybuild/toolchains/gobff.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py
index 63e7c0c0a4..be7aa4c5b4 100644
--- a/easybuild/toolchains/goblf.py
+++ b/easybuild/toolchains/goblf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py
index 3128c87091..94f2d609d7 100644
--- a/easybuild/toolchains/gofbf.py
+++ b/easybuild/toolchains/gofbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py
index 6d046413a8..a9574c08b5 100644
--- a/easybuild/toolchains/golf.py
+++ b/easybuild/toolchains/golf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py
index 3b99dce5c0..f6e17f6d2a 100644
--- a/easybuild/toolchains/golfc.py
+++ b/easybuild/toolchains/golfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py
index ba93882a1b..419ab5b178 100644
--- a/easybuild/toolchains/gomkl.py
+++ b/easybuild/toolchains/gomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py
index 95620d8360..5507b23a84 100644
--- a/easybuild/toolchains/gomklc.py
+++ b/easybuild/toolchains/gomklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py
index 7af87123c9..3473e117cd 100644
--- a/easybuild/toolchains/gompi.py
+++ b/easybuild/toolchains/gompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py
index 0beb2b4d10..5c62fbb876 100644
--- a/easybuild/toolchains/gompic.py
+++ b/easybuild/toolchains/gompic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py
index 099f7bc11a..ed8fb837f5 100644
--- a/easybuild/toolchains/goolf.py
+++ b/easybuild/toolchains/goolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py
index 76cf4db890..bf5e4f6b24 100644
--- a/easybuild/toolchains/goolfc.py
+++ b/easybuild/toolchains/goolfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py
index 529d17dc5e..9452d5a08f 100644
--- a/easybuild/toolchains/gpsmpi.py
+++ b/easybuild/toolchains/gpsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py
index e87f3a5de7..6666906f12 100644
--- a/easybuild/toolchains/gpsolf.py
+++ b/easybuild/toolchains/gpsolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py
index 7456b4011e..aa56351070 100644
--- a/easybuild/toolchains/gqacml.py
+++ b/easybuild/toolchains/gqacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py
index 68f1a6d0f0..2e89020c43 100644
--- a/easybuild/toolchains/gsmpi.py
+++ b/easybuild/toolchains/gsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py
index cb917ca7d3..c58266d8e0 100644
--- a/easybuild/toolchains/gsolf.py
+++ b/easybuild/toolchains/gsolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py
index 38ec224a5b..80841b9aad 100644
--- a/easybuild/toolchains/iccifort.py
+++ b/easybuild/toolchains/iccifort.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py
index 149cfc8e39..b0370e0b59 100644
--- a/easybuild/toolchains/iccifortcuda.py
+++ b/easybuild/toolchains/iccifortcuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py
index 2055346b37..3d8e574701 100644
--- a/easybuild/toolchains/ictce.py
+++ b/easybuild/toolchains/ictce.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py
index 9f689279cb..ae493a20ed 100644
--- a/easybuild/toolchains/ifbf.py
+++ b/easybuild/toolchains/ifbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py
index 07a6b1e567..292a6ea902 100644
--- a/easybuild/toolchains/iibff.py
+++ b/easybuild/toolchains/iibff.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py
index 605445cc5c..7e55d71dbb 100644
--- a/easybuild/toolchains/iimkl.py
+++ b/easybuild/toolchains/iimkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py
index 89325c2e67..e74005a6be 100644
--- a/easybuild/toolchains/iimklc.py
+++ b/easybuild/toolchains/iimklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py
index 9337365ec6..a8dca0f953 100644
--- a/easybuild/toolchains/iimpi.py
+++ b/easybuild/toolchains/iimpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py
index 45a8df3804..a54524cf38 100644
--- a/easybuild/toolchains/iimpic.py
+++ b/easybuild/toolchains/iimpic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py
index 5ff1466ff3..a1bc6809fb 100644
--- a/easybuild/toolchains/iiqmpi.py
+++ b/easybuild/toolchains/iiqmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py
index 7feda33805..c06815d98b 100644
--- a/easybuild/toolchains/impich.py
+++ b/easybuild/toolchains/impich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py
index 952bff34ce..4963682668 100644
--- a/easybuild/toolchains/impmkl.py
+++ b/easybuild/toolchains/impmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py
index ddfee3aaae..22322e88dd 100644
--- a/easybuild/toolchains/intel-para.py
+++ b/easybuild/toolchains/intel-para.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py
index 659b05faf2..3842933b9a 100644
--- a/easybuild/toolchains/intel.py
+++ b/easybuild/toolchains/intel.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py
index 27fdfbebfd..4ffb3b81ee 100644
--- a/easybuild/toolchains/intel_compilers.py
+++ b/easybuild/toolchains/intel_compilers.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py
index bdd59abaab..fa84a3a759 100644
--- a/easybuild/toolchains/intelcuda.py
+++ b/easybuild/toolchains/intelcuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py
index cd3313d600..2db44455f7 100644
--- a/easybuild/toolchains/iofbf.py
+++ b/easybuild/toolchains/iofbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py
index 033277919c..9f556b6970 100644
--- a/easybuild/toolchains/iomkl.py
+++ b/easybuild/toolchains/iomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py
index f40cd78e16..afd182955a 100644
--- a/easybuild/toolchains/iomklc.py
+++ b/easybuild/toolchains/iomklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py
index b578c45892..2ec287cb77 100644
--- a/easybuild/toolchains/iompi.py
+++ b/easybuild/toolchains/iompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py
index 5b675f5621..8220be6f7c 100644
--- a/easybuild/toolchains/iompic.py
+++ b/easybuild/toolchains/iompic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py
index a8a772adcb..2960b5ed22 100644
--- a/easybuild/toolchains/ipsmpi.py
+++ b/easybuild/toolchains/ipsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py
index e4c5476cd4..82500946c3 100644
--- a/easybuild/toolchains/iqacml.py
+++ b/easybuild/toolchains/iqacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py
index 4ccdc87054..527af51051 100644
--- a/easybuild/toolchains/ismkl.py
+++ b/easybuild/toolchains/ismkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py
index 90cdb068a8..364c4eca88 100644
--- a/easybuild/toolchains/linalg/__init__.py
+++ b/easybuild/toolchains/linalg/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py
index 268e3b0cc2..2a30636243 100644
--- a/easybuild/toolchains/linalg/acml.py
+++ b/easybuild/toolchains/linalg/acml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -73,15 +73,13 @@ def __init__(self, *args, **kwargs):
def _set_blas_variables(self):
"""Fix the map a bit"""
- if self.options.get('32bit', None):
- raise EasyBuildError("_set_blas_variables: 32bit ACML not (yet) supported")
try:
for root in self.get_software_root(self.BLAS_MODULE_NAME):
subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY]
self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs]
- self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True)
+ self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR)
incdirs = [os.path.join(x, 'include') for x in subdirs]
- self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True)
+ self._add_dependency_cpp_headers(root, extra_dirs=incdirs)
except Exception:
raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP "
"with compiler family %s", self.COMPILER_FAMILY)
diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py
index bc894bb108..5adbcda66c 100644
--- a/easybuild/toolchains/linalg/atlas.py
+++ b/easybuild/toolchains/linalg/atlas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py
index 201681c51f..db18ec85d3 100644
--- a/easybuild/toolchains/linalg/blacs.py
+++ b/easybuild/toolchains/linalg/blacs.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py
index d89f3b8d83..f4ba16574f 100644
--- a/easybuild/toolchains/linalg/blis.py
+++ b/easybuild/toolchains/linalg/blis.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py
index 5f6f5cfe87..99f963f84c 100644
--- a/easybuild/toolchains/linalg/flame.py
+++ b/easybuild/toolchains/linalg/flame.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py
index 0e88c1a9b2..8b874cc767 100644
--- a/easybuild/toolchains/linalg/flexiblas.py
+++ b/easybuild/toolchains/linalg/flexiblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,7 +34,7 @@
from easybuild.tools.toolchain.linalg import LinAlg
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
@@ -48,11 +48,11 @@ def det_flexiblas_backend_libs():
# System-wide (config directory):
# OPENBLAS
# library = libflexiblas_openblas.so
- out, _ = run_cmd("flexiblas list", simple=False, trace=False)
+ res = run_shell_cmd("flexiblas list", hidden=True)
shlib_ext = get_shared_lib_ext()
flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M)
- flexiblas_libs = flexiblas_lib_regex.findall(out)
+ flexiblas_libs = flexiblas_lib_regex.findall(res.output)
backend_libs = []
for flexiblas_lib in flexiblas_libs:
diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py
index dde792455b..9e32da3a2c 100644
--- a/easybuild/toolchains/linalg/fujitsussl.py
+++ b/easybuild/toolchains/linalg/fujitsussl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py
index e8a99a651a..253d32d4cc 100644
--- a/easybuild/toolchains/linalg/gotoblas.py
+++ b/easybuild/toolchains/linalg/gotoblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py
index 9dc89da827..19f81963b9 100644
--- a/easybuild/toolchains/linalg/intelmkl.py
+++ b/easybuild/toolchains/linalg/intelmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -132,9 +132,6 @@ def _set_blas_variables(self):
raise EasyBuildError("_set_blas_variables: interface_mt unsupported combination with compiler family %s",
self.COMPILER_FAMILY)
- if self.options.get('32bit', None):
- # 32bit
- self.BLAS_LIB_MAP.update({"lp64": ''})
if self.options.get('i8', None):
# ilp64/i8
self.BLAS_LIB_MAP.update({"lp64": '_ilp64'})
@@ -146,28 +143,21 @@ def _set_blas_variables(self):
found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0]
ver = LooseVersion(found_version)
if ver < LooseVersion('10.3'):
- if self.options.get('32bit', None):
- self.BLAS_LIB_DIR = ['lib/32']
- else:
- self.BLAS_LIB_DIR = ['lib/em64t']
+ self.BLAS_LIB_DIR = ['lib/em64t']
self.BLAS_INCLUDE_DIR = ['include']
else:
- if self.options.get('32bit', None):
- raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)",
- found_version)
+ if ver >= LooseVersion('2021'):
+ if os.path.islink(os.path.join(root, 'mkl', 'latest')):
+ found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
+ basedir = os.path.join('mkl', found_version)
else:
- if ver >= LooseVersion('2021'):
- if os.path.islink(os.path.join(root, 'mkl', 'latest')):
- found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
- basedir = os.path.join('mkl', found_version)
- else:
- basedir = 'mkl'
-
- self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')]
- if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
- self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64'))
- elif ver < LooseVersion('2021'):
- self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64'))
+ basedir = 'mkl'
+
+ self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')]
+ if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
+ self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64'))
+ elif ver < LooseVersion('2021'):
+ self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64'))
self.BLAS_INCLUDE_DIR = [os.path.join(basedir, 'include')]
@@ -201,11 +191,7 @@ def _set_scalapack_variables(self):
self.SCALAPACK_LIB.append("mkl_solver%(lp64)s_sequential")
self.SCALAPACK_LIB_MT.append("mkl_solver%(lp64)s")
- if self.options.get('32bit', None):
- # 32 bit
- self.SCALAPACK_LIB_MAP.update({"lp64_sc": '_core'})
-
- elif self.options.get('i8', None):
+ if self.options.get('i8', None):
# ilp64/i8
self.SCALAPACK_LIB_MAP.update({"lp64_sc": '_ilp64'})
diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py
index e26c9e2961..9f173e889f 100644
--- a/easybuild/toolchains/linalg/lapack.py
+++ b/easybuild/toolchains/linalg/lapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py
index 6d7e93c16d..773952497c 100644
--- a/easybuild/toolchains/linalg/libsci.py
+++ b/easybuild/toolchains/linalg/libsci.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py
index 18118d1f01..607235181d 100644
--- a/easybuild/toolchains/linalg/openblas.py
+++ b/easybuild/toolchains/linalg/openblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py
index 980e3bfdab..b4c12e6963 100644
--- a/easybuild/toolchains/linalg/scalapack.py
+++ b/easybuild/toolchains/linalg/scalapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py
index c5abd0f595..1445f255b7 100644
--- a/easybuild/toolchains/mpi/__init__.py
+++ b/easybuild/toolchains/mpi/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py
index 8eab736d82..92cca9d4dc 100644
--- a/easybuild/toolchains/mpi/craympich.py
+++ b/easybuild/toolchains/mpi/craympich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py
index ad65c463cf..65c411f0d1 100644
--- a/easybuild/toolchains/mpi/fujitsumpi.py
+++ b/easybuild/toolchains/mpi/fujitsumpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py
index 8bc67fe788..74dfb7fb57 100644
--- a/easybuild/toolchains/mpi/intelmpi.py
+++ b/easybuild/toolchains/mpi/intelmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -97,6 +97,6 @@ def set_variables(self):
super(IntelMPI, self).set_variables()
# add -mt_mpi flag to ensure linking against thread-safe MPI library when OpenMP is enabled
if self.options.get('openmp', None) and self.options.get('usempi', None):
- mt_mpi_option = ['mt_mpi']
+ mt_mpi_option = ['-mt_mpi']
for flags_var, _ in COMPILER_FLAGS:
self.variables.nappend(flags_var, mt_mpi_option)
diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py
index 7a067a0d37..6584a88d5d 100644
--- a/easybuild/toolchains/mpi/mpich.py
+++ b/easybuild/toolchains/mpi/mpich.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py
index b6f1f6f082..24653a662d 100644
--- a/easybuild/toolchains/mpi/mpich2.py
+++ b/easybuild/toolchains/mpi/mpich2.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py
index 43202b59d3..f3001a8897 100644
--- a/easybuild/toolchains/mpi/mpitrampoline.py
+++ b/easybuild/toolchains/mpi/mpitrampoline.py
@@ -1,5 +1,5 @@
##
-# Copyright 2022-2024 Ghent University
+# Copyright 2022-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py
index 4f69c970f5..d75616753f 100644
--- a/easybuild/toolchains/mpi/mvapich2.py
+++ b/easybuild/toolchains/mpi/mvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py
index 91d10923e7..44e9b3d380 100644
--- a/easybuild/toolchains/mpi/openmpi.py
+++ b/easybuild/toolchains/mpi/openmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py
index 07e8c9af78..03daf3db82 100644
--- a/easybuild/toolchains/mpi/psmpi.py
+++ b/easybuild/toolchains/mpi/psmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py
index b7355cd3e1..7af5ed98b9 100644
--- a/easybuild/toolchains/mpi/qlogicmpi.py
+++ b/easybuild/toolchains/mpi/qlogicmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py
index 1fee1d51b7..8907031955 100644
--- a/easybuild/toolchains/mpi/spectrummpi.py
+++ b/easybuild/toolchains/mpi/spectrummpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvhpc.py b/easybuild/toolchains/nvhpc.py
index b4b871f84f..9a0db2bbd7 100644
--- a/easybuild/toolchains/nvhpc.py
+++ b/easybuild/toolchains/nvhpc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Bart Oldeman
+# Copyright 2015-2025 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/nvofbf.py b/easybuild/toolchains/nvofbf.py
index 62d3cb4d37..06bc8fda64 100644
--- a/easybuild/toolchains/nvofbf.py
+++ b/easybuild/toolchains/nvofbf.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py
index 3feedaae35..578d3f9262 100644
--- a/easybuild/toolchains/nvompi.py
+++ b/easybuild/toolchains/nvompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py
index b9c5abc298..648cefc874 100644
--- a/easybuild/toolchains/nvompic.py
+++ b/easybuild/toolchains/nvompic.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2024 Ghent University
-# Copyright 2016-2024 Forschungszentrum Juelich
+# Copyright 2016-2025 Ghent University
+# Copyright 2016-2025 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py
index cbaee42bc0..8eb03cca89 100644
--- a/easybuild/toolchains/nvpsmpi.py
+++ b/easybuild/toolchains/nvpsmpi.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2024 Ghent University
-# Copyright 2016-2024 Forschungszentrum Juelich
+# Copyright 2016-2025 Ghent University
+# Copyright 2016-2025 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py
index 0b916c30a0..90a7d63257 100644
--- a/easybuild/toolchains/nvpsmpic.py
+++ b/easybuild/toolchains/nvpsmpic.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2024 Ghent University
-# Copyright 2016-2024 Forschungszentrum Juelich
+# Copyright 2016-2025 Ghent University
+# Copyright 2016-2025 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pgi.py b/easybuild/toolchains/pgi.py
index df2258bda4..1dc6153992 100644
--- a/easybuild/toolchains/pgi.py
+++ b/easybuild/toolchains/pgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Bart Oldeman
+# Copyright 2015-2025 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py
index 8b22d1d35f..58624761d0 100644
--- a/easybuild/toolchains/pmkl.py
+++ b/easybuild/toolchains/pmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py
index f28f445296..cac22b48d5 100644
--- a/easybuild/toolchains/pomkl.py
+++ b/easybuild/toolchains/pomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py
index 9de2e52050..e6dc33ecba 100644
--- a/easybuild/toolchains/pompi.py
+++ b/easybuild/toolchains/pompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py
index 8a8b17cfaa..7d9d25e6b1 100644
--- a/easybuild/toolchains/system.py
+++ b/easybuild/toolchains/system.py
@@ -1,5 +1,5 @@
##
-# Copyright 2019-2024 Ghent University
+# Copyright 2019-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py
index 8d34fc25c5..60a2671221 100644
--- a/easybuild/tools/__init__.py
+++ b/easybuild/tools/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,14 +37,4 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
-import distutils.version
-import warnings
from easybuild.tools.loose_version import LooseVersion # noqa(F401)
-
-
-class StrictVersion(distutils.version.StrictVersion):
- """Temporary wrapper over distuitls StrictVersion that silences the deprecation warning"""
- def __init__(self, *args, **kwargs):
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", category=DeprecationWarning)
- distutils.version.StrictVersion.__init__(self, *args, **kwargs)
diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py
index 0a3c133fc4..2d9f64f22c 100644
--- a/easybuild/tools/asyncprocess.py
+++ b/easybuild/tools/asyncprocess.py
@@ -1,6 +1,6 @@
##
# Copyright 2005 Josiah Carlson
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson.
# and released under the GPL v2 on March 14, 2012
diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py
index f5ce1ebc01..d2a0127389 100644
--- a/easybuild/tools/build_details.py
+++ b/easybuild/tools/build_details.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
index 0bbff702a1..044f53faac 100644
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,12 +40,12 @@
import tempfile
from copy import copy
from datetime import datetime
+from enum import IntEnum
from easybuild.base import fancylogger
from easybuild.base.exceptions import LoggedException
from easybuild.tools.version import VERSION, this_is_easybuild
-
# EasyBuild message prefix
EB_MSG_PREFIX = "=="
@@ -55,17 +55,71 @@
# allow some experimental experimental code
EXPERIMENTAL = False
-DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html'
+DEPRECATED_DOC_URL = 'https://docs.easybuild.io/deprecated-functionality/'
DRY_RUN_BUILD_DIR = None
DRY_RUN_SOFTWARE_INSTALL_DIR = None
DRY_RUN_MODULES_INSTALL_DIR = None
+CWD_NOTFOUND_ERROR = (
+ "Current working directory does not exist! It was either unexpectedly removed "
+ "by an external process to EasyBuild or the filesystem is misbehaving."
+)
+
DEVEL_LOG_LEVEL = logging.DEBUG - 1
logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL')
+class EasyBuildExit(IntEnum):
+ """
+ Table of exit codes
+ """
+ SUCCESS = 0
+ ERROR = 1
+ # core errors
+ OPTION_ERROR = 2
+ VALUE_ERROR = 3
+ MISSING_EASYCONFIG = 4
+ EASYCONFIG_ERROR = 5
+ MISSING_EASYBLOCK = 6
+ EASYBLOCK_ERROR = 7
+ MODULE_ERROR = 8
+ # step errors in order of execution
+ FAIL_FETCH_STEP = 10
+ FAIL_READY_STEP = 11
+ FAIL_SOURCE_STEP = 12
+ FAIL_PATCH_STEP = 13
+ FAIL_PREPARE_STEP = 14
+ FAIL_CONFIGURE_STEP = 15
+ FAIL_BUILD_STEP = 16
+ FAIL_TEST_STEP = 17
+ FAIL_INSTALL_STEP = 18
+ FAIL_EXTENSIONS_STEP = 19
+ FAIL_POST_ITER_STEP = 20
+ FAIL_POST_PROC_STEP = 21
+ FAIL_SANITY_CHECK_STEP = 22
+ FAIL_CLEANUP_STEP = 23
+ FAIL_MODULE_STEP = 24
+ FAIL_PERMISSIONS_STEP = 25
+ FAIL_PACKAGE_STEP = 26
+ FAIL_TEST_CASES_STEP = 27
+ # errors on missing things
+ MISSING_SOURCES = 30
+ MISSING_DEPENDENCY = 31
+ MISSING_SYSTEM_DEPENDENCY = 32
+ MISSING_EB_DEPENDENCY = 33
+ # errors on specific task failures
+ FAIL_SYSTEM_CHECK = 40
+ FAIL_DOWNLOAD = 41
+ FAIL_CHECKSUM = 42
+ FAIL_EXTRACT = 43
+ FAIL_PATCH_APPLY = 44
+ FAIL_SANITY_CHECK = 45
+ FAIL_MODULE_WRITE = 46
+ FAIL_GITHUB = 47
+
+
class EasyBuildError(LoggedException):
"""
EasyBuildError is thrown when EasyBuild runs into something horribly wrong.
@@ -75,12 +129,13 @@ class EasyBuildError(LoggedException):
# always include location where error was raised from, even under 'python -O'
INCLUDE_LOCATION = True
- def __init__(self, msg, *args):
+ def __init__(self, msg, *args, exit_code=EasyBuildExit.ERROR, **kwargs):
"""Constructor: initialise EasyBuildError instance."""
if args:
msg = msg % args
- LoggedException.__init__(self, msg)
+ LoggedException.__init__(self, msg, exit_code=exit_code, **kwargs)
self.msg = msg
+ self.exit_code = exit_code
def __str__(self):
"""Return string representation of this EasyBuildError instance."""
@@ -167,7 +222,7 @@ def nosupport(self, msg, ver):
def error(self, msg, *args, **kwargs):
"""Print error message and raise an EasyBuildError."""
- ebmsg = "EasyBuild crashed with an error %s: " % self.caller_info()
+ ebmsg = "EasyBuild encountered an error %s: " % self.caller_info()
fancylogger.FancyLogger.error(self, ebmsg + msg, *args, **kwargs)
def devel(self, msg, *args, **kwargs):
@@ -335,8 +390,18 @@ def print_error(msg, *args, **kwargs):
if args:
msg = msg % args
+ # grab exit code, if specified;
+ # also consider deprecated 'exitCode' option
+ exitCode = kwargs.pop('exitCode', None)
+ exit_code = kwargs.pop('exit_code', exitCode)
+ if exitCode is not None:
+ _init_easybuildlog.deprecated("'exitCode' option in print_error function is replaced with 'exit_code'", '6.0')
+
+ # use 1 as defaut exit code
+ if exit_code is None:
+ exit_code = 1
+
log = kwargs.pop('log', None)
- exitCode = kwargs.pop('exitCode', 1)
opt_parser = kwargs.pop('opt_parser', None)
exit_on_error = kwargs.pop('exit_on_error', True)
silent = kwargs.pop('silent', False)
@@ -348,7 +413,7 @@ def print_error(msg, *args, **kwargs):
if opt_parser:
opt_parser.print_shorthelp()
sys.stderr.write("ERROR: %s\n" % msg)
- sys.exit(exitCode)
+ sys.exit(exit_code)
elif log is not None:
raise EasyBuildError(msg)
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 2dd71388cd..664080d8f2 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,11 +45,12 @@
import tempfile
import time
from abc import ABCMeta
+from string import ascii_letters
from easybuild.base import fancylogger
from easybuild.base.frozendict import FrozenDictKnownKeys
-from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type
+from easybuild.base.wrapper import create_base_metaclass
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit
try:
import rich # noqa
@@ -95,10 +96,11 @@
DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env'
DEFAULT_ENVVAR_USERS_MODULES = 'HOME'
DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds)
-DEFAULT_JOB_BACKEND = 'GC3Pie'
+DEFAULT_JOB_BACKEND = 'Slurm'
DEFAULT_JOB_EB_CMD = 'eb'
DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
+DEFAULT_MAX_PARALLEL = 16
DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++'
DEFAULT_MNS = 'EasyBuildMNS'
DEFAULT_MODULE_SYNTAX = 'Lua'
@@ -167,13 +169,29 @@
LOCAL_VAR_NAMING_CHECK_WARN = WARN
LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN]
-
OUTPUT_STYLE_AUTO = 'auto'
OUTPUT_STYLE_BASIC = 'basic'
OUTPUT_STYLE_NO_COLOR = 'no_color'
OUTPUT_STYLE_RICH = 'rich'
OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
+SEARCH_PATH_BIN_DIRS = ['bin']
+SEARCH_PATH_HEADER_DIRS = ['include']
+SEARCH_PATH_LIB_DIRS = ['lib', 'lib64']
+
+PYTHONPATH = 'PYTHONPATH'
+EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
+PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
+
+# options to handle header search paths in environment of modules
+MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath'
+MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths'
+MOD_SEARCH_PATH_HEADERS = {
+ MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'],
+ MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'],
+}
+DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH
+
class Singleton(ABCMeta):
"""Serves as metaclass for classes that should implement the Singleton pattern.
@@ -260,6 +278,8 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'rpath_filter',
'rpath_override_dirs',
'required_linked_shared_libs',
+ 'search_path_cpp_headers',
+ 'search_path_linker',
'skip',
'software_commit',
'stop',
@@ -268,13 +288,12 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'test_report_env_filter',
'testoutput',
'umask',
- 'wait_on_lock',
'zip_logs',
],
False: [
- 'add_dummy_to_minimal_toolchains',
'add_system_to_minimal_toolchains',
'allow_modules_tool_mismatch',
+ 'allow_unresolved_templates',
'backup_patched_files',
'consider_archived_easyconfigs',
'container_build_image',
@@ -285,6 +304,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'enforce_checksums',
'experimental',
'extended_dry_run',
+ 'fail_on_mod_files_gcccore',
'force',
'generate_devel_module',
'group_writable_installdir',
@@ -294,9 +314,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'ignore_locks',
'ignore_test_failure',
'install_latest_eb_release',
+ 'keep_debug_symbols',
'logtostdout',
'minimal_toolchains',
- 'module_extensions',
'module_only',
'package',
'parallel_extensions_install',
@@ -315,7 +335,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'skip_test_step',
'sticky_bit',
'terse',
- 'trace',
'unit_testing_mode',
'upload_test_report',
'update_modules_tool_cache',
@@ -334,10 +353,13 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'lib64_fallback_sanity_check',
'lib64_lib_symlink',
'map_toolchains',
+ 'module_extensions',
'modules_tool_version_check',
'mpi_tests',
'pre_create_installdir',
'show_progress_bar',
+ 'strict_rpath_sanity_check',
+ 'trace',
],
EMPTY_LIST: [
'accept_eula_for',
@@ -374,9 +396,15 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
DEFAULT_MAX_FAIL_RATIO_PERMS: [
'max_fail_ratio_adjust_permissions',
],
+ DEFAULT_MAX_PARALLEL: [
+ 'max_parallel',
+ ],
DEFAULT_MINIMAL_BUILD_ENV: [
'minimal_build_env',
],
+ DEFAULT_MOD_SEARCH_PATH_HEADERS: [
+ 'module_search_path_headers',
+ ],
DEFAULT_PKG_RELEASE: [
'package_release',
],
@@ -407,6 +435,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
OUTPUT_STYLE_AUTO: [
'output_style',
],
+ PYTHONPATH: [
+ 'prefer_python_search_path',
+ ]
}
# build option that do not have a perfectly matching command line option
BUILD_OPTIONS_OTHER = {
@@ -415,13 +446,13 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'command_line',
'external_modules_metadata',
'extra_ec_paths',
+ 'mod_depends_on', # deprecated
'robot_path',
'valid_module_classes',
'valid_stops',
],
False: [
'dry_run',
- 'mod_depends_on',
'recursive_mod_unload',
'retain_all_deps',
'silent',
@@ -478,6 +509,8 @@ class ConfigurationVariables(BaseConfigurationVariables):
'buildpath',
'config',
'containerpath',
+ 'failed_install_build_dirs_path',
+ 'failed_install_logs_path',
'installpath',
'installpath_modules',
'installpath_software',
@@ -506,7 +539,10 @@ def get_items_check_required(self):
"""
missing = [x for x in self.KNOWN_KEYS if x not in self]
if len(missing) > 0:
- raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing)
+ raise EasyBuildError(
+ "Cannot determine value for configuration variables %s. Please specify it.", ', '.join(missing),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
return self.items()
@@ -535,11 +571,14 @@ def init(options, config_options_dict):
# make sure source path is a list
sourcepath = tmpdict['sourcepath']
- if isinstance(sourcepath, string_type):
+ if isinstance(sourcepath, str):
tmpdict['sourcepath'] = sourcepath.split(':')
_log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath']))
elif not isinstance(sourcepath, (tuple, list)):
- raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath)
+ raise EasyBuildError(
+ "Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance
variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True)
@@ -580,10 +619,6 @@ def init_build_options(build_options=None, cmdline_options=None):
_log.info("Auto-enabling ignoring of OS dependencies")
cmdline_options.ignore_osdeps = True
- if not cmdline_options.accept_eula_for and cmdline_options.accept_eula:
- _log.deprecated("Use accept-eula-for configuration setting rather than accept-eula.", '5.0')
- cmdline_options.accept_eula_for = cmdline_options.accept_eula
-
cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks]
active_build_options.update({key: getattr(cmdline_options, key) for key in cmdline_build_option_names})
# other options which can be derived but have no perfectly matching cmdline option
@@ -603,12 +638,12 @@ def init_build_options(build_options=None, cmdline_options=None):
# seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys
bo = {}
for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]:
- for default in build_options_by_default:
+ for default, options in build_options_by_default.items():
if default == EMPTY_LIST:
- for opt in build_options_by_default[default]:
+ for opt in options:
bo[opt] = []
else:
- bo.update({opt: default for opt in build_options_by_default[default]})
+ bo.update({opt: default for opt in options})
bo.update(active_build_options)
# BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance
@@ -621,16 +656,13 @@ def build_option(key, **kwargs):
build_options = BuildOptions()
if key in build_options:
return build_options[key]
- elif key == 'accept_eula':
- _log.deprecated("Use accept_eula_for build option rather than accept_eula.", '5.0')
- return build_options['accept_eula_for']
elif 'default' in kwargs:
return kwargs['default']
else:
error_msg = "Undefined build option: '%s'. " % key
error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() "
error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI."
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
def update_build_option(key, value):
@@ -695,7 +727,10 @@ def install_path(typ=None):
known_types = ['modules', 'software']
if typ not in known_types:
- raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types))
+ raise EasyBuildError(
+ "Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
variables = ConfigurationVariables()
@@ -748,7 +783,7 @@ def container_path():
def get_modules_tool():
"""
- Return modules tool (EnvironmentModulesC, Lmod, ...)
+ Return modules tool (EnvironmentModules, Lmod, ...)
"""
# 'modules_tool' key will only be present if EasyBuild config is initialized
return ConfigurationVariables().get('modules_tool', None)
@@ -787,7 +822,10 @@ def get_output_style():
output_style = OUTPUT_STYLE_BASIC
if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH:
- raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH)
+ raise EasyBuildError(
+ "Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH,
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
return output_style
@@ -812,8 +850,10 @@ def log_file_format(return_directory=False, ec=None, date=None, timestamp=None):
logfile_format = ConfigurationVariables()['logfile_format']
if not isinstance(logfile_format, tuple) or len(logfile_format) != 2:
- raise EasyBuildError("Incorrect log file format specification, should be 2-tuple (, ): %s",
- logfile_format)
+ raise EasyBuildError(
+ "Incorrect log file format specification, should be 2-tuple (, ): %s", logfile_format,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
idx = int(not return_directory)
res = ConfigurationVariables()['logfile_format'][idx] % {
@@ -843,6 +883,42 @@ def log_path(ec=None):
return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp)
+def get_failed_install_build_dirs_path(ec):
+ """
+ Return the location where the build directory is copied to if installation failed
+
+ :param ec: dict-like value with 'name' and 'version' keys defined
+ """
+ base_path = ConfigurationVariables()['failed_install_build_dirs_path']
+ if not base_path:
+ return None
+
+ try:
+ name, version = ec['name'], ec['version']
+ except KeyError:
+ raise EasyBuildError("The 'name' and 'version' keys are required.")
+
+ return os.path.join(base_path, f'{name}-{version}')
+
+
+def get_failed_install_logs_path(ec):
+ """
+ Return the location where log files are copied to if installation failed
+
+ :param ec: dict-like value with 'name' and 'version' keys defined
+ """
+ base_path = ConfigurationVariables()['failed_install_logs_path']
+ if not base_path:
+ return None
+
+ try:
+ name, version = ec['name'], ec['version']
+ except KeyError:
+ raise EasyBuildError("The 'name' and 'version' keys are required.")
+
+ return os.path.join(base_path, f'{name}-{version}')
+
+
def get_build_log_path():
"""
Return (temporary) directory for build log
@@ -920,7 +996,10 @@ def find_last_log(curlog):
sorted_paths = [p for (_, p) in sorted(paths)]
except OSError as err:
- raise EasyBuildError("Failed to locate/select/order log files matching '%s': %s", glob_pattern, err)
+ raise EasyBuildError(
+ "Failed to locate/select/order log files matching '%s': %s", glob_pattern, err,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
try:
# log of current session is typically listed last, should be taken into account
diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py
index 6ac667b5bd..d2ff5d150e 100644
--- a/easybuild/tools/configobj.py
+++ b/easybuild/tools/configobj.py
@@ -27,8 +27,6 @@
from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
-from easybuild.tools.py2vs3 import string_type
-
# imported lazily to avoid startup performance hit if it isn't used
compiler = None
@@ -562,11 +560,11 @@ def __getitem__(self, key):
"""Fetch the item and do string interpolation."""
val = dict.__getitem__(self, key)
if self.main.interpolation:
- if isinstance(val, string_type):
+ if isinstance(val, str):
return self._interpolate(key, val)
if isinstance(val, list):
def _check(entry):
- if isinstance(entry, string_type):
+ if isinstance(entry, str):
return self._interpolate(key, entry)
return entry
new = [_check(entry) for entry in val]
@@ -588,7 +586,7 @@ def __setitem__(self, key, value, unrepr=False):
``unrepr`` must be set when setting a value to a dictionary, without
creating a new sub-section.
"""
- if not isinstance(key, (bytes, string_type)):
+ if not isinstance(key, (bytes, str)):
raise ValueError('The key "%s" is not a string.' % key)
# add the comment
@@ -622,11 +620,11 @@ def __setitem__(self, key, value, unrepr=False):
if key not in self:
self.scalars.append(key)
if not self.main.stringify:
- if isinstance(value, string_type):
+ if isinstance(value, str):
pass
elif isinstance(value, (list, tuple)):
for entry in value:
- if not isinstance(entry, string_type):
+ if not isinstance(entry, str):
raise TypeError('Value is not a string "%s".' % entry)
else:
raise TypeError('Value is not a string "%s".' % value)
@@ -878,31 +876,24 @@ def walk(self, function, raise_errors=True,
"""
out = {}
# scalars first
- for i in range(len(self.scalars)):
- entry = self.scalars[i]
+ for i, entry in enumerate(self.scalars):
try:
val = function(self, entry, **keywargs)
- # bound again in case name has changed
- entry = self.scalars[i]
- out[entry] = val
except Exception:
if raise_errors:
raise
- else:
- entry = self.scalars[i]
- out[entry] = False
+ val = False
+ # bound again in case name has changed
+ entry = self.scalars[i]
+ out[entry] = val
# then sections
- for i in range(len(self.sections)):
- entry = self.sections[i]
+ for i, entry in enumerate(self.sections):
if call_on_sections:
try:
function(self, entry, **keywargs)
except Exception:
if raise_errors:
raise
- else:
- entry = self.sections[i]
- out[entry] = False
# bound again in case name has changed
entry = self.sections[i]
# previous result is discarded
@@ -946,7 +937,7 @@ def as_bool(self, key):
return val
else:
try:
- if not isinstance(val, string_type):
+ if not isinstance(val, str):
# TODO: Why do we raise a KeyError here?
raise KeyError()
else:
@@ -1210,7 +1201,7 @@ def __init__(self, infile=None, options=None, configspec=None, encoding=None,
self._load(infile, configspec)
def _load(self, infile, configspec):
- if isinstance(infile, string_type):
+ if isinstance(infile, str):
self.filename = infile
if os.path.isfile(infile):
with open(infile, 'r') as fh:
@@ -1431,7 +1422,7 @@ def _handle_bom(self, infile):
else:
infile = newline
# UTF8 - don't decode
- if isinstance(infile, string_type):
+ if isinstance(infile, str):
return infile.splitlines(True)
else:
return infile
@@ -1439,7 +1430,7 @@ def _handle_bom(self, infile):
return self._decode(infile, encoding)
# No BOM discovered and no encoding specified, just return
- if isinstance(infile, (bytes, string_type)):
+ if isinstance(infile, (bytes, str)):
# infile read from a file will be a single string
return infile.splitlines(True)
return infile
@@ -1457,7 +1448,7 @@ def _decode(self, infile, encoding):
if is a string, it also needs converting to a list.
"""
- if isinstance(infile, string_type):
+ if isinstance(infile, str):
# can't be unicode
# NOTE: Could raise a ``UnicodeDecodeError``
return infile.decode(encoding).splitlines(True)
@@ -1482,7 +1473,7 @@ def _str(self, value):
Used by ``stringify`` within validate, to turn non-string values
into strings.
"""
- if not isinstance(value, string_type):
+ if not isinstance(value, str):
return str(value)
else:
return value
@@ -1730,7 +1721,7 @@ def _quote(self, value, multiline=True):
return self._quote(value[0], multiline=False) + ','
return ', '.join([self._quote(val, multiline=False)
for val in value])
- if not isinstance(value, string_type):
+ if not isinstance(value, str):
if self.stringify:
value = str(value)
else:
@@ -2275,7 +2266,7 @@ def reload(self):
This method raises a ``ReloadError`` if the ConfigObj doesn't have
a filename attribute pointing to a file.
"""
- if not isinstance(self.filename, string_type):
+ if not isinstance(self.filename, str):
raise ReloadError()
filename = self.filename
diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py
index 215ba61c0c..2a310c4acc 100644
--- a/easybuild/tools/containers/__init__.py
+++ b/easybuild/tools/containers/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py
index 4cb252dac4..4bbdaff914 100644
--- a/easybuild/tools/containers/apptainer.py
+++ b/easybuild/tools/containers/apptainer.py
@@ -1,4 +1,4 @@
-# Copyright 2022-2024 Ghent University
+# Copyright 2022-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -29,13 +29,13 @@
import os
import re
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.containers.singularity import SingularityContainer
from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
from easybuild.tools.config import build_option, container_path
from easybuild.tools.filetools import remove_file, which
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
class ApptainerContainer(SingularityContainer):
@@ -48,15 +48,15 @@ class ApptainerContainer(SingularityContainer):
def apptainer_version():
"""Get Apptainer version."""
version_cmd = "apptainer --version"
- out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True)
- if ec:
- raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out))
+ res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
- res = re.search(r"\d+\.\d+(\.\d+)?", out.strip())
- if not res:
- raise EasyBuildError("Error parsing Apptainer version: %s" % out)
+ regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
+ if not regex_res:
+ raise EasyBuildError(f"Error parsing Apptainer version: {res.output}")
- return res.group(0)
+ return regex_res.group(0)
def build_image(self, recipe_path):
"""Build container image by calling out to 'sudo apptainer build'."""
@@ -108,5 +108,5 @@ def build_image(self, recipe_path):
cmd = ' '.join(['sudo', cmd_env, apptainer, 'build', cmd_opts, img_path, recipe_path])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd)
- run_cmd(cmd, stream_output=True)
+ run_shell_cmd(cmd, stream_output=True)
print_msg("Apptainer image created at %s" % img_path, log=self.log)
diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py
index faf3b0cc7a..3371719c3d 100644
--- a/easybuild/tools/containers/base.py
+++ b/easybuild/tools/containers/base.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py
index 6824c16864..3a90c98e43 100644
--- a/easybuild/tools/containers/common.py
+++ b/easybuild/tools/containers/common.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py
index 9061bba762..76f5713e09 100644
--- a/easybuild/tools/containers/docker.py
+++ b/easybuild/tools/containers/docker.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,7 @@
from easybuild.tools.containers.utils import det_os_deps
from easybuild.tools.filetools import remove_dir
from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
DOCKER_TMPL_HEADER = """\
@@ -164,7 +164,7 @@ def build_image(self, dockerfile):
docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.'])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd)
- run_cmd(docker_cmd, path=tempdir, stream_output=True)
+ run_shell_cmd(docker_cmd, work_dir=tempdir, stream_output=True)
print_msg("Docker image created at %s" % container_name, log=self.log)
remove_dir(tempdir)
diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py
index 809fdc35d1..3fdac7acd8 100644
--- a/easybuild/tools/containers/singularity.py
+++ b/easybuild/tools/containers/singularity.py
@@ -1,4 +1,4 @@
-# Copyright 2017-2024 Ghent University
+# Copyright 2017-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,14 +34,13 @@
import re
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
from easybuild.tools.config import build_option, container_path
from easybuild.tools.containers.base import ContainerGenerator
from easybuild.tools.filetools import read_file, remove_file, which
-from easybuild.tools.run import run_cmd
-from easybuild.tools.py2vs3 import string_type
+from easybuild.tools.run import run_shell_cmd
ARCH = 'arch' # Arch Linux
@@ -163,15 +162,15 @@ class SingularityContainer(ContainerGenerator):
def singularity_version():
"""Get Singularity version."""
version_cmd = "singularity --version"
- out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True)
- if ec:
- raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out))
+ res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
- res = re.search(r"\d+\.\d+(\.\d+)?", out.strip())
- if not res:
- raise EasyBuildError("Error parsing Singularity version: %s" % out)
+ regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
+ if not regex_res:
+ raise EasyBuildError(f"Error parsing Singularity version: {res.output}")
- return res.group(0)
+ return regex_res.group(0)
def resolve_template(self):
"""Return template container recipe."""
@@ -299,7 +298,7 @@ def resolve_template_data(self):
install_os_deps = []
for osdep in osdeps:
- if isinstance(osdep, string_type):
+ if isinstance(osdep, str):
install_os_deps.append("yum install --quiet --assumeyes %s" % osdep)
# tuple entry indicates multiple options
elif isinstance(osdep, tuple):
@@ -404,5 +403,5 @@ def build_image(self, recipe_path):
cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd)
- run_cmd(cmd, stream_output=True)
+ run_shell_cmd(cmd, stream_output=True)
print_msg("Singularity image created at %s" % img_path, log=self.log)
diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py
index a575254335..956bffad63 100644
--- a/easybuild/tools/containers/utils.py
+++ b/easybuild/tools/containers/utils.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,10 +34,9 @@
from functools import reduce
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.filetools import which
-from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
def det_os_deps(easyconfigs):
@@ -49,7 +48,7 @@ def det_os_deps(easyconfigs):
res = set()
os_deps = reduce(operator.add, [obj['ec']['osdependencies'] for obj in easyconfigs], [])
for os_dep in os_deps:
- if isinstance(os_dep, string_type):
+ if isinstance(os_dep, str):
res.add(os_dep)
elif isinstance(os_dep, tuple):
res.update(os_dep)
@@ -71,20 +70,23 @@ def check_tool(tool_name, min_tool_version=None):
if not tool_path:
return False
- print_msg("{0} tool found at {1}".format(tool_name, tool_path))
+ print_msg(f"{tool_name} tool found at {tool_path}")
if not min_tool_version:
return True
- version_cmd = "{0} --version".format(tool_name)
- out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True)
- if ec:
- raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out))
- res = re.search(r"\d+\.\d+(\.\d+)?", out.strip())
- if not res:
- raise EasyBuildError("Error parsing version for tool {0}".format(tool_name))
- tool_version = res.group(0)
+ version_cmd = f"{tool_name} --version"
+ res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}")
+
+ regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
+ if not regex_res:
+ raise EasyBuildError(f"Error parsing version for tool {tool_name}")
+
+ tool_version = regex_res.group(0)
version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version)
if version_ok:
- print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version))
+ print_msg(f"{tool_name} version '{tool_version}' is {min_tool_version} or higher ... OK")
+
return version_ok
diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py
index 134566812d..3b55a392f6 100644
--- a/easybuild/tools/convert.py
+++ b/easybuild/tools/convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,7 +35,6 @@
from easybuild.base import fancylogger
from easybuild.base.wrapper import Wrapper
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
_log = fancylogger.getLogger('tools.convert', fname=False)
@@ -52,7 +51,7 @@ def __init__(self, obj):
"""Support the conversion of obj to something"""
self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.__dict__['data'] = None
- if isinstance(obj, string_type):
+ if isinstance(obj, str):
self.data = self._from_string(obj)
else:
raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj)
diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py
index d5b9cd4628..d7fba3885d 100644
--- a/easybuild/tools/docs.py
+++ b/easybuild/tools/docs.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -42,6 +42,7 @@
import os
from collections import OrderedDict
from easybuild.tools import LooseVersion
+from string import ascii_lowercase
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories
@@ -49,7 +50,7 @@
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import EasyConfigParser
+from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, EasyConfigParser
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE
@@ -61,8 +62,7 @@
from easybuild.tools.config import build_option
from easybuild.tools.filetools import read_file
from easybuild.tools.modules import modules_tool
-from easybuild.tools.py2vs3 import ascii_lowercase
-from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
+from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
from easybuild.tools.toolchain.utilities import search_toolchain
from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES
from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table, nub, quote_str
@@ -315,7 +315,7 @@ def avail_easyconfig_licenses_md():
return '\n'.join(doc)
-def avail_easyconfig_params_md(title, grouped_params):
+def avail_easyconfig_params_md(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in MarkDown format.
"""
@@ -328,13 +328,14 @@ def avail_easyconfig_params_md(title, grouped_params):
for grpname in grouped_params:
# group section title
title = "%s%s parameters" % (grpname[0].upper(), grpname[1:])
- table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
+ table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"]
keys = sorted(grouped_params[grpname].keys())
values = [grouped_params[grpname][key] for key in keys]
table_values = [
['`%s`' % name for name in keys], # parameter name
[x[0].replace('<', '<').replace('>', '>') for x in values], # description
- ['`' + str(quote_str(x[1])) + '`' for x in values] # default value
+ ['`' + str(quote_str(x[1])) + '`' for x in values], # default value
+ ['`%s`' % alternative_params[name] if name in alternative_params else '' for name in keys],
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
@@ -343,7 +344,7 @@ def avail_easyconfig_params_md(title, grouped_params):
return '\n'.join(doc)
-def avail_easyconfig_params_rst(title, grouped_params):
+def avail_easyconfig_params_rst(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in RST format.
"""
@@ -357,13 +358,14 @@ def avail_easyconfig_params_rst(title, grouped_params):
for grpname in grouped_params:
# group section title
title = "%s parameters" % grpname
- table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
+ table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"]
keys = sorted(grouped_params[grpname].keys())
values = [grouped_params[grpname][key] for key in keys]
table_values = [
['``%s``' % name for name in keys], # parameter name
[x[0] for x in values], # description
- [str(quote_str(x[1])) for x in values] # default value
+ [str(quote_str(x[1])) for x in values], # default value
+ ['``%s``' % alternative_params[name] if name in alternative_params else '' for name in keys],
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
@@ -372,14 +374,14 @@ def avail_easyconfig_params_rst(title, grouped_params):
return '\n'.join(doc)
-def avail_easyconfig_params_json():
+def avail_easyconfig_params_json(*args):
"""
Compose overview of available easyconfig parameters, in json format.
"""
raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
-def avail_easyconfig_params_txt(title, grouped_params):
+def avail_easyconfig_params_txt(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in plain text format.
"""
@@ -399,7 +401,17 @@ def avail_easyconfig_params_txt(title, grouped_params):
# line by parameter
for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
- doc.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
+ line = ' '.join([
+ '{0:<{nw}} ',
+ '{1:}',
+ '[default: {2:}]',
+ ]).format(name, descr, str(quote_str(dflt)), nw=nw)
+
+ alternative = alternative_params.get(name)
+ if alternative:
+ line += ' {alternative: %s}' % alternative
+
+ doc.append(line)
doc.append('')
return '\n'.join(doc)
@@ -418,6 +430,9 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT):
extra_params = app.extra_options()
params.update(extra_params)
+ # reverse mapping of alternative easyconfig parameter names
+ alternative_params = {v: k for k, v in ALTERNATIVE_EASYCONFIG_PARAMETERS.items()}
+
# compose title
title = "Available easyconfig parameters"
if extra_params:
@@ -443,7 +458,7 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT):
del grouped_params[grpname]
# compose output, according to specified format (txt, rst, ...)
- return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params])
+ return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params, alternative_params])
def avail_easyconfig_templates(output_format=FORMAT_TXT):
@@ -463,16 +478,16 @@ def avail_easyconfig_templates_txt():
# step 1: add TEMPLATE_NAMES_EASYCONFIG
doc.append('Template names/values derived from easyconfig instance')
- for name in TEMPLATE_NAMES_EASYCONFIG:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, curDoc in TEMPLATE_NAMES_EASYCONFIG.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, curDoc))
doc.append('')
# step 2: add SOFTWARE_VERSIONS
doc.append('Template names/values for (short) software versions')
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name))
- doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name))
- doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name))
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, prefix, name))
+ doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, prefix, name))
+ doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, prefix, name))
doc.append('')
# step 3: add remaining config
@@ -491,20 +506,20 @@ def avail_easyconfig_templates_txt():
# step 5: template_values can/should be updated from outside easyconfig
# (eg the run_step code in EasyBlock)
doc.append('Template values set outside EasyBlock runstep')
- for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
doc.append('')
# some template values are only defined dynamically,
# see template_constant_dict function in easybuild.framework.easyconfigs.templates
doc.append('Template values which are defined dynamically')
- for name in TEMPLATE_NAMES_DYNAMIC:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_DYNAMIC.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
doc.append('')
doc.append('Template constants that can be used in easyconfigs')
- for cst in TEMPLATE_CONSTANTS:
- doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1]))
+ for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items():
+ doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, cur_doc, value))
return '\n'.join(doc)
@@ -515,8 +530,8 @@ def avail_easyconfig_templates_rst():
title = 'Template names/values derived from easyconfig instance'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
- [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+ list(TEMPLATE_NAMES_EASYCONFIG.values()),
]
doc = rst_title_and_table(title, table_titles, table_values)
doc.append('')
@@ -524,10 +539,10 @@ def avail_easyconfig_templates_rst():
title = 'Template names/values for (short) software versions'
ver = []
ver_desc = []
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- ver.append('``%%(%smajver)s``' % pref)
- ver.append('``%%(%sshortver)s``' % pref)
- ver.append('``%%(%sver)s``' % pref)
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ ver.append('``%%(%smajver)s``' % prefix)
+ ver.append('``%%(%sshortver)s``' % prefix)
+ ver.append('``%%(%sver)s``' % prefix)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (.)' % name)
ver_desc.append('full version for %s' % name)
@@ -550,24 +565,24 @@ def avail_easyconfig_templates_rst():
title = 'Template values set outside EasyBlock runstep'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
- [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
title = 'Template values which are defined dynamically'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
- [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+ list(TEMPLATE_NAMES_DYNAMIC.values()),
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
title = 'Template constants that can be used in easyconfigs'
- titles = ['Constant', 'Template value', 'Template name']
+ titles = ['Constant', 'Template description', 'Template value']
table_values = [
- ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
- [cst[2] for cst in TEMPLATE_CONSTANTS],
- ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+ ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+ [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+ ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
]
doc.extend(rst_title_and_table(title, titles, table_values))
@@ -580,8 +595,8 @@ def avail_easyconfig_templates_md():
title = 'Template names/values derived from easyconfig instance'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
- [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+ list(TEMPLATE_NAMES_EASYCONFIG.values()),
]
doc = md_title_and_table(title, table_titles, table_values, title_level=2)
doc.append('')
@@ -589,10 +604,10 @@ def avail_easyconfig_templates_md():
title = 'Template names/values for (short) software versions'
ver = []
ver_desc = []
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- ver.append('``%%(%smajver)s``' % pref)
- ver.append('``%%(%sshortver)s``' % pref)
- ver.append('``%%(%sver)s``' % pref)
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ ver.append('``%%(%smajver)s``' % prefix)
+ ver.append('``%%(%sshortver)s``' % prefix)
+ ver.append('``%%(%sver)s``' % prefix)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (``.``)' % name)
ver_desc.append('full version for %s' % name)
@@ -616,27 +631,28 @@ def avail_easyconfig_templates_md():
title = 'Template values set outside EasyBlock runstep'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
- [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
doc.append('')
title = 'Template values which are defined dynamically'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
- [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+ list(TEMPLATE_NAMES_DYNAMIC.values()),
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
doc.append('')
title = 'Template constants that can be used in easyconfigs'
- titles = ['Constant', 'Template value', 'Template name']
+ titles = ['Constant', 'Template description', 'Template value']
table_values = [
- ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
- [cst[2] for cst in TEMPLATE_CONSTANTS],
- ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+ ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+ [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+ ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
]
+
doc.extend(md_title_and_table(title, titles, table_values, title_level=2))
return '\n'.join(doc)
@@ -807,8 +823,8 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
- for key in all_software:
- for entry in all_software[key]:
+ for key, entries in all_software.items():
+ for entry in entries:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
@@ -1094,7 +1110,7 @@ def list_toolchains(output_format=FORMAT_TXT):
# start with dict that maps toolchain name to corresponding subclass of Toolchain
# filter deprecated 'dummy' toolchain
- tcs = {tc.NAME: tc for tc in all_tcs if tc.NAME != DUMMY_TOOLCHAIN_NAME}
+ tcs = {tc.NAME: tc for tc in all_tcs}
for tcname in sorted(tcs):
tcc = tcs[tcname]
@@ -1125,7 +1141,7 @@ def list_toolchains_md(tcs):
none_txt = '*(none)*'
# Initialize an empty list of lists for the table data
- table_values = [[] for i in range(len(table_titles))]
+ table_values = [[] for _ in range(len(table_titles))]
for col_id, col_name in enumerate(table_titles):
if col_name == 'NAME':
@@ -1141,6 +1157,8 @@ def list_toolchains_md(tcs):
entry = 'cray-mpich'
elif col_name == 'LINALG':
entry = 'cray-libsci'
+ else:
+ entry = none_txt
# Combine the linear algebra libraries into a single column
elif col_name == 'LINALG':
linalg = []
@@ -1185,7 +1203,7 @@ def list_toolchains_rst(tcs):
none_txt = '*(none)*'
# Initialize an empty list of lists for the table data
- table_values = [[] for i in range(len(table_titles))]
+ table_values = [[] for _ in range(len(table_titles))]
for col_id, col_name in enumerate(table_titles):
if col_name == 'NAME':
@@ -1201,6 +1219,8 @@ def list_toolchains_rst(tcs):
entry = 'cray-mpich'
elif col_name == 'LINALG':
entry = 'cray-libsci'
+ else:
+ entry = none_txt
# Combine the linear algebra libraries into a single column
elif col_name == 'LINALG':
linalg = []
@@ -1309,15 +1329,19 @@ def get_easyblock_classes(package_name):
"""
Get list of all easyblock classes in specified easyblocks.* package
"""
- easyblocks = []
+ easyblocks = set()
modules = import_available_modules(package_name)
for mod in modules:
+ easyblock_found = False
for name, _ in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
- if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks:
- easyblocks.append(eb_class)
+ if eb_class.__module__.startswith(package_name) and EasyBlock in inspect.getmro(eb_class):
+ easyblocks.add(eb_class)
+ easyblock_found = True
+ if not easyblock_found:
+ raise RuntimeError("No easyblocks found in module: %s", mod.__name__)
return easyblocks
diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py
index a5cc708ce2..cf6bb2e09c 100644
--- a/easybuild/tools/environment.py
+++ b/easybuild/tools/environment.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -54,8 +54,8 @@ def write_changes(filename):
"""
try:
with open(filename, 'w') as script:
- for key in _changes:
- script.write('export %s=%s\n' % (key, shell_quote(_changes[key])))
+ for key, changed_value in _changes.items():
+ script.write('export %s=%s\n' % (key, shell_quote(changed_value)))
except IOError as err:
raise EasyBuildError("Failed to write to %s: %s", filename, err)
reset_changes()
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 973f83e1f5..00494e5bd9 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -48,6 +48,7 @@
import inspect
import itertools
import os
+import pathlib
import platform
import re
import shutil
@@ -55,19 +56,23 @@
import stat
import ssl
import sys
+import tarfile
import tempfile
import time
import zlib
from functools import partial
+from html.parser import HTMLParser
+import urllib.request as std_urllib
from easybuild.base import fancylogger
-from easybuild.tools import LooseVersion, run
+from easybuild.tools import LooseVersion
# import build_log must stay, to use of EasyBuildLog
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
-from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
-from easybuild.tools.config import build_option, install_path
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
+from easybuild.tools.build_log import dry_run_msg, print_msg, print_warning
+from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
-from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type
+from easybuild.tools.hooks import load_source
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
try:
@@ -122,13 +127,25 @@
CHECKSUM_TYPE_MD5 = 'md5'
CHECKSUM_TYPE_SHA256 = 'sha256'
-DEFAULT_CHECKSUM = CHECKSUM_TYPE_MD5
+DEFAULT_CHECKSUM = CHECKSUM_TYPE_SHA256
+
+
+def _hashlib_md5():
+ """
+ Wrapper function for hashlib.md5,
+ to set usedforsecurity to False when supported (Python >= 3.9)
+ """
+ kwargs = {}
+ if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
+ kwargs = {'usedforsecurity': False}
+ return hashlib.md5(**kwargs)
+
# map of checksum types to checksum functions
CHECKSUM_FUNCTIONS = {
'adler32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.adler32)),
'crc32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.crc32)),
- CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, hashlib.md5()),
+ CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, _hashlib_md5()),
'sha1': lambda p: calc_block_checksum(p, hashlib.sha1()),
CHECKSUM_TYPE_SHA256: lambda p: calc_block_checksum(p, hashlib.sha256()),
'sha512': lambda p: calc_block_checksum(p, hashlib.sha512()),
@@ -394,7 +411,7 @@ def remove(paths):
:param paths: path(s) to remove
"""
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
paths = [paths]
_log.info("Removing %d files & directories", len(paths))
@@ -408,6 +425,22 @@ def remove(paths):
raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path)
+def get_cwd(must_exist=True):
+ """
+ Retrieve current working directory
+ """
+ try:
+ cwd = os.getcwd()
+ except FileNotFoundError as err:
+ if must_exist is True:
+ raise EasyBuildError(CWD_NOTFOUND_ERROR)
+
+ _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err)
+ cwd = None
+
+ return cwd
+
+
def change_dir(path):
"""
Change to directory at specified location.
@@ -415,22 +448,23 @@ def change_dir(path):
:param path: location to change to
:return: previous location we were in
"""
- # determining the current working directory can fail if we're in a non-existing directory
- try:
- cwd = os.getcwd()
- except OSError as err:
- _log.debug("Failed to determine current working directory (but proceeding anyway: %s", err)
- cwd = None
+ # determine origin working directory: can fail if non-existent
+ prev_dir = get_cwd(must_exist=False)
try:
os.chdir(path)
except OSError as err:
- raise EasyBuildError("Failed to change from %s to %s: %s", cwd, path, err)
+ raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err)
- return cwd
+ # determine final working directory: must exist
+ # stoplight meant to catch filesystems in a faulty state
+ get_cwd()
+
+ return prev_dir
-def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=None):
+def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False,
+ trace=True):
"""
Extract file at given path to specified directory
:param fn: path to file to extract
@@ -439,18 +473,13 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
:param extra_options: extra options to pass to extract command
:param overwrite: overwrite existing unpacked file
:param forced: force extraction in (extended) dry run mode
- :param change_into_dir: change into resulting directory;
- None (current default) implies True, but this is deprecated,
- this named argument should be set to False or True explicitely
- (in a future major release, default will be changed to False)
+ :param change_into_dir: change into resulting directorys
+ :param trace: produce trace output for extract command being run
:return: path to directory (in case of success)
"""
- if change_into_dir is None:
- _log.deprecated("extract_file function was called without specifying value for change_into_dir", '5.0')
- change_into_dir = True
if not os.path.isfile(fn) and not build_option('extended_dry_run'):
- raise EasyBuildError("Can't extract file %s: no such file", fn)
+ raise EasyBuildError(f"Can't extract file {fn}: no such file")
mkdir(dest, parents=True)
@@ -458,24 +487,24 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
abs_dest = os.path.abspath(dest)
# change working directory
- _log.debug("Unpacking %s in directory %s", fn, abs_dest)
+ _log.debug(f"Unpacking {fn} in directory {abs_dest}")
cwd = change_dir(abs_dest)
if cmd:
# complete command template with filename
cmd = cmd % fn
- _log.debug("Using specified command to unpack %s: %s", fn, cmd)
+ _log.debug(f"Using specified command to unpack {fn}: {cmd}")
else:
cmd = extract_cmd(fn, overwrite=overwrite)
- _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd)
+ _log.debug(f"Using command derived from file extension to unpack {fn}: {cmd}")
if not cmd:
- raise EasyBuildError("Can't extract file %s with unknown filetype", fn)
+ raise EasyBuildError(f"Can't extract file {fn} with unknown filetype")
if extra_options:
- cmd = "%s %s" % (cmd, extra_options)
+ cmd = f"{cmd} {extra_options}"
- run.run_cmd(cmd, simple=True, force_in_dry_run=forced)
+ run_shell_cmd(cmd, in_dry_run=forced, hidden=not trace)
# note: find_base_dir also changes into the base dir!
base_dir = find_base_dir()
@@ -484,14 +513,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
# change back to where we came from (unless that was a non-existing directory)
if not change_into_dir:
if cwd is None:
- raise EasyBuildError("Can't change back to non-existing directory after extracting %s in %s", fn, dest)
+ raise EasyBuildError(f"Can't change back to non-existing directory after extracting {fn} in {dest}")
else:
change_dir(cwd)
return base_dir
-def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=None):
+def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN):
"""
Return (first) path in $PATH for specified command, or None if command is not found
@@ -500,17 +529,6 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None,
:param log_ok: Log an info message where the command has been found (if any)
:param on_error: What to do if the command was not found, default: WARN. Possible values: IGNORE, WARN, ERROR
"""
- if log_error is not None:
- _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0')
- # If set, make sure on_error is at least WARN
- if log_error and on_error == IGNORE:
- on_error = WARN
- elif not log_error and on_error is None: # If set to False, use IGNORE unless on_error is also set
- on_error = IGNORE
- # Set default
- # TODO: After removal of log_error from the parameters, on_error=WARN can be used instead of this
- if on_error is None:
- on_error = WARN
if on_error not in (IGNORE, WARN, ERROR):
raise EasyBuildError("Invalid value for 'on_error': %s", on_error)
@@ -585,12 +603,25 @@ def normalize_path(path):
return start_slashes + os.path.sep.join(filtered_comps)
+def is_parent_path(path1, path2):
+ """
+ Return True if path1 is a prefix of path2
+
+ :param path1: absolute or relative path
+ :param path2: absolute or relative path
+ """
+ path1 = os.path.realpath(path1)
+ path2 = os.path.realpath(path2)
+ common_path = os.path.commonprefix([path1, path2])
+ return common_path == path1
+
+
def is_alt_pypi_url(url):
- """Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash."""
+ """Determine whether specified URL is already an alternative PyPI URL, i.e. whether it contains a hash."""
# example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz
alt_url_regex = re.compile('/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/[^/]+$')
res = bool(alt_url_regex.search(url))
- _log.debug("Checking whether '%s' is an alternate PyPI URL using pattern '%s'...: %s",
+ _log.debug("Checking whether '%s' is an alternative PyPI URL using pattern '%s'...: %s",
url, alt_url_regex.pattern, res)
return res
@@ -638,7 +669,7 @@ def handle_starttag(self, tag, attrs):
def derive_alt_pypi_url(url):
- """Derive alternate PyPI URL for given URL."""
+ """Derive alternative PyPI URL for given URL."""
alt_pypi_url = None
# example input URL: https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz
@@ -687,9 +718,9 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
if argline == '' or '#' in argline[0]:
continue # permit comment lines: ignore them
- if os.path.isfile(os.path.join(os.getcwd(), argline)):
+ if os.path.isfile(os.path.join(get_cwd(), argline)):
# expand existing relative path to absolute
- argline = os.path.join(os.path.join(os.getcwd(), argline))
+ argline = os.path.join(os.path.join(get_cwd(), argline))
if os.path.isfile(argline):
# argline is a file path, so read that instead
_log.debug('File included in parse_http_header_fields_urlpat: %s' % argline)
@@ -746,7 +777,7 @@ def det_file_size(http_header):
return res
-def download_file(filename, url, path, forced=False):
+def download_file(filename, url, path, forced=False, trace=True):
"""Download a file from the given URL, to the specified path."""
insecure = build_option('insecure_download')
@@ -842,7 +873,10 @@ def download_file(filename, url, path, forced=False):
if error_re.match(str(err)):
switch_to_requests = True
except Exception as err:
- raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err)
+ raise EasyBuildError(
+ "Unexpected error occurred when trying to download %s to %s: %s", url, path, err,
+ exit_code=EasyBuildExit.FAIL_DOWNLOAD
+ )
if not downloaded and attempt_cnt < max_attempts:
_log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path))
@@ -855,11 +889,13 @@ def download_file(filename, url, path, forced=False):
if downloaded:
_log.info("Successful download of file %s from url %s to path %s" % (filename, url, path))
- trace_msg("download succeeded: %s" % url)
+ if trace:
+ trace_msg("download succeeded: %s" % url)
return path
else:
_log.warning("Download of %s to %s failed, done trying" % (url, path))
- trace_msg("download failed: %s" % url)
+ if trace:
+ trace_msg("download failed: %s" % url)
return None
@@ -1055,7 +1091,10 @@ def locate_files(files, paths, ignore_subdirs=None):
if files_to_find:
filenames = ', '.join([f for (_, f) in files_to_find])
paths = ', '.join(paths)
- raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths)
+ raise EasyBuildError(
+ "One or more files not found: %s (search paths: %s)", filenames, paths,
+ exit_code=EasyBuildExit.MISSING_EASYCONFIG
+ )
return [os.path.abspath(f) for f in files]
@@ -1206,12 +1245,16 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM):
Compute checksum of specified file.
:param path: Path of file to compute checksum for
- :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'sha256', 'sha512', 'size')
+ :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5', 'sha1', 'sha256', 'sha512', 'size')
"""
if checksum_type not in CHECKSUM_FUNCTIONS:
raise EasyBuildError("Unknown checksum type (%s), supported types are: %s",
checksum_type, CHECKSUM_FUNCTIONS.keys())
+ if checksum_type in ['adler32', 'crc32', 'md5', 'sha1', 'size']:
+ _log.deprecated("Checksum type %s is deprecated. Use sha256 (default) or sha512 instead" % checksum_type,
+ '6.0')
+
try:
checksum = CHECKSUM_FUNCTIONS[checksum_type](path)
except IOError as err:
@@ -1249,8 +1292,7 @@ def verify_checksum(path, checksums, computed_checksums=None):
Verify checksum of specified file.
:param path: path of file to verify checksum of
- :param checksums: checksum values to compare to
- (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b')
+ :param checksums: checksum values (and type, optionally, default is sha256), e.g., 'af314', ('sha', '5ec1b')
:param computed_checksums: Optional dictionary of (current) checksum(s) for this file
indexed by the checksum type (e.g. 'sha256').
Each existing entry will be used, missing ones will be computed.
@@ -1276,8 +1318,11 @@ def verify_checksum(path, checksums, computed_checksums=None):
checksum = checksum[filename]
except KeyError:
raise EasyBuildError("Missing checksum for %s in %s", filename, checksum)
+ if not verify_checksum(path, checksum, computed_checksums):
+ return False
+ continue
- if isinstance(checksum, string_type):
+ if isinstance(checksum, str):
# if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters)
if len(checksum) == 64:
typ = CHECKSUM_TYPE_SHA256
@@ -1305,7 +1350,7 @@ def verify_checksum(path, checksums, computed_checksums=None):
# no matching checksums
return False
else:
- raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+ raise EasyBuildError("Invalid checksum spec '%s': should be a string (SHA256), "
"2-tuple (type, value), or tuple of alternative checksum specs.",
checksum)
@@ -1328,7 +1373,7 @@ def verify_checksum(path, checksums, computed_checksums=None):
def is_sha256_checksum(value):
"""Check whether provided string is a SHA256 checksum."""
res = False
- if isinstance(value, string_type):
+ if isinstance(value, str):
if re.match('^[0-9a-f]{64}$', value):
res = True
_log.debug("String value '%s' has the correct format to be a SHA256 checksum", value)
@@ -1352,14 +1397,14 @@ def get_local_dirs_purged():
# and hidden directories
ignoredirs = ["easybuild"]
- lst = os.listdir(os.getcwd())
+ lst = os.listdir(get_cwd())
lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs]
return lst
lst = get_local_dirs_purged()
- new_dir = os.getcwd()
+ new_dir = get_cwd()
while len(lst) == 1:
- new_dir = os.path.join(os.getcwd(), lst[0])
+ new_dir = os.path.join(get_cwd(), lst[0])
if not os.path.isdir(new_dir):
break
@@ -1381,13 +1426,12 @@ def find_extension(filename):
suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True)
pat = r'(?P%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes])
res = re.search(pat, filename, flags=re.IGNORECASE)
+
if res:
- ext = res.group('ext')
+ return res.group('ext')
else:
raise EasyBuildError("%s has unknown file extension", filename)
- return ext
-
def extract_cmd(filepath, overwrite=False):
"""
@@ -1500,17 +1544,19 @@ def create_patch_info(patch_spec):
# string value as patch argument can be either path where patch should be applied,
# or path to where a non-patch file should be copied
- elif isinstance(patch_arg, string_type):
+ elif isinstance(patch_arg, str):
if patch_spec[0].endswith('.patch'):
patch_info['sourcepath'] = patch_arg
# non-patch files are assumed to be files to copy
else:
patch_info['copy'] = patch_arg
else:
- raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
- str(patch_spec))
+ raise EasyBuildError(
+ "Wrong patch spec '%s', only int/string are supported as 2nd element", str(patch_spec),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
- elif isinstance(patch_spec, string_type):
+ elif isinstance(patch_spec, str):
validate_patch_spec(patch_spec)
patch_info = {'name': patch_spec}
elif isinstance(patch_spec, dict):
@@ -1519,13 +1565,17 @@ def create_patch_info(patch_spec):
if key in valid_keys:
patch_info[key] = patch_spec[key]
else:
- raise EasyBuildError("Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
- str(patch_spec), key, valid_keys)
+ raise EasyBuildError(
+ "Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
+ str(patch_spec), key, valid_keys, exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
# Dict must contain at least the patchfile name
if 'name' not in patch_info.keys():
- raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
- str(patch_spec))
+ raise EasyBuildError(
+ "Wrong patch spec '%s', when using a dict 'name' entry must be supplied", str(patch_spec),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
if 'copy' not in patch_info.keys():
validate_patch_spec(patch_info['name'])
else:
@@ -1534,9 +1584,11 @@ def create_patch_info(patch_spec):
"this implies you want to copy a file to the 'copy' location)",
str(patch_spec))
else:
- error_msg = "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " \
- "(with possible keys %s): %s" % (valid_keys, patch_spec)
- raise EasyBuildError(error_msg)
+ error_msg = (
+ "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict "
+ f"(with possible keys {valid_keys}): {patch_spec}"
+ )
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR)
return patch_info
@@ -1544,12 +1596,13 @@ def create_patch_info(patch_spec):
def validate_patch_spec(patch_spec):
allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
- msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
- msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
- _log.deprecated(msg, '5.0')
+ raise EasyBuildError(
+ "Wrong patch spec (%s), extension type should be any of %s.", patch_spec, ', '.join(allowed_patch_exts),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
-def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
+def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False):
"""
Apply a patch to source code in directory dest
- assume unified diff created with "diff -ru old new"
@@ -1557,34 +1610,30 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
Raises EasyBuildError on any error and returns True on success
"""
- if use_git_am:
- _log.deprecated("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'", '5.0')
- use_git = True
-
if build_option('extended_dry_run'):
# skip checking of files in dry run mode
patch_filename = os.path.basename(patch_file)
- dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent'))
+ dry_run_msg(f"* applying patch file {patch_filename}", silent=build_option('silent'))
elif not os.path.isfile(patch_file):
- raise EasyBuildError("Can't find patch %s: no such file", patch_file)
+ raise EasyBuildError(f"Can't find patch {patch_file}: no such file")
elif fn and not os.path.isfile(fn):
- raise EasyBuildError("Can't patch file %s: no such file", fn)
+ raise EasyBuildError(f"Can't patch file {fn}: no such file")
# copy missing files
if copy:
if build_option('extended_dry_run'):
- dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent'))
+ dry_run_msg(f" {patch_file} copied to {dest}", silent=build_option('silent'))
else:
copy_file(patch_file, dest)
- _log.debug("Copied patch %s to dir %s" % (patch_file, dest))
+ _log.debug(f"Copied patch {patch_file} to dir {dest}")
# early exit, work is done after copying
return True
elif not os.path.isdir(dest):
- raise EasyBuildError("Can't patch directory %s: no such directory", dest)
+ raise EasyBuildError(f"Can't patch directory {dest}: no such directory")
# use absolute paths
abs_patch_file = os.path.abspath(patch_file)
@@ -1599,14 +1648,14 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
patch_subextension = os.path.splitext(patch_stem)[1]
if patch_subextension == ".patch":
workdir = tempfile.mkdtemp(prefix='eb-patch-')
- _log.debug("Extracting the patch to: %s", workdir)
+ _log.debug(f"Extracting the patch to: {workdir}")
# extracting the patch
extracted_dir = extract_file(abs_patch_file, workdir, change_into_dir=False)
abs_patch_file = os.path.join(extracted_dir, patch_stem)
if use_git:
verbose = '--verbose ' if build_option('debug') else ''
- patch_cmd = "git apply %s%s" % (verbose, abs_patch_file)
+ patch_cmd = f"git apply {verbose}{abs_patch_file}"
else:
if level is None and build_option('extended_dry_run'):
level = ''
@@ -1619,63 +1668,79 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
patched_files = det_patched_files(path=abs_patch_file)
if not patched_files:
- raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch",
- abs_patch_file)
+ msg = f"Can't guess patchlevel from patch {abs_patch_file}: no testfile line found in patch"
+ raise EasyBuildError(msg)
level = guess_patch_level(patched_files, abs_dest)
if level is None: # level can also be 0 (zero), so don't use "not level"
# no match
- raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, abs_dest)
+ raise EasyBuildError(f"Can't determine patch level for patch {patch_file} from directory {abs_dest}")
else:
- _log.debug("Guessed patch level %d for patch %s" % (level, patch_file))
+ _log.debug(f"Guessed patch level {level} for patch {patch_file}")
else:
- _log.debug("Using specified patch level %d for patch %s" % (level, patch_file))
+ _log.debug(f"Using specified patch level {level} for patch {patch_file}")
backup_option = '-b ' if build_option('backup_patched_files') else ''
- patch_cmd = "patch " + backup_option + "-p%s -i %s" % (level, abs_patch_file)
+ patch_cmd = f"patch {backup_option} -p{level} -i {abs_patch_file}"
- out, ec = run.run_cmd(patch_cmd, simple=False, path=abs_dest, log_ok=False, trace=False)
+ res = run_shell_cmd(patch_cmd, fail_on_error=False, hidden=True, work_dir=abs_dest)
+
+ if res.exit_code:
+ msg = f"Couldn't apply patch file {patch_file}. "
+ msg += f"Process exited with code {res.exit_code}: {res.output}"
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_PATCH_APPLY)
- if ec:
- raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out)
return True
-def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_match=None):
+def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb',
+ on_missing_match=None, match_all=False, single_line=True):
"""
Apply specified list of regex substitutions.
:param paths: list of paths to files to patch (or just a single filepath)
- :param regex_subs: list of substitutions to apply, specified as (, )
+ :param regex_subs: list of substitutions to apply,
+ specified as (, )
:param backup: create backup of original file with specified suffix (no backup if value evaluates to False)
:param on_missing_match: Define what to do when no match was found in the file.
Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing
Defaults to the value of --strict
+ :param match_all: Expect to match all patterns in all files
+ instead of at least one per file for error/warning reporting
+ :param single_line: Replace first match of each pattern for each line in the order of the patterns.
+ If False the patterns are applied in order to the full text and may match line breaks.
"""
if on_missing_match is None:
on_missing_match = build_option('strict')
allowed_values = (ERROR, IGNORE, WARN)
if on_missing_match not in allowed_values:
- raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)',
- on_missing_match, ', '.join(allowed_values))
+ raise ValueError('Invalid value passed to on_missing_match: %s (allowed: %s)',
+ on_missing_match, ', '.join(allowed_values))
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
paths = [paths]
+ if (not isinstance(regex_subs, (list, tuple)) or
+ not all(isinstance(sub, (list, tuple)) and len(sub) == 2 for sub in regex_subs)):
+ raise ValueError('Parameter regex_subs must be a list of 2-element tuples. Got:', regex_subs)
+
+ flags = 0 if single_line else re.M
+ compiled_regex_subs = [(re.compile(regex, flags) if isinstance(regex, str) else regex, subtxt)
+ for (regex, subtxt) in regex_subs]
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
paths_str = ', '.join(paths)
dry_run_msg("applying regex substitutions to file(s): %s" % paths_str, silent=build_option('silent'))
- for regex, subtxt in regex_subs:
- dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt))
+ for regex, subtxt in compiled_regex_subs:
+ dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex.pattern, subtxt))
else:
- _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs)
-
- compiled_regex_subs = [(re.compile(regex), subtxt) for (regex, subtxt) in regex_subs]
+ _log.info("Applying following regex substitutions to %s: %s",
+ paths, [(regex.pattern, subtxt) for regex, subtxt in compiled_regex_subs])
+ replacement_failed_msgs = []
for path in paths:
try:
# make sure that file can be opened in text mode;
@@ -1695,32 +1760,49 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m
if backup:
copy_file(path, path + backup)
replacement_msgs = []
+ replaced = [False] * len(compiled_regex_subs)
with open_file(path, 'w') as out_file:
- lines = txt_utf8.split('\n')
- del txt_utf8
- for line_id, line in enumerate(lines):
- for regex, subtxt in compiled_regex_subs:
- match = regex.search(line)
- if match:
+ if single_line:
+ lines = txt_utf8.split('\n')
+ del txt_utf8
+ for line_id, line in enumerate(lines):
+ for i, (regex, subtxt) in enumerate(compiled_regex_subs):
+ match = regex.search(line)
+ if match:
+ origtxt = match.group(0)
+ replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" %
+ (line_id + 1, origtxt, subtxt))
+ replaced[i] = True
+ line = regex.sub(subtxt, line)
+ lines[line_id] = line
+ out_file.write('\n'.join(lines))
+ else:
+ for i, (regex, subtxt) in enumerate(compiled_regex_subs):
+ def do_replace(match):
origtxt = match.group(0)
- replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" %
- (line_id + 1, origtxt, subtxt))
- line = regex.sub(subtxt, line)
- lines[line_id] = line
- out_file.write('\n'.join(lines))
+ # pylint: disable=cell-var-from-loop
+ cur_subtxt = match.expand(subtxt)
+ # pylint: disable=cell-var-from-loop
+ replacement_msgs.append("Replaced: '%s' -> '%s'" % (origtxt, cur_subtxt))
+ return cur_subtxt
+ txt_utf8, replaced[i] = regex.subn(do_replace, txt_utf8)
+ out_file.write(txt_utf8)
if replacement_msgs:
_log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs))
- else:
- msg = 'Nothing found to replace in %s' % path
- if on_missing_match == ERROR:
- raise EasyBuildError(msg)
- elif on_missing_match == WARN:
- _log.warning(msg)
- else:
- _log.info(msg)
-
+ if (match_all and not all(replaced)) or (not match_all and not any(replaced)):
+ errors = ["Nothing found to replace '%s'" % regex.pattern
+ for cur_replaced, (regex, _) in zip(replaced, compiled_regex_subs) if not cur_replaced]
+ replacement_failed_msgs.append(', '.join(errors) + ' in ' + path)
except (IOError, OSError) as err:
raise EasyBuildError("Failed to patch %s: %s", path, err)
+ if replacement_failed_msgs:
+ msg = '\n'.join(replacement_failed_msgs)
+ if on_missing_match == ERROR:
+ raise EasyBuildError(msg)
+ elif on_missing_match == WARN:
+ _log.warning(msg)
+ else:
+ _log.info(msg)
def modify_env(old, new):
@@ -1748,7 +1830,7 @@ def convert_name(name, upper=False):
def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False, onlydirs=False, recursive=True,
- group_id=None, relative=True, ignore_errors=False, skip_symlinks=None):
+ group_id=None, relative=True, ignore_errors=False):
"""
Change permissions for specified path, using specified permission bits
@@ -1765,11 +1847,6 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False
and directories (if onlyfiles is False) in path
"""
- if skip_symlinks is not None:
- depr_msg = "Use of 'skip_symlinks' argument for 'adjust_permissions' is deprecated "
- depr_msg += "(symlinks are never followed anymore)"
- _log.deprecated(depr_msg, '4.0')
-
provided_path = os.path.abspath(provided_path)
if recursive:
@@ -1929,9 +2006,15 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
# climb up until we hit an existing path or the empty string (for relative paths)
while existing_parent_path and not os.path.exists(existing_parent_path):
existing_parent_path = os.path.dirname(existing_parent_path)
- makedirs(path, exist_ok=True)
+ os.makedirs(path, exist_ok=True)
else:
os.mkdir(path)
+ except FileExistsError as err:
+ if os.path.exists(path):
+ # This may happen if a parallel build creates the directory after we checked for its existence
+ _log.debug("Directory creation aborted as it seems it was already created: %s", err)
+ else:
+ raise EasyBuildError("Failed to create directory %s: %s", path, err)
except OSError as err:
raise EasyBuildError("Failed to create directory %s: %s", path, err)
@@ -1972,7 +2055,7 @@ def check_lock(lock_name):
Check whether a lock with specified name already exists.
If it exists, either wait until it's released, or raise an error
- (depending on --wait-on-lock configuration option).
+ (depending on --wait-on-lock-* configuration option).
"""
lock_path = det_lock_path(lock_name)
if os.path.exists(lock_path):
@@ -1981,22 +2064,6 @@ def check_lock(lock_name):
wait_interval = build_option('wait_on_lock_interval')
wait_limit = build_option('wait_on_lock_limit')
- # --wait-on-lock is deprecated, should use --wait-on-lock-limit and --wait-on-lock-interval instead
- wait_on_lock = build_option('wait_on_lock')
- if wait_on_lock is not None:
- depr_msg = "Use of --wait-on-lock is deprecated, use --wait-on-lock-limit and --wait-on-lock-interval"
- _log.deprecated(depr_msg, '5.0')
-
- # if --wait-on-lock-interval has default value and --wait-on-lock is specified too, the latter wins
- # (required for backwards compatibility)
- if wait_interval == DEFAULT_WAIT_ON_LOCK_INTERVAL and wait_on_lock > 0:
- wait_interval = wait_on_lock
-
- # if --wait-on-lock-limit is not specified we need to wait indefinitely if --wait-on-lock is specified,
- # since the original semantics of --wait-on-lock was that it specified the waiting time interval (no limit)
- if not wait_limit:
- wait_limit = -1
-
# wait limit could be zero (no waiting), -1 (no waiting limit) or non-zero value (waiting limit in seconds)
if wait_limit != 0:
wait_time = 0
@@ -2115,13 +2182,6 @@ def path_matches(path, paths):
return False
-def rmtree2(path, n=3):
- """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems."""
-
- _log.deprecated("Use 'remove_dir' rather than 'rmtree2'", '5.0')
- remove_dir(path)
-
-
def find_backup_name_candidate(src_file):
"""Returns a non-existing file to be used as destination for backup files"""
@@ -2175,7 +2235,7 @@ def move_logs(src_logfile, target_logfile):
try:
# there may be multiple log files, due to log rotation
- app_logs = glob.glob('%s*' % src_logfile)
+ app_logs = glob.glob(f'{src_logfile}*')
for app_log in app_logs:
# retain possible suffix
new_log_path = target_logfile + app_log[src_logfile_len:]
@@ -2186,11 +2246,11 @@ def move_logs(src_logfile, target_logfile):
# move log to target path
move_file(app_log, new_log_path)
- _log.info("Moved log file %s to %s" % (src_logfile, new_log_path))
+ _log.info(f"Moved log file {src_logfile} to {new_log_path}")
if zip_log_cmd:
- run.run_cmd("%s %s" % (zip_log_cmd, new_log_path))
- _log.info("Zipped log %s using '%s'", new_log_path, zip_log_cmd)
+ run_shell_cmd(f"{zip_log_cmd} {new_log_path}")
+ _log.info(f"Zipped log {new_log_path} using '{zip_log_cmd}'")
except (IOError, OSError) as err:
raise EasyBuildError("Failed to move log file(s) %s* to new log file %s*: %s",
@@ -2228,11 +2288,6 @@ def cleanup(logfile, tempdir, testing, silent=False):
print_msg(msg, log=None, silent=testing or silent)
-def copytree(src, dst, symlinks=False, ignore=None):
- """DEPRECATED and removed. Use copy_dir"""
- _log.deprecated("Use 'copy_dir' rather than 'copytree'", '4.0')
-
-
def encode_string(name):
"""
This encoding function handles funky software names ad infinitum, like:
@@ -2280,21 +2335,6 @@ def decode_class_name(name):
return decode_string(name)
-def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None):
- """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead"""
- _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0')
-
-
-def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None):
- """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead"""
- _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0')
-
-
-def parse_log_for_error(txt, regExp=None, stdout=True, msg=None):
- """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead"""
- _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0')
-
-
def det_size(path):
"""
Determine total size of given filepath (in bytes).
@@ -2309,7 +2349,7 @@ def det_size(path):
if os.path.exists(fullpath):
installsize += os.path.getsize(fullpath)
except OSError as err:
- _log.warn("Could not determine install size: %s" % err)
+ _log.warning("Could not determine install size: %s" % err)
return installsize
@@ -2336,7 +2376,7 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None):
# always consider $LM_LICENSE_FILE
lic_env_vars = ['LM_LICENSE_FILE']
- if isinstance(custom_env_vars, string_type):
+ if isinstance(custom_env_vars, str):
lic_env_vars.insert(0, custom_env_vars)
elif custom_env_vars is not None:
lic_env_vars = custom_env_vars + lic_env_vars
@@ -2629,7 +2669,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs):
:param force_in_dry_run: force running the command during dry run
:param kwargs: additional named arguments to pass down to copy_dir
"""
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
paths = [paths]
_log.info("Copying %d files & directories to %s", len(paths), target_path)
@@ -2647,17 +2687,17 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs):
raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path)
-def get_source_tarball_from_git(filename, targetdir, git_config):
+def get_source_tarball_from_git(filename, target_dir, git_config):
"""
Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it
- :param filename: name of the archive to save the code to (must be .tar.gz)
- :param targetdir: target directory where to save the archive to
+ :param filename: name of the archive file to save the code to (including extension)
+ :param target_dir: target directory where to save the archive to
:param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit
"""
# sanity check on git_config value being passed
if not isinstance(git_config, dict):
- raise EasyBuildError("Found unexpected type of value for 'git_config' argument: %s" % type(git_config))
+ raise EasyBuildError("Found unexpected type of value for 'git_config' argument: {type(git_config)}")
# Making a copy to avoid modifying the object with pops
git_config = git_config.copy()
@@ -2673,7 +2713,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
# input validation of git_config dict
if git_config:
- raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: %s", git_config)
+ raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: {git_config}")
if not repo_name:
raise EasyBuildError("repo_name not specified in git_config parameter")
@@ -2687,102 +2727,185 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
if not url:
raise EasyBuildError("url not specified in git_config parameter")
- if not filename.endswith('.tar.gz'):
- raise EasyBuildError("git_config currently only supports filename ending in .tar.gz")
-
# prepare target directory and clone repository
- mkdir(targetdir, parents=True)
- targetpath = os.path.join(targetdir, filename)
+ mkdir(target_dir, parents=True)
+
+ # compose base git command
+ git_cmd = 'git'
+ if extra_config_params is not None:
+ git_cmd_params = [f"-c {param}" for param in extra_config_params]
+ git_cmd += f" {' '.join(git_cmd_params)}"
# compose 'git clone' command, and run it
- if extra_config_params:
- git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params])
- else:
- git_cmd = 'git'
clone_cmd = [git_cmd, 'clone']
+ # checkout is done separately below for specific commits
+ clone_cmd.append('--no-checkout')
- if not keep_git_dir and not commit:
- # Speed up cloning by only fetching the most recent commit, not the whole history
- # When we don't want to keep the .git folder there won't be a difference in the result
- clone_cmd.extend(['--depth', '1'])
-
- if tag:
- clone_cmd.extend(['--branch', tag])
- if recursive:
- clone_cmd.append('--recursive')
- if recurse_submodules:
- clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
- else:
- # checkout is done separately below for specific commits
- clone_cmd.append('--no-checkout')
-
- clone_cmd.append('%s/%s.git' % (url, repo_name))
+ clone_cmd.append(f'{url}/{repo_name}.git')
if clone_into:
clone_cmd.append(clone_into)
tmpdir = tempfile.mkdtemp()
- cwd = change_dir(tmpdir)
- run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False)
+
+ run_shell_cmd(' '.join(clone_cmd), hidden=True, verbose_dry_run=True, work_dir=tmpdir)
# If the clone is done into a specified name, change repo_name
if clone_into:
repo_name = clone_into
+ repo_dir = os.path.join(tmpdir, repo_name)
+
+ # compose checkout command
+ checkout_cmd = [git_cmd, 'checkout']
# if a specific commit is asked for, check it out
if commit:
- checkout_cmd = [git_cmd, 'checkout', commit]
-
- if recursive or recurse_submodules:
- checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init'])
- if recursive:
- checkout_cmd.append('--recursive')
- if recurse_submodules:
- checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
-
- run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name)
-
- elif not build_option('extended_dry_run'):
- # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name
- # This doesn't make sense in dry-run mode as we don't have anything to check
- cmd = '%s describe --exact-match --tags HEAD' % git_cmd
- # Note: Disable logging to also disable the error handling in run_cmd
- (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name)
- if ec != 0 or tag not in out.splitlines():
- print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch'
- ' with the same name. You might want to alert the maintainers of %s about that issue.',
- tag, url, repo_name, repo_name)
- cmds = []
-
- if not keep_git_dir:
- # make the repo unshallow first;
- # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+
- # (first fetch seems to do nothing, unclear why)
- cmds.append('%s fetch --depth=2147483647 && git fetch --depth=2147483647' % git_cmd)
-
- cmds.append('%s checkout refs/tags/' % git_cmd + tag)
- # Clean all untracked files, e.g. from left-over submodules
- cmds.append('%s clean --force -d -x' % git_cmd)
- if recursive:
- cmds.append('%s submodule update --init --recursive' % git_cmd)
- elif recurse_submodules:
- cmds.append('%s submodule update --init ' % git_cmd)
- cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
- for cmd in cmds:
- run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)
-
- # create an archive and delete the git repo directory
- if keep_git_dir:
- tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
- else:
- tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
- run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False)
+ checkout_cmd.append(f"{commit}")
+ elif tag:
+ checkout_cmd.append(f"refs/tags/{tag}")
+
+ run_shell_cmd(' '.join(checkout_cmd), work_dir=repo_dir, hidden=True, verbose_dry_run=True)
+
+ if recursive or recurse_submodules:
+ submodule_cmd = [git_cmd, 'submodule', 'update', '--init']
+ if recursive:
+ submodule_cmd.append('--recursive')
+ if recurse_submodules:
+ submodule_pathspec = [f"':{submod_path}'" for submod_path in recurse_submodules]
+ submodule_cmd.extend(['--'] + submodule_pathspec)
+
+ run_shell_cmd(' '.join(submodule_cmd), work_dir=repo_dir, hidden=True, verbose_dry_run=True)
+
+ # Create archive
+ reproducible = not keep_git_dir # presence of .git directory renders repo unreproducible
+ archive_path = make_archive(repo_dir, archive_file=filename, archive_dir=target_dir, reproducible=reproducible)
# cleanup (repo_name dir does not exist in dry run mode)
- change_dir(cwd)
remove(tmpdir)
- return targetpath
+ return archive_path
+
+
+def make_archive(source_dir, archive_file=None, archive_dir=None, reproducible=True):
+ """
+ Create an archive file of the given directory
+ The format of the tarball is defined by the extension of the archive file name
+
+ :source_dir: string with path to directory to be archived
+ :archive_file: string with filename of archive
+ :archive_dir: string with path to directory to place the archive
+ :reproducible: make a tarball that is reproducible accross systems
+ - see https://reproducible-builds.org/docs/archives/
+ - requires uncompressed or LZMA compressed archive images
+ - gzip is currently not supported due to undeterministic data injected in its headers
+ see https://github.com/python/cpython/issues/112346
+
+ Default behaviour: reproducible tarball in .tar.xz
+ """
+ def reproducible_filter(tarinfo):
+ "Filter out system-dependent data from tarball"
+ # contents of '.git' subdir are inherently system dependent
+ if "/.git/" in tarinfo.name or tarinfo.name.endswith("/.git"):
+ return None
+ # set timestamp to epoch 0
+ tarinfo.mtime = 0
+ # reset file permissions by applying go+u,go-w
+ user_mode = tarinfo.mode & stat.S_IRWXU
+ group_mode = (user_mode >> 3) & ~stat.S_IWGRP # user mode without write
+ other_mode = group_mode >> 3 # same as group mode
+ tarinfo.mode = (tarinfo.mode & ~0o77) | group_mode | other_mode
+ # reset ownership to numeric UID/GID 0
+ # equivalent in GNU tar to 'tar --owner=0 --group=0 --numeric-owner'
+ tarinfo.uid = tarinfo.gid = 0
+ tarinfo.uname = tarinfo.gname = ""
+ return tarinfo
+
+ ext_compression_map = {
+ # taken from EXTRACT_CMDS
+ '.gtgz': 'gz',
+ '.tar.gz': 'gz',
+ '.tgz': 'gz',
+ '.tar.bz2': 'bz2',
+ '.tb2': 'bz2',
+ '.tbz': 'bz2',
+ '.tbz2': 'bz2',
+ '.tar.xz': 'xz',
+ '.txz': 'xz',
+ '.tar': '',
+ }
+ reproducible_compression = ['', 'xz']
+ default_ext = '.tar.xz'
+
+ if archive_file is None:
+ archive_file = os.path.basename(source_dir) + default_ext
+
+ try:
+ archive_ext = find_extension(archive_file)
+ except EasyBuildError:
+ if '.' in archive_file:
+ # archive filename has unknown extension (set for raise)
+ archive_ext = ''
+ else:
+ # archive filename has no extension, use default one
+ archive_ext = default_ext
+ archive_file += archive_ext
+
+ if archive_ext not in ext_compression_map:
+ # archive filename has unsupported extension
+ supported_exts = ', '.join(ext_compression_map)
+ raise EasyBuildError(
+ f"Unsupported archive format: {archive_file}. Supported tarball extensions: {supported_exts}"
+ )
+ compression = ext_compression_map[archive_ext]
+ _log.debug(f"Archive extension and compression: {archive_ext} in {compression}")
+
+ archive_path = archive_file if archive_dir is None else os.path.join(archive_dir, archive_file)
+
+ archive_specs = {
+ 'name': archive_path,
+ 'mode': f"w:{compression}",
+ 'format': tarfile.GNU_FORMAT,
+ 'encoding': "utf-8",
+ }
+
+ if reproducible:
+ if compression == 'xz':
+ # ensure a consistent compression level in reproducible tarballs with XZ
+ archive_specs['preset'] = 6
+ elif compression not in reproducible_compression:
+ # requested archive compression cannot be made reproducible
+ print_warning(
+ f"Can not create reproducible archive due to unsupported file compression ({compression}). "
+ "Please use XZ instead."
+ )
+ reproducible = False
+
+ archive_filter = reproducible_filter if reproducible else None
+
+ if build_option('extended_dry_run'):
+ # early return in dry run mode
+ dry_run_msg("Archiving '%s' into '%s'...", source_dir, archive_path)
+ return archive_path
+ _log.info("Archiving '%s' into '%s'...", source_dir, archive_path)
+
+ # TODO: replace with TarFile.add(recursive=True) when support for Python 3.6 drops
+ # since Python v3.7 tarfile automatically orders the list of files added to the archive
+ # see Tarfile.add documentation: https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.add
+ source_files = [source_dir]
+ # pathlib's glob includes hidden files
+ source_files.extend([str(filepath) for filepath in pathlib.Path(source_dir).glob("**/*")])
+ source_files.sort() # independent of locale
+
+ with tarfile.open(**archive_specs) as tar_archive:
+ for filepath in source_files:
+ # archive with target directory in its top level, remove any prefix in path
+ file_name = os.path.relpath(filepath, start=os.path.dirname(source_dir))
+ tar_archive.add(filepath, arcname=file_name, recursive=False, filter=archive_filter)
+ _log.debug("File/folder added to archive '%s': %s", archive_file, filepath)
+
+ _log.info("Archive '%s' created successfully", archive_file)
+
+ return archive_path
def move_file(path, target_path, force_in_dry_run=False):
@@ -2980,3 +3103,69 @@ def create_unused_dir(parent_folder, name):
set_gid_sticky_bits(path, recursive=True)
return path
+
+
+def get_first_non_existing_parent_path(path):
+ """
+ Get first directory that does not exist, starting at path and going up.
+ """
+ path = os.path.abspath(path)
+
+ non_existing_parent = None
+ while not os.path.exists(path):
+ non_existing_parent = path
+ path = os.path.dirname(path)
+
+ return non_existing_parent
+
+
+def create_non_existing_paths(paths, max_tries=10000):
+ """
+ Create directories with given paths (including the parent directories).
+ When a directory in the same location for any of the specified paths already exists,
+ then the suffix '_' is appended , with i iteratively picked between 0 and (max_tries-1),
+ until an index is found so that all required paths are non-existing.
+ All created directories have the same suffix.
+
+ :param paths: list of directory paths to be created
+ :param max_tries: maximum number of tries before failing
+ """
+ paths = [os.path.abspath(p) for p in paths]
+ for idx_path, path in enumerate(paths):
+ for idx_parent, parent in enumerate(paths):
+ if idx_parent != idx_path and is_parent_path(parent, path):
+ raise EasyBuildError(f"Path '{parent}' is a parent path of '{path}'.")
+
+ first_non_existing_parent_paths = [get_first_non_existing_parent_path(p) for p in paths]
+
+ non_existing_paths = paths
+ all_paths_created = False
+ suffix = -1
+ while suffix < max_tries and not all_paths_created:
+ tried_paths = []
+ if suffix >= 0:
+ non_existing_paths = [f'{p}_{suffix}' for p in paths]
+ try:
+ for path in non_existing_paths:
+ tried_paths.append(path)
+ # os.makedirs will raise OSError if directory already exists
+ os.makedirs(path)
+ all_paths_created = True
+ except OSError as err:
+ # Distinguish between error due to existing folder and anything else
+ if not os.path.exists(tried_paths[-1]):
+ raise EasyBuildError("Failed to create directory %s: %s", tried_paths[-1], err)
+ remove(tried_paths[:-1])
+ except BaseException as err:
+ remove(tried_paths)
+ raise err
+ suffix += 1
+
+ if not all_paths_created:
+ raise EasyBuildError(f"Exceeded maximum number of attempts ({max_tries}) to generate non-existing paths")
+
+ # set group ID and sticky bits, if desired
+ for path in first_non_existing_parent_paths:
+ set_gid_sticky_bits(path, recursive=True)
+
+ return non_existing_paths
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
index 6d4a1300fc..c03d92ad24 100644
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,6 +45,8 @@
import tempfile
import time
from datetime import datetime, timedelta
+from string import ascii_letters
+from urllib.request import HTTPError, URLError, urlopen
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR
@@ -52,12 +54,11 @@
from easybuild.framework.easyconfig.easyconfig import process_easyconfig
from easybuild.framework.easyconfig.parser import EasyConfigParser
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files
from easybuild.tools.filetools import det_patched_files, download_file, extract_file
from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file
-from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen
from easybuild.tools.systemtools import UNKNOWN, get_tool_version
from easybuild.tools.utilities import nub, only_if_module_is_available
from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions
@@ -207,7 +208,7 @@ def listdir(self, path):
return listing[1]
else:
self.log.warning("error: %s" % str(listing))
- raise EasyBuildError("Invalid response from github (I/O error)")
+ raise EasyBuildError("Invalid response from github (I/O error)", exit_code=EasyBuildExit.FAIL_GITHUB)
def walk(self, top=None, topdown=True):
"""
@@ -243,7 +244,7 @@ def read(self, path, api=True):
if not api:
outfile = tempfile.mkstemp()[1]
url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path])
- download_file(os.path.basename(path), url, outfile)
+ download_file(os.path.basename(path), url, outfile, trace=False)
return outfile
else:
obj = self.get_path(path).get(ref=self.branchname)[1]
@@ -314,9 +315,12 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs):
if status == 200:
_log.info("Put request successful: %s", data['message'])
elif status in [405, 409]:
- raise EasyBuildError("FAILED: %s", data['message'])
+ raise EasyBuildError("FAILED: %s", data['message'], exit_code=EasyBuildExit.FAIL_GITHUB)
else:
- raise EasyBuildError("FAILED: %s", data.get('message', "(unknown reason)"))
+ raise EasyBuildError(
+ "FAILED: %s", data.get('message', "(unknown reason)"),
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
_log.debug("get request result for %s: status: %d, data: %s", url.url, status, data)
return (status, data)
@@ -338,20 +342,23 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token=
status, data = github_api_get_request(lambda x: x.repos[account][repo].branches,
github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)",
- branch, account, repo, status, data)
+ raise EasyBuildError(
+ "Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)",
+ branch, account, repo, status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
res = None
for entry in data:
- if entry[u'name'] == branch:
+ if entry['name'] == branch:
res = entry['commit']['sha']
break
if res is None:
- error_msg = "No branch with name %s found in repo %s/%s" % (branch, account, repo)
+ error_msg = f"No branch with name {branch} found in repo {account}/{repo}"
if len(data) >= GITHUB_MAX_PER_PAGE:
- error_msg += "; only %d branches were checked (too many branches in %s/%s?)" % (len(data), account, repo)
- raise EasyBuildError(error_msg + ': ' + ', '.join([x[u'name'] for x in data]))
+ error_msg += f"; only {len(data)} branches were checked (too many branches in {account}/{repo}?)"
+ error_msg += ": " + ", ".join([x['name'] for x in data])
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_GITHUB)
return res
@@ -386,7 +393,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun
else:
error_msg = r"Specified commit SHA %s for downloading %s/%s is not valid, "
error_msg += "must be full SHA-1 (40 chars)"
- raise EasyBuildError(error_msg, commit, account, repo)
+ raise EasyBuildError(error_msg, commit, account, repo, exit_code=EasyBuildExit.VALUE_ERROR)
extracted_dir_name = '%s-%s' % (repo, commit)
base_name = '%s.tar.gz' % commit
@@ -396,7 +403,9 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun
base_name = '%s.tar.gz' % branch
latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user)
else:
- raise EasyBuildError("Either branch or commit should be specified in download_repo")
+ raise EasyBuildError(
+ "Either branch or commit should be specified in download_repo", exit_code=EasyBuildExit.VALUE_ERROR
+ )
expected_path = os.path.join(path, extracted_dir_name)
latest_sha_path = os.path.join(expected_path, 'latest-sha')
@@ -412,13 +421,16 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun
target_path = os.path.join(path, base_name)
_log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path))
- downloaded_path = download_file(base_name, url, target_path, forced=True)
+ downloaded_path = download_file(base_name, url, target_path, forced=True, trace=False)
if downloaded_path is None:
- raise EasyBuildError("Failed to download tarball for %s/%s commit %s", account, repo, commit)
+ raise EasyBuildError(
+ "Failed to download tarball for %s/%s commit %s", account, repo, commit,
+ exit_code=EasyBuildExit.FAIL_DOWNLOAD
+ )
else:
- _log.debug("%s downloaded to %s, extracting now" % (base_name, path))
+ _log.debug("%s downloaded to %s, extracting now", base_name, path)
- base_dir = extract_file(target_path, path, forced=True, change_into_dir=False)
+ base_dir = extract_file(target_path, path, forced=True, change_into_dir=False, trace=False)
extracted_path = os.path.join(base_dir, extracted_dir_name)
# check if extracted_path exists
@@ -428,7 +440,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun
error_msg += "at branch " + branch
elif commit:
error_msg += "at commit " + commit
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_EXTRACT)
write_file(latest_sha_path, latest_commit_sha, forced=True)
@@ -490,13 +502,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
if len(cands) == 1:
path = cands[0]
else:
- raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s",
- pr, extra_ec_paths)
+ raise EasyBuildError(
+ "Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
elif github_repo == GITHUB_EASYBLOCKS_REPO:
path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr)
else:
- raise EasyBuildError("Unknown repo: %s" % github_repo)
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
if path is None:
path = tempfile.mkdtemp()
@@ -512,7 +526,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
elif github_repo == GITHUB_EASYBLOCKS_REPO:
easyfiles = 'easyblocks'
else:
- raise EasyBuildError("Don't know how to fetch files from repo %s", github_repo)
+ raise EasyBuildError(
+ "Don't know how to fetch files from repo %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR
+ )
subdir = os.path.join('easybuild', easyfiles)
@@ -532,7 +548,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
# determine list of changed files via diff
diff_fn = os.path.basename(pr_data['diff_url'])
diff_filepath = os.path.join(path, diff_fn)
- download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True)
+ download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True, trace=False)
diff_txt = read_file(diff_filepath)
_log.debug("Diff for PR #%s:\n%s", pr, diff_txt)
@@ -568,7 +584,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
sha = pr_data['head']['sha']
full_url = URL_SEPARATOR.join([GITHUB_RAW, github_account, github_repo, sha, patched_file])
_log.info("Downloading %s from %s", fn, full_url)
- download_file(fn, full_url, path=os.path.join(path, fn), forced=True)
+ download_file(fn, full_url, path=os.path.join(path, fn), forced=True, trace=False)
final_path = path
@@ -586,7 +602,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
if os.path.exists(full_path):
files.append(full_path)
else:
- raise EasyBuildError("Couldn't find path to patched file %s", full_path)
+ raise EasyBuildError(
+ "Couldn't find path to patched file %s", full_path, exit_code=EasyBuildExit.OPTION_ERROR
+ )
if github_repo == GITHUB_EASYCONFIGS_REPO:
ver_file = os.path.join(final_path, 'setup.py')
@@ -665,29 +683,34 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
if len(cands) == 1:
path = cands[0]
else:
- raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s",
- commit, extra_ec_paths)
+ raise EasyBuildError(
+ "Failed to isolate path for commit %s from list of commit paths: %s",
+ commit, extra_ec_paths, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
else:
path = os.path.join(tempfile.gettempdir(), 'ecs_commit_' + commit)
elif github_repo == GITHUB_EASYBLOCKS_REPO:
path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit)
else:
- raise EasyBuildError("Unknown repo: %s" % github_repo)
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
# if no files are specified, determine which files are touched in commit
if not files:
diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff')
diff_fn = os.path.basename(diff_url)
diff_filepath = os.path.join(path, diff_fn)
- if download_file(diff_fn, diff_url, diff_filepath, forced=True):
+ if download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False):
diff_txt = read_file(diff_filepath)
_log.debug("Diff for commit %s:\n%s", commit, diff_txt)
files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
_log.debug("List of patched files for commit %s: %s", commit, files)
else:
- raise EasyBuildError("Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo)
+ raise EasyBuildError(
+ "Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# download tarball for specific commit
repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account)
@@ -697,7 +720,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
elif github_repo == GITHUB_EASYBLOCKS_REPO:
files_subdir = 'easybuild/easyblocks/'
else:
- raise EasyBuildError("Unknown repo: %s" % github_repo)
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
# symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path
mkdir(path, parents=True)
@@ -776,7 +799,9 @@ def create_gist(txt, fn, descr=None, github_user=None, github_token=None):
status, data = g.gists.post(body=body)
if status != HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data)
+ raise EasyBuildError(
+ "Failed to create gist; status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return data['html_url']
@@ -791,7 +816,9 @@ def delete_gist(gist_id, github_user=None, github_token=None):
status, data = gh.gists[gist_id].delete()
if status != HTTP_STATUS_NO_CONTENT:
- raise EasyBuildError("Failed to delete gist with ID %s: status %s, data: %s", status, data)
+ raise EasyBuildError(
+ "Failed to delete gist with ID %s: status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCONFIGS_REPO, github_user=None):
@@ -800,7 +827,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO
try:
issue = int(issue)
except ValueError as err:
- raise EasyBuildError("Failed to parse specified pull request number '%s' as an int: %s; ", issue, err)
+ raise EasyBuildError(
+ "Failed to parse specified pull request number '%s' as an int: %s; ", issue, err,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
dry_run = build_option('dry_run') or build_option('extended_dry_run')
@@ -817,7 +847,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO
status, data = pr_url.comments.post(body={'body': txt})
if not status == HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data)
+ raise EasyBuildError(
+ "Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def init_repo(path, repo_name, silent=False):
@@ -843,13 +876,15 @@ def init_repo(path, repo_name, silent=False):
workrepo = git.Repo(workdir)
workrepo.clone(repo_path)
except GitCommandError as err:
- raise EasyBuildError("Failed to clone git repo at %s: %s", workdir, err)
+ raise EasyBuildError(
+ "Failed to clone git repo at %s: %s", workdir, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# initalize repo in repo_path
try:
repo = git.Repo.init(repo_path)
except GitCommandError as err:
- raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err)
+ raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err, exit_code=EasyBuildExit.FAIL_GITHUB)
_log.debug("temporary git working directory ready at %s", repo_path)
@@ -869,7 +904,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
_log.debug("Cloning from %s", github_url)
if target_account is None:
- raise EasyBuildError("target_account not specified in setup_repo_from!")
+ raise EasyBuildError("target_account not specified in setup_repo_from!", exit_code=EasyBuildExit.OPTION_ERROR)
# salt to use for names of remotes/branches that are created
salt = ''.join(random.choice(ascii_letters) for _ in range(5))
@@ -878,7 +913,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
origin = git_repo.create_remote(remote_name, github_url)
if not origin.exists():
- raise EasyBuildError("%s does not exist?", github_url)
+ raise EasyBuildError("%s does not exist?", github_url, exit_code=EasyBuildExit.FAIL_GITHUB)
# git fetch
# can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work
@@ -887,21 +922,32 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
try:
res = origin.fetch()
except GitCommandError as err:
- raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err)
+ raise EasyBuildError(
+ "Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if res:
if res[0].flags & res[0].ERROR:
- raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note)
+ raise EasyBuildError(
+ "Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
else:
_log.debug("Fetched branch '%s' from remote %s (note: %s)", branch_name, origin, res[0].note)
else:
- raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin)
+ raise EasyBuildError(
+ "Fetching branch '%s' from remote %s failed: empty result", branch_name, origin,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# git checkout -b ; git pull
try:
origin_branch = getattr(origin.refs, branch_name)
except AttributeError:
- raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url)
+ raise EasyBuildError(
+ "Branch '%s' not found at %s", branch_name, github_url, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
_log.debug("Checking out branch '%s' from remote %s", branch_name, github_url)
try:
@@ -912,7 +958,10 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
try:
origin_branch.checkout(b=alt_branch, force=True)
except GitCommandError as err:
- raise EasyBuildError("Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err)
+ raise EasyBuildError(
+ "Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return remote_name
@@ -948,7 +997,7 @@ def setup_repo(git_repo, target_account, target_repo, branch_name, silent=False,
if res:
return res
else:
- raise EasyBuildError('\n'.join(errors))
+ raise EasyBuildError('\n'.join(errors), exit_code=EasyBuildExit.FAIL_GITHUB)
@only_if_module_is_available('git', pkgname='GitPython')
@@ -980,14 +1029,20 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
ec_paths.append(path)
if non_existing_paths:
- raise EasyBuildError("One or more non-existing paths specified: %s", ', '.join(non_existing_paths))
+ raise EasyBuildError(
+ "One or more non-existing paths specified: %s", ', '.join(non_existing_paths),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if not any(paths.values()):
- raise EasyBuildError("No paths specified")
+ raise EasyBuildError("No paths specified", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_repo = det_pr_target_repo(paths)
if pr_target_repo is None:
- raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!")
+ raise EasyBuildError(
+ "Failed to determine target repository, please specify it via --pr-target-repo!",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# initialize repository
git_working_dir = tempfile.mkdtemp(prefix='git-working-dir')
@@ -995,7 +1050,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
repo_path = os.path.join(git_working_dir, pr_target_repo)
if pr_target_repo not in [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO]:
- raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo)
+ raise EasyBuildError(
+ "Don't know how to create/update a pull request to the %s repository", pr_target_repo,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if start_account is None:
start_account = build_option('pr_target_account')
@@ -1006,7 +1064,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
target_account = build_option('github_org') or build_option('github_user')
if target_account is None:
- raise EasyBuildError("--github-org or --github-user must be specified!")
+ raise EasyBuildError(
+ "--github-org or --github-user must be specified!", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# if branch to start from is specified, we're updating an existing PR
start_branch = build_option('pr_target_branch')
@@ -1037,8 +1097,16 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']):
commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo'])
else:
+ msg = ''
+ modified_files = [os.path.basename(p) for new, p in zip(file_info['new'], file_info['paths_in_repo'])
+ if not new]
+ if modified_files:
+ msg += '\nModified: ' + ', '.join(modified_files)
+ if paths['files_to_delete']:
+ msg += '\nDeleted: ' + ', '.join(paths['files_to_delete'])
raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when "
- "modifying/deleting files or targeting the framework repo.")
+ "modifying/deleting files or targeting the framework repo." + msg,
+ exit_code=EasyBuildExit.OPTION_ERROR)
# figure out to which software name patches relate, and copy them to the right place
if paths['patch_files']:
@@ -1059,7 +1127,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
if len(hits) == 1:
deleted_paths.append(hits[0])
else:
- raise EasyBuildError("Path doesn't exist or file to delete isn't found in target branch: %s", fn)
+ raise EasyBuildError(
+ "Path doesn't exist or file to delete isn't found in target branch: %s", fn,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
dep_info = {
'ecs': [],
@@ -1078,8 +1149,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
# only consider new easyconfig files for dependencies (not updated ones)
for idx in range(len(all_dep_info['ecs'])):
if all_dep_info['new'][idx]:
- for key in dep_info:
- dep_info[key].append(all_dep_info[key][idx])
+ for key, info in dep_info.items():
+ info.append(all_dep_info[key][idx])
# checkout target branch
if pr_branch is None:
@@ -1116,8 +1187,11 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
diff_stat = git_repo.git.diff(cached=True, stat=True)
if not diff_stat:
- raise EasyBuildError("No changed files found when comparing to current develop branch. "
- "Refused to make empty pull request.")
+ raise EasyBuildError(
+ f"No changed files found when comparing to current {start_branch} branch. "
+ "Refused to make empty pull request.",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# commit
git_repo.index.commit(commit_msg)
@@ -1148,7 +1222,9 @@ def create_remote(git_repo, account, repo, https=False):
try:
remote = git_repo.create_remote(remote_name, github_url)
except GitCommandError as err:
- raise EasyBuildError("Failed to create remote %s for %s: %s", remote_name, github_url, err)
+ raise EasyBuildError(
+ "Failed to create remote %s for %s: %s", remote_name, github_url, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return remote
@@ -1163,7 +1239,9 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch):
:param branch: name of branch to push
"""
if target_account is None:
- raise EasyBuildError("target_account not specified in push_branch_to_github!")
+ raise EasyBuildError(
+ "target_account not specified in push_branch_to_github!", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# push to GitHub
remote = create_remote(git_repo, target_account, target_repo)
@@ -1180,17 +1258,24 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch):
try:
res = remote.push(branch)
except GitCommandError as err:
- raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err)
+ raise EasyBuildError(
+ "Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if res:
if res[0].ERROR & res[0].flags:
- raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s",
- branch, remote, github_url, res[0].summary)
+ raise EasyBuildError(
+ "Pushing branch '%s' to remote %s (%s) failed: %s", branch, remote, github_url, res[0].summary,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
else:
_log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary)
else:
- raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result",
- branch, remote, github_url)
+ raise EasyBuildError(
+ "Pushing branch '%s' to remote %s (%s) failed: empty result", branch, remote, github_url,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def is_patch_for(patch_name, ec):
@@ -1247,7 +1332,10 @@ def det_patch_specs(patch_paths, file_info, ec_dirs):
patch_specs.append((patch_path, soft_name))
else:
# still nothing found
- raise EasyBuildError("Failed to determine software name to which patch file %s relates", patch_path)
+ raise EasyBuildError(
+ "Failed to determine software name to which patch file %s relates", patch_path,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
return patch_specs
@@ -1514,7 +1602,7 @@ def close_pr(pr, motivation_msg=None):
"""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --close-pr")
+ raise EasyBuildError("GitHub user must be specified to use --close-pr", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -1522,7 +1610,10 @@ def close_pr(pr, motivation_msg=None):
pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True)
if pr_data['state'] == GITHUB_STATE_CLOSED:
- raise EasyBuildError("PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo)
+ raise EasyBuildError(
+ "PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
pr_owner = pr_data['user']['login']
msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_owner)
@@ -1539,8 +1630,10 @@ def close_pr(pr, motivation_msg=None):
possible_reasons = reasons_for_closing(pr_data)
if not possible_reasons:
- raise EasyBuildError("No reason specified and none found from PR data, "
- "please use --close-pr-reasons or --close-pr-msg")
+ raise EasyBuildError(
+ "No reason specified and none found from PR data, please use --close-pr-reasons or --close-pr-msg",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
motivation_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in possible_reasons])
print_msg("\nNo reason specified but found possible reasons: %s.\n" % motivation_msg, prefix=False)
@@ -1559,18 +1652,26 @@ def close_pr(pr, motivation_msg=None):
else:
github_token = fetch_github_token(github_user)
if github_token is None:
- raise EasyBuildError("GitHub token for user '%s' must be available to use --close-pr", github_user)
+ raise EasyBuildError(
+ "GitHub token for user '%s' must be available to use --close-pr", github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
pull_url = g.repos[pr_target_account][pr_target_repo].pulls[pr]
body = {'state': 'closed'}
status, data = pull_url.post(body=body)
if not status == HTTP_STATUS_OK:
- raise EasyBuildError("Failed to close PR #%s; status %s, data: %s", pr, status, data)
+ raise EasyBuildError(
+ "Failed to close PR #%s; status %s, data: %s", pr, status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if reopen:
body = {'state': 'open'}
status, data = pull_url.post(body=body)
if not status == HTTP_STATUS_OK:
- raise EasyBuildError("Failed to reopen PR #%s; status %s, data: %s", pr, status, data)
+ raise EasyBuildError(
+ "Failed to reopen PR #%s; status %s, data: %s", pr, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def list_prs(params, per_page=GITHUB_MAX_PER_PAGE, github_user=None):
@@ -1606,7 +1707,7 @@ def merge_pr(pr):
"""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --merge-pr")
+ raise EasyBuildError("GitHub user must be specified to use --merge-pr", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -1618,16 +1719,16 @@ def merge_pr(pr):
msg += "\nPR title: %s\n\n" % pr_data['title']
print_msg(msg, prefix=False)
if pr_data['user']['login'] == github_user:
- raise EasyBuildError("Please do not merge your own PRs!")
+ raise EasyBuildError("Please do not merge your own PRs!", exit_code=EasyBuildExit.OPTION_ERROR)
force = build_option('force')
dry_run = build_option('dry_run') or build_option('extended_dry_run')
if not dry_run:
if pr_data['merged']:
- raise EasyBuildError("This PR is already merged.")
+ raise EasyBuildError("This PR is already merged.", exit_code=EasyBuildExit.OPTION_ERROR)
elif pr_data['state'] == GITHUB_STATE_CLOSED:
- raise EasyBuildError("This PR is closed.")
+ raise EasyBuildError("This PR is closed.", exit_code=EasyBuildExit.OPTION_ERROR)
def merge_url(gh):
"""Utility function to fetch merge URL for a specific PR."""
@@ -1692,7 +1793,7 @@ def post_pr_labels(pr, labels):
pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr]
try:
- status, data = pr_url.labels.post(body=labels)
+ status, _ = pr_url.labels.post(body=labels)
if status == HTTP_STATUS_OK:
print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False)
return True
@@ -1711,7 +1812,10 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH):
"""
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
if pr_target_repo != GITHUB_EASYCONFIGS_REPO:
- raise EasyBuildError("Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet")
+ raise EasyBuildError(
+ "Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
tmpdir = tempfile.mkdtemp()
@@ -1819,11 +1923,17 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
# fetch GitHub token (required to perform actions on GitHub)
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to open a pull request")
+ raise EasyBuildError(
+ "GitHub user must be specified to open a pull request",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
github_token = fetch_github_token(github_user)
if github_token is None:
- raise EasyBuildError("GitHub token for user '%s' must be available to open a pull request", github_user)
+ raise EasyBuildError(
+ "GitHub token for user '%s' must be available to open a pull request", github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# GitHub organisation or GitHub user where branch is located
github_account = build_option('github_org') or github_user
@@ -1885,7 +1995,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
print_msg('\n'.join(msg), log=_log)
else:
- raise EasyBuildError("No changes in '%s' branch compared to current 'develop' branch!", branch_name)
+ raise EasyBuildError(
+ f"No changes in '{branch_name}' branch compared to current '{pr_target_branch}' branch!",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# copy repo while target branch is still checked out
tmpdir = tempfile.mkdtemp()
@@ -1918,8 +2031,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
title = "new easyblock%s for %s" % (plural, (', '.join(file_info['eb_names'])))
if title is None:
- raise EasyBuildError("Don't know how to make a PR title for this PR. "
- "Please include a title (use --pr-title)")
+ raise EasyBuildError(
+ "Don't know how to make a PR title for this PR. Please include a title (use --pr-title)",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
full_descr = "(created using `eb --new-pr`)\n"
if descr is not None:
@@ -1956,7 +2071,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
}
status, data = pulls_url.post(body=body)
if not status == HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data)
+ raise EasyBuildError(
+ "Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False)
@@ -1991,11 +2109,11 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None):
patch = patch[0]
elif isinstance(patch, dict):
patch_info = {}
- for key in patch.keys():
- patch_info[key] = patch[key]
- if 'name' not in patch_info.keys():
- raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
- str(patch))
+ for key, cur_patch in patch.items():
+ patch_info[key] = cur_patch
+ if 'name' not in patch_info:
+ msg = f"Wrong patch spec '{patch}', when using a dict 'name' entry must be supplied"
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR)
patch = patch_info['name']
if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path),
@@ -2014,7 +2132,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub username (--github-user) must be specified!")
+ raise EasyBuildError("GitHub username (--github-user) must be specified!", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
if pr_target_repo is None:
@@ -2086,7 +2204,10 @@ def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None)
commit_msg = build_option('pr_commit_msg')
if commit_msg is None:
- raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when using --update-pr")
+ raise EasyBuildError(
+ "A meaningful commit message must be specified via --pr-commit-msg when using --update-pr",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if github_account is None:
github_account = build_option('github_user') or build_option('github_org')
@@ -2117,7 +2238,10 @@ def update_pr(pr_id, paths, ecs, commit_msg=None):
pr_target_repo = det_pr_target_repo(paths)
if pr_target_repo is None:
- raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!")
+ raise EasyBuildError(
+ "Failed to determine target repository, please specify it via --pr-target-repo!",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo)
@@ -2179,7 +2303,9 @@ def check_github():
print_msg("OK\n", log=_log, prefix=False)
else:
print_msg("FAIL (%s)", ', '.join(online_state), log=_log, prefix=False)
- raise EasyBuildError("checking status of GitHub integration must be done online")
+ raise EasyBuildError(
+ "checking status of GitHub integration must be done online", exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# GitHub user
print_msg("* GitHub user...", log=_log, prefix=False, newline=False)
@@ -2356,7 +2482,7 @@ def check_github():
msg = '\n'.join([
'',
"One or more checks FAILed, GitHub configuration not fully complete!",
- "See http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#configuration for help.",
+ "See https://docs.easybuild.io/integration-with-github/#github_configuration for help.",
'',
])
print_msg(msg, log=_log, prefix=False)
@@ -2413,7 +2539,9 @@ def install_github_token(github_user, silent=False):
:param silent: keep quiet (don't print any messages)
"""
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to install GitHub token")
+ raise EasyBuildError(
+ "GitHub user must be specified to install GitHub token", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# check if there's a token available already
current_token = fetch_github_token(github_user)
@@ -2424,8 +2552,10 @@ def install_github_token(github_user, silent=False):
msg = "WARNING: overwriting installed token '%s' for user '%s'..." % (current_token, github_user)
print_msg(msg, prefix=False, silent=silent)
else:
- raise EasyBuildError("Installed token '%s' found for user '%s', not overwriting it without --force",
- current_token, github_user)
+ raise EasyBuildError(
+ "Installed token '%s' found for user '%s', not overwriting it without --force",
+ current_token, github_user, exit_code=EasyBuildExit.OPTION_ERROR
+ )
# get token to install
token = getpass.getpass(prompt="Token: ").strip()
@@ -2436,7 +2566,10 @@ def install_github_token(github_user, silent=False):
if valid_token:
print_msg("Token seems to be valid, installing it.", prefix=False, silent=silent)
else:
- raise EasyBuildError("Token validation failed, not installing it. Please verify your token and try again.")
+ raise EasyBuildError(
+ "Token validation failed, not installing it. Please verify your token and try again.",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# install token
keyring.set_password(KEYRING_GITHUB_TOKEN, github_user, token)
@@ -2510,7 +2643,7 @@ def find_easybuild_easyconfig(github_user=None):
if file_versions:
fn = sorted(file_versions)[-1][1]
else:
- raise EasyBuildError("Couldn't find any EasyBuild easyconfigs")
+ raise EasyBuildError("Couldn't find any EasyBuild easyconfigs", exit_code=EasyBuildExit.MISSING_EASYCONFIG)
eb_file = os.path.join(eb_parent_path, fn)
return eb_file
@@ -2537,8 +2670,11 @@ def check_suites_url(gh):
# first check combined commit status (set by e.g. Travis CI)
status, commit_status_data = github_api_get_request(commit_status_url, github_user)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get status of commit %s from %s/%s (status: %d %s)",
- commit_sha, account, repo, status, commit_status_data)
+ raise EasyBuildError(
+ "Failed to get status of commit %s from %s/%s (status: %d %s)",
+ commit_sha, account, repo, status, commit_status_data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
commit_status_count = commit_status_data['total_count']
combined_commit_status = commit_status_data['state']
@@ -2586,7 +2722,9 @@ def check_suites_url(gh):
break
else:
app_name = check_suite_data.get('app', {}).get('name', 'UNKNOWN')
- raise EasyBuildError("Unknown check suite status set by %s: '%s'", app_name, status)
+ raise EasyBuildError(
+ "Unknown check suite status set by %s: '%s'", app_name, status, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return result
@@ -2604,13 +2742,16 @@ def pr_url(gh):
try:
status, pr_data = github_api_get_request(pr_url, github_user, **parameters)
except HTTPError as err:
- raise EasyBuildError("Failed to get data for PR #%d from %s/%s (%s)\n"
- "Please check PR #, account and repo.",
- pr, pr_target_account, pr_target_repo, err)
+ raise EasyBuildError(
+ "Failed to get data for PR #%d from %s/%s (%s)\nPlease check PR #, account and repo.",
+ pr, pr_target_account, pr_target_repo, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, pr_data)
+ raise EasyBuildError(
+ "Failed to get data for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, pr_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if full:
# also fetch status of last commit
@@ -2624,8 +2765,10 @@ def comments_url(gh):
status, comments_data = github_api_get_request(comments_url, github_user, **parameters)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, comments_data)
+ raise EasyBuildError(
+ "Failed to get comments for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, comments_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
pr_data['issue_comments'] = comments_data
# also fetch reviews
@@ -2635,8 +2778,10 @@ def reviews_url(gh):
status, reviews_data = github_api_get_request(reviews_url, github_user, **parameters)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, reviews_data)
+ raise EasyBuildError(
+ "Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, reviews_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
pr_data['reviews'] = reviews_data
return pr_data, pr_url
@@ -2683,7 +2828,9 @@ def sync_pr_with_develop(pr_id):
"""Sync pull request with specified ID with current develop branch."""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --sync-pr-with-develop")
+ raise EasyBuildError(
+ "GitHub user must be specified to use --sync-pr-with-develop", exit_code=EasyBuildExit.OPTION_ERROR
+ )
target_account = build_option('pr_target_account')
target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -2706,7 +2853,9 @@ def sync_branch_with_develop(branch_name):
"""Sync branch with specified name with current develop branch."""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --sync-branch-with-develop")
+ raise EasyBuildError(
+ "GitHub user must be specified to use --sync-branch-with-develop", exit_code=EasyBuildExit.OPTION_ERROR
+ )
target_account = build_option('pr_target_account')
target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py
index f018c40607..0165717b39 100644
--- a/easybuild/tools/hooks.py
+++ b/easybuild/tools/hooks.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2017-2024 Ghent University
+# Copyright 2017-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,9 +33,9 @@
import os
from easybuild.base import fancylogger
-from easybuild.tools.py2vs3 import load_source
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option
+from importlib.util import spec_from_file_location, module_from_spec
_log = fancylogger.getLogger('hooks', fname=False)
@@ -44,6 +44,7 @@
CLEANUP_STEP = 'cleanup'
CONFIGURE_STEP = 'configure'
EXTENSIONS_STEP = 'extensions'
+EXTRACT_STEP = 'extract'
FETCH_STEP = 'fetch'
INSTALL_STEP = 'install'
MODULE_STEP = 'module'
@@ -55,7 +56,6 @@
PREPARE_STEP = 'prepare'
READY_STEP = 'ready'
SANITYCHECK_STEP = 'sanitycheck'
-SOURCE_STEP = 'source'
TEST_STEP = 'test'
TESTCASES_STEP = 'testcases'
@@ -67,6 +67,7 @@
END = 'end'
CANCEL = 'cancel'
+CRASH = 'crash'
FAIL = 'fail'
RUN_SHELL_CMD = 'run_shell_cmd'
@@ -76,7 +77,7 @@
HOOK_SUFF = '_hook'
# list of names for steps in installation procedure (in order of execution)
-STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP,
+STEP_NAMES = [FETCH_STEP, READY_STEP, EXTRACT_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP,
INSTALL_STEP, EXTENSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
@@ -107,6 +108,7 @@
POST_PREF + BUILD_AND_INSTALL_LOOP,
END,
CANCEL,
+ CRASH,
FAIL,
PRE_PREF + RUN_SHELL_CMD,
POST_PREF + RUN_SHELL_CMD,
@@ -118,6 +120,14 @@
_cached_hooks = {}
+def load_source(filename, path):
+ """Load file as Python module"""
+ spec = spec_from_file_location(filename, path)
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
def load_hooks(hooks_path):
"""Load defined hooks (if any)."""
diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py
index 0171d74f10..96e5bfd66c 100644
--- a/easybuild/tools/include.py
+++ b/easybuild/tools/include.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,7 +37,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import expand_glob_paths, read_file, symlink
+from easybuild.tools.filetools import expand_glob_paths, read_file, symlink, EASYBLOCK_CLASS_PREFIX
# these are imported just to we can reload them later
import easybuild.tools.module_naming_scheme
import easybuild.toolchains
@@ -157,7 +157,8 @@ def verify_imports(pymods, pypkg, from_path):
def is_software_specific_easyblock(module):
"""Determine whether Python module at specified location is a software-specific easyblock."""
- return bool(re.search(r'^class EB_.*\(.*\):\s*$', read_file(module), re.M))
+ # All software-specific easyblocks start with the prefix and derive from another class, at least EasyBlock
+ return bool(re.search(r"^class %s[^(:]+\([^)]+\):\s*$" % EASYBLOCK_CLASS_PREFIX, read_file(module), re.M))
def include_easyblocks(tmpdir, paths):
@@ -198,10 +199,8 @@ def include_easyblocks(tmpdir, paths):
# hard inject location to included (generic) easyblocks into Python search path
# only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py
- new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')
- easybuild.easyblocks.__path__.insert(0, new_path)
- new_path = os.path.join(new_path, 'generic')
- easybuild.easyblocks.generic.__path__.insert(0, new_path)
+ easybuild.easyblocks.__path__.insert(0, easyblocks_dir)
+ easybuild.easyblocks.generic.__path__.insert(0, os.path.join(easyblocks_dir, 'generic'))
# sanity check: verify that included easyblocks can be imported (from expected location)
for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py
index 50ee563083..771331118a 100644
--- a/easybuild/tools/jenkins.py
+++ b/easybuild/tools/jenkins.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py
index 1219883740..9b372c5848 100644
--- a/easybuild/tools/job/backend.py
+++ b/easybuild/tools/job/backend.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,6 +32,7 @@
"""
from abc import ABCMeta, abstractmethod
+from types import SimpleNamespace
from easybuild.base import fancylogger
from easybuild.tools.config import get_job_backend
@@ -69,7 +70,7 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None):
See the `Job`:class: constructor for an explanation of what
the arguments are.
"""
- pass
+ return SimpleNamespace()
@abstractmethod
def queue(self, job, dependencies=frozenset()):
diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py
index f6b57e0f2c..b912490d1f 100644
--- a/easybuild/tools/job/gc3pie.py
+++ b/easybuild/tools/job/gc3pie.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
# Copyright 2015 S3IT, University of Zurich
#
# This file is part of EasyBuild,
@@ -104,6 +104,10 @@ def __init__(self, *args, **kwargs):
def _check_version(self):
"""Check whether GC3Pie version complies with required version."""
+ deprecation_msg = "The GC3Pie job backend is no longer maintained and will be removed"
+ deprecation_msg += ", please use a different job backend"
+ _log.deprecated(deprecation_msg, '6.0')
+
try:
from pkg_resources import get_distribution, DistributionNotFound
pkg = get_distribution('gc3pie')
diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py
index bf90b57063..222479bd29 100644
--- a/easybuild/tools/job/pbs_python.py
+++ b/easybuild/tools/job/pbs_python.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,6 +39,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
@@ -320,8 +321,8 @@ def _submit(self):
self.log.debug("Job hold attributes: %s" % hold_attributes[0].value)
# add a bunch of variables (added by qsub)
- # also set PBS_O_WORKDIR to os.getcwd()
- os.environ.setdefault('WORKDIR', os.getcwd())
+ # also set PBS_O_WORKDIR to current working dir
+ os.environ.setdefault('WORKDIR', get_cwd())
defvars = ['MAIL', 'HOME', 'PATH', 'SHELL', 'WORKDIR']
pbsvars = ["PBS_O_%s=%s" % (x, os.environ.get(x, 'NOTFOUND_%s' % x)) for x in defvars]
diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py
index 2666cbd6af..28d39d278e 100644
--- a/easybuild/tools/job/slurm.py
+++ b/easybuild/tools/job/slurm.py
@@ -1,5 +1,5 @@
##
-# Copyright 2018-2024 Ghent University
+# Copyright 2018-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,7 +37,7 @@
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.filetools import which
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
_log = fancylogger.getLogger('slurm', fname=False)
@@ -78,8 +78,8 @@ def __init__(self, *args, **kwargs):
def _check_version(self):
"""Check whether version of Slurm complies with required version."""
- (out, _) = run_cmd("sbatch --version", trace=False)
- slurm_ver = out.strip().split(' ')[-1]
+ res = run_shell_cmd("sbatch --version", hidden=True)
+ slurm_ver = res.output.strip().split(' ')[-1]
self.log.info("Found Slurm version %s", slurm_ver)
if LooseVersion(slurm_ver) < LooseVersion(self.REQ_VERSION):
@@ -116,16 +116,16 @@ def queue(self, job, dependencies=frozenset()):
else:
submit_cmd += ' --%s "%s"' % (key, job.job_specs[key])
- (out, _) = run_cmd(submit_cmd, trace=False)
+ cmd_res = run_shell_cmd(submit_cmd, hidden=True)
jobid_regex = re.compile("^Submitted batch job (?P[0-9]+)")
- res = jobid_regex.search(out)
- if res:
- job.jobid = res.group('jobid')
+ regex_res = jobid_regex.search(cmd_res.output)
+ if regex_res:
+ job.jobid = regex_res.group('jobid')
self.log.info("Job submitted, got job ID %s", job.jobid)
else:
- raise EasyBuildError("Failed to determine job ID from output of submission command: %s", out)
+ raise EasyBuildError("Failed to determine job ID from output of submission command: %s", cmd_res.output)
self._submitted.append(job)
@@ -142,7 +142,7 @@ def complete(self):
job_ids.append(job.jobid)
if job_ids:
- run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False)
+ run_shell_cmd("scontrol release %s" % ' '.join(job_ids), hidden=True)
submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted])
print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log)
diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py
index 1855fee74a..a454912ed0 100644
--- a/easybuild/tools/loose_version.py
+++ b/easybuild/tools/loose_version.py
@@ -1,21 +1,18 @@
-# This file contains the LooseVersion class based on the class with the same name
-# as present in Python 3.7.4 distutils.
-# The original class is licensed under the Python Software Foundation License Version 2.
-# It was slightly simplified as needed to make it shorter and easier to read.
-# In particular the following changes were made:
-# - Subclass object directly instead of abstract Version class
-# - Fully init the class in the constructor removing the parse method
-# - Always set self.vstring and self.version
-# - Shorten the comparison operators as the NotImplemented case doesn't apply anymore
-# - Changes to documentation and formatting
+"""
+This file contains the LooseVersion class based on the class with the same name
+as present in Python 3.7.4 distutils.
+The original class is licensed under the Python Software Foundation License Version 2.
+It was slightly simplified as needed to make it shorter and easier to read.
+In particular the following changes were made:
+- Subclass object directly instead of abstract Version class
+- Fully init the class in the constructor removing the parse method
+- Always set self.vstring and self.version
+- Shorten the comparison operators as the NotImplemented case doesn't apply anymore
+- Changes to documentation and formatting
+"""
import re
-# Modified: Make this compatible with Python 2
-try:
- from itertools import zip_longest
-except ImportError:
- # Python 2
- from itertools import izip_longest as zip_longest
+from itertools import zip_longest
class LooseVersion(object):
@@ -53,6 +50,22 @@ def version(self):
"""Readonly access to the parsed version (list or None)"""
return self._version
+ def is_prerelease(self, other, markers):
+ """Check if this is a prerelease of other
+
+ Markers is a list of strings that denote a prerelease
+ """
+ if isinstance(other, str):
+ vstring = other
+ else:
+ vstring = other._vstring
+ if self._vstring.startswith(vstring):
+ prerelease = self._vstring[len(vstring):]
+ for marker in markers:
+ if prerelease.startswith(marker):
+ return True
+ return False
+
def __str__(self):
return self._vstring
@@ -64,17 +77,19 @@ def _cmp(self, other):
if isinstance(other, str):
other = LooseVersion(other)
- # Modified: Behave the same in Python 2 & 3 when parts are of different types
- # Taken from https://bugs.python.org/issue14894
- for i, j in zip_longest(self.version, other.version, fillvalue=''):
- if not type(i) is type(j):
+ # Modified: Use string comparison for different types and fill with zeroes/empty strings
+ # Based on https://bugs.python.org/issue14894
+ for i, j in zip_longest(self.version, other.version):
+ if i is None:
+ i = 0 if isinstance(j, int) else ''
+ elif j is None:
+ j = 0 if isinstance(i, int) else ''
+ elif not type(i) is type(j):
i = str(i)
j = str(j)
- if i == j:
- continue
- elif i < j:
+ if i < j:
return -1
- else: # i > j
+ if i > j:
return 1
return 0
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 5704d24cb0..eebafb3c4e 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,19 +36,20 @@
* Damian Alvarez (Forschungszentrum Juelich GmbH)
"""
import copy
+import itertools
import os
import re
import tempfile
+from collections import defaultdict
from contextlib import contextmanager
-from easybuild.tools import LooseVersion
from textwrap import wrap
from easybuild.base import fancylogger
+from easybuild.tools import LooseVersion
from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.config import build_option, get_module_syntax, install_path
from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import get_subclasses, nub, quote_str
@@ -154,7 +155,7 @@ def start_module_creation(self):
raise EasyBuildError('Module creation already in process. '
'You cannot create multiple modules at the same time!')
# Mapping of keys/env vars to paths already added
- self.added_paths_per_key = dict()
+ self.added_paths_per_key = defaultdict(set)
txt = self.MODULE_SHEBANG
if txt:
txt += '\n'
@@ -206,16 +207,20 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
return os.path.join(mod_path, mod_path_suffix)
- def _filter_paths(self, key, paths):
- """Filter out paths already added to key and return the remaining ones"""
+ def _filter_paths(self, key, paths, warn_exists=True):
+ """
+ Filter out paths already added to key and return the remaining ones
+
+ :param warn_exists: Show a warning for paths already added to the key
+ """
if self.added_paths_per_key is None:
# For compatibility this is only a warning for now and we don't filter any paths
print_warning('Module creation has not been started. Call start_module_creation first!')
return paths
- added_paths = self.added_paths_per_key.setdefault(key, set())
+ added_paths = self.added_paths_per_key[key]
# paths can be a string
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
if paths in added_paths:
filtered_paths = None
else:
@@ -227,15 +232,17 @@ def _filter_paths(self, key, paths):
paths = list(paths)
filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
if filtered_paths != paths:
- removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
- print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s",
- key, removed_paths,
- log=self.log)
+ if warn_exists:
+ removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
+ print_warning("Suppressed adding the following path(s) to $%s of the module "
+ "as they were already added: %s",
+ key, removed_paths,
+ log=self.log)
if not filtered_paths:
filtered_paths = None
return filtered_paths
- def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':', warn_exists=True):
"""
Generate append-path statements for the given list of paths.
@@ -243,13 +250,16 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
:param paths: list of paths to append
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
+ :param warn_exists: Show a warning if any path was already added to the variable
"""
- paths = self._filter_paths(key, paths)
+ paths = self._filter_paths(key, paths, warn_exists=warn_exists)
if paths is None:
return ''
- return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+ return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+ delim=delim)
- def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':', warn_exists=True):
"""
Generate prepend-path statements for the given list of paths.
@@ -257,11 +267,14 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
:param paths: list of paths to append
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
+ :param warn_exists: Show a warning if any path was already added to the variable
"""
- paths = self._filter_paths(key, paths)
+ paths = self._filter_paths(key, paths, warn_exists=warn_exists)
if paths is None:
return ''
- return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+ return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+ delim=delim)
def _modulerc_check_module_version(self, module_version):
"""
@@ -347,7 +360,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
module_version_statement = "module-version %(modname)s %(sym_version)s"
- # for Environment Modules we need to guard the module-version statement,
+ # for EnvironmentModulesC we need to guard the module-version statement,
# to avoid "Duplicate version symbol" warning messages where EasyBuild trips over,
# which occur because the .modulerc is parsed twice
# "module-info version " returns its argument if that argument is not a symbolic version (yet),
@@ -388,7 +401,7 @@ def is_loaded(self, mod_names):
:param mod_names: (list of) module name(s) to check load status for
"""
- if isinstance(mod_names, string_type):
+ if isinstance(mod_names, str):
res = self.IS_LOADED_TEMPLATE % mod_names
else:
res = [self.IS_LOADED_TEMPLATE % m for m in mod_names]
@@ -416,7 +429,7 @@ def unpack_setenv_value(self, env_var_name, env_var_val):
use_pushenv = False
# value may be specified as a string, or as a dict for special cases
- if isinstance(env_var_val, string_type):
+ if isinstance(env_var_val, str):
value = env_var_val
elif isinstance(env_var_val, dict):
@@ -484,13 +497,13 @@ def getenv_cmd(self, envvar, default=None):
"""
raise NotImplementedError
- def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=False, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -552,15 +565,16 @@ def unload_module(self, mod_name):
"""
raise NotImplementedError
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend-path or append-path statements for the given list of paths.
:param key: environment variable to prepend/append paths to
- :param paths: list of paths to prepend
+ :param paths: list of paths to prepend/append
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
raise NotImplementedError
@@ -610,21 +624,6 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None):
"""
raise NotImplementedError
- def _generate_extension_list(self):
- """
- Generate a string with a list of extensions.
-
- The name and version are separated by name_version_sep and each extension is separated by ext_sep
- """
- return self.app.make_extension_string()
-
- def _generate_extensions_list(self):
- """
- Generate a list of all extensions in name/version format
- """
- exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',')
- return exts_str.split(',') if exts_str else []
-
def _generate_help_text(self):
"""
Generate syntax-independent help text used for `module help`.
@@ -671,7 +670,7 @@ def _generate_help_text(self):
lines.extend(self._generate_section("Compatible modules", compatible_modules_txt))
# Extensions (if any)
- extensions = self._generate_extension_list()
+ extensions = self.app.make_extension_string()
lines.extend(self._generate_section("Included extensions", '\n'.join(wrap(extensions, 78))))
return '\n'.join(lines)
@@ -682,21 +681,13 @@ def _generate_multi_deps_list(self):
"""
multi_deps = []
if self.app.cfg['multi_deps']:
- for key in sorted(self.app.cfg['multi_deps'].keys()):
- mod_list = []
- txt = ''
- vlist = self.app.cfg['multi_deps'].get(key)
- for idx in range(len(vlist)):
- for deplist in self.app.cfg.multi_deps:
- for dep in deplist:
- if dep['name'] == key and dep['version'] == vlist[idx]:
- modname = dep['short_mod_name']
- # indicate which version is loaded by default (unless that's disabled)
- if idx == 0 and self.app.cfg['multi_deps_load_default']:
- modname += ' (default)'
- mod_list.append(modname)
- txt += ', '.join(mod_list)
- multi_deps.append(txt)
+ for name in sorted(self.app.cfg['multi_deps']):
+ mod_list = [dep['short_mod_name'] for dep in itertools.chain.from_iterable(self.app.cfg.multi_deps)
+ if dep['name'] == name]
+ # indicate that the first version is loaded by default if enabled
+ if self.app.cfg['multi_deps_load_default']:
+ mod_list[0] += ' (default)'
+ multi_deps.append(', '.join(mod_list))
return multi_deps
@@ -728,7 +719,7 @@ def _generate_whatis_lines(self):
if multi_deps:
whatis.append("Compatible modules: %s" % ', '.join(multi_deps))
- extensions = self._generate_extension_list()
+ extensions = self.app.make_extension_string()
if extensions:
whatis.append("Extensions: %s" % extensions)
@@ -758,8 +749,18 @@ def check_group(self, group, error_msg=None):
:param group: string with the group name
:param error_msg: error message to print for users outside that group
"""
- self.log.warning("Can't generate robust check in TCL modules for users belonging to group %s.", group)
- return ''
+ if self.modules_tool.supports_tcl_check_group:
+ if error_msg is None:
+ error_msg = "You are not part of '%s' group of users that have access to this software; " % group
+ error_msg += "Please consult with user support how to become a member of this group"
+
+ error_msg = 'error "%s"' % error_msg
+ res = self.conditional_statement('module-info usergroups %s' % group, error_msg, negative=True)
+ else:
+ self.log.warning("Can't generate robust check in Tcl modules for users belonging to group %s.", group)
+ res = ''
+
+ return res
def comment(self, msg):
"""Return string containing given message as a comment."""
@@ -778,7 +779,7 @@ def conditional_statement(self, conditions, body, negative=False, else_body=None
:param cond_or: combine multiple conditions using 'or' (default is to combine with 'and')
:param cond_tmpl: template for condition expression (default: '%s')
"""
- if isinstance(conditions, string_type):
+ if isinstance(conditions, str):
conditions = [conditions]
if cond_or:
@@ -817,19 +818,20 @@ def get_description(self, conflict=True):
"""
Generate a description.
"""
- txt = '\n'.join([
+ lines = [
"proc ModulesHelp { } {",
" puts stderr {%s" % re.sub(r'([{}\[\]])', r'\\\1', self._generate_help_text()),
" }",
'}',
'',
+ ]
+
+ lines.extend([
+ "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line)
+ for line in self._generate_whatis_lines()
])
- lines = [
- '%(whatis_lines)s',
- '',
- "set root %(installdir)s",
- ]
+ lines.extend(['', "set root " + self.app.installdir])
if self.app.cfg['moduleloadnoconflict']:
cond_unload = self.conditional_statement(self.is_loaded('%(name)s'), "module unload %(name)s")
@@ -846,41 +848,36 @@ def get_description(self, conflict=True):
# - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4'
lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)])
- whatis_lines = [
- "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line)
- for line in self._generate_whatis_lines()
- ]
- txt += '\n'.join([''] + lines + ['']) % {
- 'name': self.app.name,
- 'version': self.app.version,
- 'whatis_lines': '\n'.join(whatis_lines),
- 'installdir': self.app.installdir,
- }
-
- return txt
+ return '\n'.join(lines + [''])
def getenv_cmd(self, envvar, default=None):
"""
Return module-syntax specific code to get value of specific environment variable.
"""
if default is None:
- cmd = '$::env(%s)' % envvar
+ if self.modules_tool.supports_tcl_getenv:
+ cmd = '[getenv %s]' % envvar
+ else:
+ cmd = '$::env(%s)' % envvar
else:
- values = {
- 'default': default,
- 'envvar': '::env(%s)' % envvar,
- }
- cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values
+ if self.modules_tool.supports_tcl_getenv:
+ cmd = '[getenv %s "%s"]' % (envvar, default)
+ else:
+ values = {
+ 'default': default,
+ 'envvar': '::env(%s)' % envvar,
+ }
+ cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values
return cmd
- def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=None, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
(if None: enable if recursive_mod_unload build option or depends_on is True)
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -889,7 +886,10 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
body.extend([self.unload_module(m).strip() for m in unload_modules])
load_template = self.LOAD_TEMPLATE
# Lmod 7.6.1+ supports depends-on which does this most nicely:
- if build_option('mod_depends_on') or depends_on:
+ if (build_option('mod_depends_on') and self.modules_tool.supports_depends_on) or depends_on:
+ if depends_on is not None:
+ depr_msg = "'depends_on' argument of module generator method 'load_module' should not be used anymore"
+ self.log.deprecated(depr_msg, '6.0')
if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends-on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
@@ -900,8 +900,13 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
cond_tmpl = None
+ # Environment Modules v4+ safely handles automatic module load by not reloading already
+ # loaded module. No safe guard test is required and it should even be avoided to get the
+ # module dependency correctly tracked.
+ safe_auto_load = self.modules_tool.supports_safe_auto_load
+
if recursive_unload is None:
- recursive_unload = build_option('recursive_mod_unload') or depends_on
+ recursive_unload = build_option('recursive_mod_unload') or depends_on or safe_auto_load
if recursive_unload:
# wrapping the 'module load' statement with an 'is-loaded or mode == unload'
@@ -912,7 +917,7 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
# see also http://lmod.readthedocs.io/en/latest/210_load_storms.html
cond_tmpl = "[ module-info mode remove ] || %s"
- if depends_on:
+ if depends_on or safe_auto_load:
if multi_dep_mods and len(multi_dep_mods) > 1:
parent_mod_name = os.path.dirname(mod_name)
guard = self.is_loaded(multi_dep_mods[1:])
@@ -956,15 +961,16 @@ def msg_on_unload(self, msg):
print_cmd = "puts stderr %s" % quote_str(msg, tcl=True)
return '\n'.join(['', self.conditional_statement("module-info mode unload", print_cmd, indent=False)])
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend-path or append-path statements for the given list of paths.
:param key: environment variable to prepend/append paths to
- :param paths: list of paths to prepend
+ :param paths: list of paths to prepend/append
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
if prepend:
update_type = 'prepend'
@@ -975,7 +981,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
return ''
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
self.log.debug("Wrapping %s into a list before using it to %s path %s", paths, update_type, key)
paths = [paths]
@@ -996,7 +1002,8 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
else:
abspaths.append(path)
- statements = ['%s-path\t%s\t\t%s\n' % (update_type, key, p) for p in abspaths]
+ delim_opt = '' if delim == ':' else f' -d "{delim}"'
+ statements = [f'{update_type}-path{delim_opt}\t{key}\t\t{p}\n' for p in abspaths]
return ''.join(statements)
def set_alias(self, key, value):
@@ -1008,7 +1015,7 @@ def set_alias(self, key, value):
def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None):
"""
- Create a .version file inside the package module folder in order to set the default version for TMod
+ Create a .version file inside the package module folder in order to set the default version
:param module_dir_path: module directory path, e.g. $HOME/easybuild/modules/all/Bison
:param module_version: module version, e.g. 3.0.4
@@ -1147,6 +1154,7 @@ class ModuleGeneratorLua(ModuleGenerator):
PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")'
UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)'
+ UPDATE_PATH_TEMPLATE_DELIM = '%s_path("%s", %s, "%s")'
START_STR = '[==['
END_STR = ']==]'
@@ -1181,21 +1189,12 @@ def check_group(self, group, error_msg=None):
:param group: string with the group name
:param error_msg: error message to print for users outside that group
"""
- lmod_version = self.modules_tool.version
- min_lmod_version = '6.0.8'
+ if error_msg is None:
+ error_msg = "You are not part of '%s' group of users that have access to this software; " % group
+ error_msg += "Please consult with user support how to become a member of this group"
- if LooseVersion(lmod_version) >= LooseVersion(min_lmod_version):
- if error_msg is None:
- error_msg = "You are not part of '%s' group of users that have access to this software; " % group
- error_msg += "Please consult with user support how to become a member of this group"
-
- error_msg = 'LmodError("' + error_msg + '")'
- res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True)
- else:
- warn_msg = "Can't generate robust check in Lua modules for users belonging to group %s. "
- warn_msg += "Lmod version not recent enough (%s), should be >= %s"
- self.log.warning(warn_msg, group, lmod_version, min_lmod_version)
- res = ''
+ error_msg = 'LmodError("' + error_msg + '")'
+ res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True)
return res
@@ -1223,7 +1222,7 @@ def conditional_statement(self, conditions, body, negative=False, else_body=None
:param cond_or: combine multiple conditions using 'or' (default is to combine with 'and')
:param cond_tmpl: template for condition expression (default: '%s')
"""
- if isinstance(conditions, string_type):
+ if isinstance(conditions, str):
conditions = [conditions]
if cond_or:
@@ -1262,48 +1261,32 @@ def get_description(self, conflict=True):
"""
Generate a description.
"""
- txt = '\n'.join([
+ lines = [
'help(%s%s' % (self.START_STR, self.check_str(self._generate_help_text())),
'%s)' % self.END_STR,
'',
- ])
-
- lines = [
- "%(whatis_lines)s",
- '',
- 'local root = "%(installdir)s"',
]
+ for line in self._generate_whatis_lines():
+ lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR))
+
+ lines.extend(['', 'local root = "%s"' % self.app.installdir])
+
if self.app.cfg['moduleloadnoconflict']:
self.log.info("Nothing to do to ensure no conflicts can occur on load when using Lua modules files/Lmod")
elif conflict:
# conflict on 'name' part of module name (excluding version part at the end)
lines.extend(['', 'conflict("%s")' % os.path.dirname(self.app.short_mod_name)])
-
- whatis_lines = []
- for line in self._generate_whatis_lines():
- whatis_lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR))
-
- if build_option('module_extensions'):
- extensions_list = self._generate_extensions_list()
-
+ extensions_list = self.app.make_extension_string(name_version_sep='/', ext_sep=',')
if extensions_list:
- extensions_stmt = 'extensions("%s")' % ','.join([str(x) for x in extensions_list])
+ extensions_stmt = 'extensions("%s")' % extensions_list
# put this behind a Lmod version check as 'extensions' is only (well) supported since Lmod 8.2.8,
# see https://lmod.readthedocs.io/en/latest/330_extensions.html#module-extensions and
# https://github.com/TACC/Lmod/issues/428
lines.extend(['', self.conditional_statement(self.check_version("8", "2", "8"), extensions_stmt)])
- txt += '\n'.join([''] + lines + ['']) % {
- 'name': self.app.name,
- 'version': self.app.version,
- 'whatis_lines': '\n'.join(whatis_lines),
- 'installdir': self.app.installdir,
- 'homepage': self.app.cfg['homepage'],
- }
-
- return txt
+ return '\n'.join(lines + [''])
def getenv_cmd(self, envvar, default=None):
"""
@@ -1315,14 +1298,14 @@ def getenv_cmd(self, envvar, default=None):
cmd = 'os.getenv("%s") or "%s"' % (envvar, default)
return cmd
- def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=None, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
(if None: enable if recursive_mod_unload build option or depends_on is True)
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -1332,7 +1315,10 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
load_template = self.LOAD_TEMPLATE
# Lmod 7.6+ supports depends_on which does this most nicely:
- if build_option('mod_depends_on') or depends_on:
+ if (build_option('mod_depends_on') and self.modules_tool.supports_depends_on) or depends_on:
+ if depends_on is not None:
+ depr_msg = "'depends_on' argument of module generator method 'load_module' should not be used anymore"
+ self.log.deprecated(depr_msg, '6.0')
if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends_on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
@@ -1427,7 +1413,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath,
modulerc_txt=modulerc_txt)
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend_path or append_path statements for the given list of paths
@@ -1436,6 +1422,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
if prepend:
update_type = 'prepend'
@@ -1446,7 +1433,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
return ''
- if isinstance(paths, string_type):
+ if isinstance(paths, str):
self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key)
paths = [paths]
@@ -1468,7 +1455,10 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
else:
abspaths.append('root')
- statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
+ if delim != ':':
+ statements = [self.UPDATE_PATH_TEMPLATE_DELIM % (update_type, key, p, delim) for p in abspaths]
+ else:
+ statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
statements.append('')
return '\n'.join(statements)
diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py
index e54f04c31e..646366409d 100644
--- a/easybuild/tools/module_naming_scheme/__init__.py
+++ b/easybuild/tools/module_naming_scheme/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2024 Ghent University
+# Copyright 2011-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py
index cdafc97cbb..b3eff8ed69 100644
--- a/easybuild/tools/module_naming_scheme/categorized_mns.py
+++ b/easybuild/tools/module_naming_scheme/categorized_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py
index 93ff663e77..c3d3acc2d2 100644
--- a/easybuild/tools/module_naming_scheme/easybuild_mns.py
+++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py
index d0e4ea5966..b14425d9ce 100644
--- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py
+++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -204,10 +204,10 @@ def det_modpath_extensions(self, ec):
comp_name_ver = None
if ec['name'] in extend_comps:
- for key in COMP_NAME_VERSION_TEMPLATES:
+ for key, comp_tmpl in COMP_NAME_VERSION_TEMPLATES.items():
comp_names = key.split(',')
if ec['name'] in comp_names:
- comp_name, comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key]
+ comp_name, comp_ver_tmpl = comp_tmpl
comp_versions = {ec['name']: self.det_full_version(ec)}
if ec['name'] == 'ifort':
# 'icc' key should be provided since it's the only one used in the template
diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
index 5603b7db47..c90717a4eb 100644
--- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
+++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py
index fd60dbbe90..cf7bfcd393 100644
--- a/easybuild/tools/module_naming_scheme/mns.py
+++ b/easybuild/tools/module_naming_scheme/mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2024 Ghent University
+# Copyright 2011-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,9 +33,9 @@
import re
from easybuild.base import fancylogger
+from easybuild.base.wrapper import create_base_metaclass
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import Singleton
-from easybuild.tools.py2vs3 import create_base_metaclass
DEVEL_MODULE_SUFFIX = '-easybuild-devel'
diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py
index b85032e6e6..e0627090d0 100644
--- a/easybuild/tools/module_naming_scheme/toolchain.py
+++ b/easybuild/tools/module_naming_scheme/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py
index 4fff90f690..7061f5cebb 100644
--- a/easybuild/tools/module_naming_scheme/utilities.py
+++ b/easybuild/tools/module_naming_scheme/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,7 +40,6 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
from easybuild.tools.utilities import get_subclasses, import_available_modules
@@ -64,12 +63,12 @@ def det_full_ec_version(ec):
# prepend/append version prefix/suffix
versionprefix = ec.get('versionprefix', '')
- if versionprefix and not isinstance(versionprefix, string_type):
+ if versionprefix and not isinstance(versionprefix, str):
raise EasyBuildError("versionprefix value should be a string, found '%s': %s (full spec: %s)",
type(versionprefix).__name__, versionprefix, ec)
versionsuffix = ec.get('versionsuffix', '')
- if versionsuffix and not isinstance(versionsuffix, string_type):
+ if versionsuffix and not isinstance(versionsuffix, str):
raise EasyBuildError("versionsuffix value should be a string, found '%s': %s (full spec: %s)",
type(versionsuffix).__name__, versionsuffix, ec)
@@ -94,7 +93,7 @@ def avail_module_naming_schemes():
def is_valid_module_name(mod_name):
"""Check whether the specified value is a valid module name."""
# module name must be a string
- if not isinstance(mod_name, string_type):
+ if not isinstance(mod_name, str):
_log.warning("Wrong type for module name %s (%s), should be a string" % (mod_name, type(mod_name)))
return False
# module name must be relative path
diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py
index 8929ccc560..237b234e39 100644
--- a/easybuild/tools/modules.py
+++ b/easybuild/tools/modules.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -41,21 +41,24 @@
import os
import re
import shlex
+from enum import Enum
from easybuild.base import fancylogger
-from easybuild.tools import StrictVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning
-from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET
-from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS
+from easybuild.tools import LooseVersion
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning
+from easybuild.tools.config import ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE, LOADED_MODULES_ACTIONS, PURGE
+from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET
from easybuild.tools.config import build_option, get_modules_tool, install_path
from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars
from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
-from easybuild.tools.py2vs3 import subprocess_popen_text
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.utilities import get_subclasses, nub
+
+MODULE_LOAD_ENV_HEADERS = 'CPP_HEADERS'
+
# software root/version environment variable name prefixes
ROOT_ENV_VAR_NAME_PREFIX = "EBROOT"
VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION"
@@ -132,6 +135,355 @@
_log = fancylogger.getLogger('modules', fname=False)
+class ModEnvVarType(Enum):
+ """
+ Possible types of ModuleEnvironmentVariable:
+ - STRING: (list of) strings with no further meaning
+ - PATH: (list of) of paths to existing directories or files
+ - PATH_WITH_FILES: (list of) of paths to existing directories containing
+ one or more files
+ - PATH_WITH_TOP_FILES: (list of) of paths to existing directories
+ containing one or more files in its top directory
+ - """
+ STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES = range(0, 4)
+
+
+class ModuleEnvironmentVariable:
+ """
+ Environment variable data structure for modules
+ Contents of environment variable is a list of unique strings
+ """
+
+ def __init__(self, contents, delimiter=os.pathsep, prepend=True, var_type=ModEnvVarType.PATH_WITH_FILES):
+ """
+ Initialize new environment variable
+ Actual contents of the environment variable are held in self.contents
+ By default, the environment variable is a list of paths with files in them
+ Existence of paths and their contents are not checked at init
+ """
+ self.contents = contents
+ self.delimiter = delimiter
+ self.mod_prepend = prepend
+ self.type = var_type
+
+ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+
+ def __repr__(self):
+ return repr(self.contents)
+
+ def __str__(self):
+ return self.delimiter.join(self.contents)
+
+ def __iter__(self):
+ return iter(self.contents)
+
+ @property
+ def contents(self):
+ return self._contents
+
+ @contents.setter
+ def contents(self, value):
+ """Enforce that contents is a list of strings"""
+ if isinstance(value, str):
+ value = [value]
+
+ try:
+ str_list = [str(path) for path in value]
+ except TypeError as err:
+ raise TypeError("ModuleEnvironmentVariable.contents must be a list of strings") from err
+
+ self._contents = nub(str_list) # remove duplicates and keep order
+
+ @property
+ def type(self):
+ return self._type
+
+ @type.setter
+ def type(self, value):
+ """Convert type to VarType"""
+ if isinstance(value, ModEnvVarType):
+ self._type = value
+ else:
+ try:
+ self._type = ModEnvVarType[value]
+ except KeyError as err:
+ raise EasyBuildError(f"Cannot create ModuleEnvironmentVariable with type {value}") from err
+
+ def append(self, item):
+ """Shortcut to append to list of contents"""
+ self.contents += [item]
+
+ def extend(self, item):
+ """Shortcut to extend list of contents"""
+ self.contents += item
+
+ def prepend(self, item):
+ """Shortcut to prepend item to list of contents"""
+ self.contents = [item] + self.contents
+
+ def update(self, item):
+ """Shortcut to replace list of contents with item"""
+ self.contents = item
+
+ def remove(self, *args):
+ """Shortcut to remove items from list of contents"""
+ try:
+ self.contents.remove(*args)
+ except ValueError:
+ # item is not in the list, move along
+ self.log.debug(f"ModuleEnvironmentVariable does not contain item: {' '.join(args)}")
+
+ @property
+ def is_path(self):
+ path_like_types = [
+ ModEnvVarType.PATH,
+ ModEnvVarType.PATH_WITH_FILES,
+ ModEnvVarType.PATH_WITH_TOP_FILES,
+ ]
+ return self.type in path_like_types
+
+
+class ModuleLoadEnvironment:
+ """
+ Changes to environment variables that should be made when environment module is loaded.
+ - Environment variables are defined as ModuleEnvironmentVariables instances
+ with attribute name equal to environment variable name.
+ - Aliases are arbitrary names that serve to apply changes to lists of
+ environment variables
+ - Environment variables are public attributes, with names containing
+ uppercase letters and '_'
+ - Other attributes like aliases are private, with names starting with '_'
+ - Environment variables are kept in a private dict to avoid name collisions
+ """
+
+ def __init__(self, aliases=None):
+ """
+ Initialize default environment definition
+
+ :aliases: dict defining environment variables aliases
+ """
+ # The following regex patterns are needed to properly distinguish
+ # between public environment variables and private attributes. Set them
+ # directly into __dict__ to bypass class getter and setter.
+ self.__dict__['regex'] = {}
+ self.regex['mangled_attr'] = re.compile('^_[A-Za-z]+__') # mangled attributes: _ClassName__VAR_NAME
+ self.regex['private_attr'] = re.compile('^_[a-z][a-z_]+$') # private attributes: _var_name
+ self.regex['env_var_name'] = re.compile('^[A-Z_]+[A-Z0-9_]+$') # environment variables: {__}VAR_NAME_00_SUFFIX
+
+ self._log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+
+ self._aliases = {}
+ if aliases is not None:
+ try:
+ for alias_name, alias_vars in aliases.items():
+ self.update_alias(alias_name, alias_vars)
+ except AttributeError as err:
+ raise EasyBuildError(
+ "Wrong format for aliases defitions passed to ModuleLoadEnvironment. "
+ f"Expected a dictionary but got: {type(aliases)}."
+ ) from err
+
+ self._env_vars = {}
+ self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')]
+ self.CLASSPATH = ['*.jar']
+ self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64
+ self.CMAKE_PREFIX_PATH = ['']
+ self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS]
+ self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
+ self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
+ self.MANPATH = ['man', os.path.join('share', 'man')]
+ self.PATH = SEARCH_PATH_BIN_DIRS + ['sbin']
+ self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']]
+ self.XDG_DATA_DIRS = ['share']
+
+ # environment variables with known aliases
+ # e.g. search paths to C/C++ headers
+ for envar_name in self._aliases.get(MODULE_LOAD_ENV_HEADERS, []):
+ setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS)
+
+ def __getattr__(self, name):
+ """
+ Return requested attribute from either the private attributes in self.__dict__
+ or the public ModuleEnvironmentVariables in self._env_vars
+ """
+ if self.regex['private_attr'].match(name):
+ return self.__dict__[name]
+
+ name = self._unmangle_env_var_name(name)
+ return self.__dict__['_env_vars'][name]
+
+ def __setattr__(self, name, value):
+ """
+ Specific restrictions for ModuleLoadEnvironment attributes:
+ - public attributes are instances of ModuleEnvironmentVariable
+ - private attributes are allowed with lowercase names starting with single underscore
+ """
+ if self.regex['private_attr'].match(name):
+ # do not control protected/private attributes
+ return super().__setattr__(name, value)
+
+ try:
+ # set public environment variable
+ name = self._unmangle_env_var_name(name)
+ self._env_vars[name] = self._set_module_environment_variable(name, value)
+ except TypeError as err:
+ raise EasyBuildError(
+ f"Cannot define ModuleEnvironmentVariable ${name} with the following attributes: {value}"
+ ) from err
+
+ return True
+
+ def __delattr__(self, name):
+ """
+ Delete private attributes or public ModuleEnvironmentVariables
+ Fails on missing attributes
+ """
+
+ if self.regex['private_attr'].match(name):
+ del self.__dict__[name]
+ return True
+
+ name = self._unmangle_env_var_name(name)
+ try:
+ del self.__dict__['_env_vars'][name]
+ except KeyError as err:
+ raise EasyBuildError(
+ f"Cannot delete environment variable from ModuleEnvironmentVariable: {name} not found."
+ ) from err
+ return True
+
+ def _unmangle_env_var_name(self, name):
+ """
+ Unmangle environment variable names that were originally set with a leading double underscore
+ """
+ if self.regex['mangled_attr'].match(name):
+ # environment variable with 2+ leading underscores: __UPPER_CASE
+ # undo mangling into _ClassName__UPPER_CASE
+ split_name = name.split('__')
+ split_name[0] = ''
+ name = '__'.join(split_name)
+ return name
+
+ def _set_module_environment_variable(self, name, value):
+ """
+ Specific restrictions for ModuleEnvironmentVariable attributes:
+ - attribute names are uppercase with underscores
+ - dictionaries are unpacked into arguments of ModuleEnvironmentVariable
+ - controls variables with special types (e.g. PATH, LD_LIBRARY_PATH)
+ """
+ if not self.regex['env_var_name'].match(name):
+ raise EasyBuildError(
+ "Name of ModuleLoadEnvironment attribute does not conform to shell naming rules, "
+ f"it must only have upper-case letters and underscores: '{name}'"
+ )
+
+ if not isinstance(value, dict):
+ value = {'contents': value}
+
+ # special variables that require files in their top directories
+ if name in ('LD_LIBRARY_PATH', 'PATH'):
+ value.update({'var_type': ModEnvVarType.PATH_WITH_TOP_FILES})
+
+ return ModuleEnvironmentVariable(**value)
+
+ @property
+ def vars(self):
+ """Return list of public ModuleEnvironmentVariable"""
+ return list(self._env_vars)
+
+ def __iter__(self):
+ """Make the class iterable"""
+ yield from self.vars
+
+ def items(self):
+ """
+ Return key-value pairs for each attribute that is a ModuleEnvironmentVariable
+ - key = attribute name
+ - value = its "contents" attribute
+ """
+ for attr in self.vars:
+ yield attr, self._env_vars[attr]
+
+ def update(self, new_env):
+ """Update contents of environment from given dictionary"""
+ try:
+ for envar_name, envar_contents in new_env.items():
+ setattr(self, envar_name, envar_contents)
+ except AttributeError as err:
+ raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err
+
+ def replace(self, new_env):
+ """Replace contents of environment with given dictionary"""
+ for var in self.vars:
+ self.remove(var)
+ self.update(new_env)
+
+ def remove(self, var_name):
+ """
+ Remove ModuleEnvironmentVariable attribute from instance
+ Silently goes through if attribute is already missing
+ """
+ if var_name in self.vars:
+ del self._env_vars[var_name]
+
+ @property
+ def as_dict(self):
+ """
+ Return dict with mapping of ModuleEnvironmentVariables names with their contents
+ """
+ return dict(self.items())
+
+ @property
+ def environ(self):
+ """
+ Return dict with mapping of ModuleEnvironmentVariables names with their contents
+ Equivalent in shape to os.environ
+ """
+ return {envar_name: str(envar_contents) for envar_name, envar_contents in self.items()}
+
+ def alias(self, alias):
+ """
+ Return iterator to search path variables for given alias
+ """
+ try:
+ yield from [self._env_vars[var_name] for var_name in self._aliases[alias]]
+ except KeyError as err:
+ raise EasyBuildError(f"Unknown search path alias: {alias}") from err
+ except AttributeError as err:
+ raise EasyBuildError(f"Missing environment variable in '{alias} alias") from err
+
+ def alias_vars(self, alias):
+ """
+ Return list of environment variable names aliased by given alias
+ """
+ try:
+ return self._aliases[alias]
+ except KeyError as err:
+ raise EasyBuildError(f"Unknown search path alias: {alias}") from err
+
+ def update_alias(self, alias, value):
+ """
+ Update existing or non-existing alias with given search paths variables
+ """
+ if isinstance(value, str):
+ value = [value]
+
+ try:
+ self._aliases[alias] = [str(envar) for envar in value]
+ except TypeError as err:
+ raise TypeError("ModuleLoadEnvironment aliases must be a list of strings") from err
+
+ def set_alias_vars(self, alias, value):
+ """
+ Set value of search paths variables for given alias
+ """
+ try:
+ for envar_name in self._aliases[alias]:
+ setattr(self, envar_name, value)
+ except KeyError as err:
+ raise EasyBuildError(f"Unknown search path alias: {alias}") from err
+
+
class ModulesTool(object):
"""An abstract interface to a tool that deals with modules."""
# name of this modules tool (used in log/warning/error messages)
@@ -147,11 +499,13 @@ class ModulesTool(object):
COMMAND_SHELL = None
# option to determine the version
VERSION_OPTION = '--version'
- # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
+ # minimal required version (cannot include -beta or rc)
REQ_VERSION = None
+ # minimal required version to check user's group in modulefile
+ REQ_VERSION_TCL_CHECK_GROUP = None
# deprecated version limit (support for versions below this version is deprecated)
DEPR_VERSION = None
- # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
+ # maximum version allowed (cannot include -beta or rc)
MAX_VERSION = None
# the regexp, should have a "version" group (multiline search)
VERSION_REGEXP = None
@@ -210,6 +564,9 @@ def __init__(self, mod_paths=None, testing=False):
self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch'))
self.set_and_check_version()
self.supports_depends_on = False
+ self.supports_tcl_getenv = False
+ self.supports_tcl_check_group = False
+ self.supports_safe_auto_load = False
def __str__(self):
"""String representation of this ModulesTool instance."""
@@ -242,14 +599,6 @@ def set_and_check_version(self):
if res:
self.version = res.group('version')
self.log.info("Found %s version %s", self.NAME, self.version)
-
- # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid),
- # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release
- self.version = self.version.replace('rc', 'b').replace('-beta', 'b1')
- if len(self.version.split('.')) > 3:
- self.version = '.'.join(self.version.split('.')[:3])
-
- self.log.info("Converted actual version to '%s'" % self.version)
else:
raise EasyBuildError("Failed to determine %s version from option '%s' output: %s",
self.NAME, self.VERSION_OPTION, txt)
@@ -262,9 +611,10 @@ def set_and_check_version(self):
elif build_option('modules_tool_version_check'):
self.log.debug("Checking whether %s version %s meets requirements", self.NAME, self.version)
+ version = LooseVersion(self.version)
if self.REQ_VERSION is not None:
self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION)
- if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION):
+ if version < self.REQ_VERSION or version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']):
raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s",
self.NAME, self.REQ_VERSION, self.version)
else:
@@ -272,18 +622,14 @@ def set_and_check_version(self):
if self.DEPR_VERSION is not None:
self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION)
- if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION):
+ if version < self.DEPR_VERSION or version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']):
depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION)
depr_msg += "found version %s" % self.version
-
- if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'):
- self.log.warning(depr_msg)
- else:
- self.log.deprecated(depr_msg, '5.0')
+ self.log.deprecated(depr_msg, '6.0')
if self.MAX_VERSION is not None:
self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION)
- if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION):
+ if self.version > self.MAX_VERSION and not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']):
raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s",
self.NAME, self.MAX_VERSION, self.version)
else:
@@ -310,31 +656,32 @@ def check_module_function(self, allow_mismatch=False, regex=None):
if self.testing:
# grab 'module' function definition from environment if it's there; only during testing
try:
- out, ec = os.environ['module'], 0
+ output, exit_code = os.environ['module'], EasyBuildExit.SUCCESS
except KeyError:
- out, ec = None, 1
+ output, exit_code = None, EasyBuildExit.FAIL_SYSTEM_CHECK
else:
cmd = "type module"
- out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False)
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False)
+ output, exit_code = res.output, res.exit_code
if regex is None:
regex = r".*%s" % os.path.basename(self.cmd)
mod_cmd_re = re.compile(regex, re.M)
mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.NAME)
- if ec == 0:
- if mod_cmd_re.search(out):
+ if exit_code == EasyBuildExit.SUCCESS:
+ if mod_cmd_re.search(output):
self.log.debug("Found pattern '%s' in defined 'module' function." % mod_cmd_re.pattern)
else:
msg = "%s not found in defined 'module' function.\n" % mod_details
msg += "Specify the correct modules tool to avoid weird problems due to this mismatch, "
msg += "see the --modules-tool and --avail-modules-tools command line options.\n"
if allow_mismatch:
- msg += "Obtained definition of 'module' function: %s" % out
+ msg += "Obtained definition of 'module' function: %s" % output
self.log.warning(msg)
else:
msg += "Or alternatively, use --allow-modules-tool-mismatch to stop treating this as an error. "
- msg += "Obtained definition of 'module' function: %s" % out
+ msg += "Obtained definition of 'module' function: %s" % output
raise EasyBuildError(msg)
else:
# module function may not be defined (weird, but fine)
@@ -551,18 +898,14 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r
return wrapped_mod
- def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, maybe_partial=True):
+ def exist(self, mod_names, skip_avail=False, maybe_partial=True):
"""
Check if modules with specified names exists.
:param mod_names: list of module names
- :param mod_exists_regex_template: DEPRECATED and unused
:param skip_avail: skip checking through 'module avail', only check via 'module show'
:param maybe_partial: indicates if the module name may be a partial module name
"""
- if mod_exists_regex_template is not None:
- self.log.deprecated('mod_exists_regex_template is no longer used', '5.0')
-
def mod_exists_via_show(mod_name):
"""
Helper function to check whether specified module name exists through 'module show'.
@@ -598,7 +941,8 @@ def mod_exists_via_show(mod_name):
self.log.debug("Skipping warning line '%s'", line)
continue
- # skip lines that start with 'module-' (like 'module-version'),
+ # skip lines that start with 'module-' (like 'module-version')
+ # that may appear with EnvironmentModulesC or EnvironmentModulesTcl,
# see https://github.com/easybuilders/easybuild-framework/issues/3376
if line.startswith('module-'):
self.log.debug("Skipping line '%s' since it starts with 'module-'", line)
@@ -718,7 +1062,8 @@ def show(self, mod_name):
ans = MODULE_SHOW_CACHE[key]
self.log.debug("Found cached result for 'module show %s' with key '%s': %s", mod_name, key, ans)
else:
- ans = self.run_module('show', mod_name, check_output=False, return_stderr=True)
+ ans = self.run_module('show', mod_name, check_output=False, return_stderr=True,
+ check_exit_code=False)
MODULE_SHOW_CACHE[key] = ans
self.log.debug("Cached result for 'module show %s' with key '%s': %s", mod_name, key, ans)
@@ -753,7 +1098,7 @@ def modulefile_path(self, mod_name, strip_ext=False):
:param mod_name: module name
:param strip_ext: strip (.lua) extension from module fileame (if present)"""
# (possible relative) path is always followed by a ':', and may be prepended by whitespace
- # this works for both environment modules and Lmod
+ # this works for both Environment Modules and Lmod
modpath_re = re.compile(r'^\s*(?P[^/\n]*/[^\s]+):$', re.M)
modpath = self.get_value_from_modulefile(mod_name, modpath_re)
@@ -824,24 +1169,22 @@ def run_module(self, *args, **kwargs):
key, old_value, new_value)
cmd_list = self.compose_cmd_list(args)
- full_cmd = ' '.join(cmd_list)
- self.log.debug("Running module command '%s' from %s" % (full_cmd, os.getcwd()))
-
- proc = subprocess_popen_text(cmd_list, env=environ)
+ cmd = ' '.join(cmd_list)
+ # note: module commands are always run in dry mode, and are kept hidden in trace and dry run output
+ res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, use_bash=False, split_stderr=True,
+ hidden=True, in_dry_run=True, output_file=False)
# stdout will contain python code (to change environment etc)
# stderr will contain text (just like the normal module command)
- (stdout, stderr) = proc.communicate()
- self.log.debug("Output of module command '%s': stdout: %s; stderr: %s" % (full_cmd, stdout, stderr))
+ stdout, stderr = res.output, res.stderr
# also catch and check exit code
- exit_code = proc.returncode
- if kwargs.get('check_exit_code', True) and exit_code != 0:
+ if kwargs.get('check_exit_code', True) and res.exit_code != EasyBuildExit.SUCCESS:
raise EasyBuildError("Module command '%s' failed with exit code %s; stderr: %s; stdout: %s",
- ' '.join(cmd_list), exit_code, stderr, stdout)
+ cmd, res.exit_code, stderr, stdout)
if kwargs.get('check_output', True):
- self.check_module_output(full_cmd, stdout, stderr)
+ self.check_module_output(cmd, stdout, stderr)
if kwargs.get('return_stderr', False):
return stderr
@@ -958,7 +1301,7 @@ def check_loaded_modules(self):
"use the --allow-loaded-modules configuration option.",
"To specify action to take when loaded modules are detected, use %s." % opt,
'',
- "See http://easybuild.readthedocs.io/en/latest/Detecting_loaded_modules.html for more information.",
+ "See https://docs.easybuild.io/detecting-loaded-modules/ for more information.",
])
action = build_option('detect_loaded_modules')
@@ -1189,11 +1532,12 @@ def update(self):
class EnvironmentModulesC(ModulesTool):
- """Interface to (C) environment modules (modulecmd)."""
+ """Interface to (C) Environment Modules (modulecmd)."""
NAME = "Environment Modules"
COMMAND = "modulecmd"
REQ_VERSION = '3.2.10'
MAX_VERSION = '3.99'
+ DEPR_VERSION = '3.999'
VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*'
def run_module(self, *args, **kwargs):
@@ -1203,7 +1547,7 @@ def run_module(self, *args, **kwargs):
if isinstance(args[0], (list, tuple,)):
args = args[0]
- # some versions of Cray's environment modules tool (3.2.10.x) include a "source */init/bash" command
+ # some versions of Cray's Environment Modules tool (3.2.10.x) include a "source */init/bash" command
# in the output of some "modulecmd python load" calls, which is not a valid Python command,
# which must be stripped out to avoid "invalid syntax" errors when evaluating the output
def tweak_stdout(txt):
@@ -1245,9 +1589,9 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name):
class EnvironmentModulesTcl(EnvironmentModulesC):
- """Interface to (Tcl) environment modules (modulecmd.tcl)."""
+ """Interface to (ancient Tcl-only) Environment Modules (modulecmd.tcl)."""
NAME = "ancient Tcl-only Environment Modules"
- # Tcl environment modules have no --terse (yet),
+ # ancient Tcl-only Environment Modules have no --terse (yet),
# -t must be added after the command ('avail', 'list', etc.)
TERSE_OPTION = (1, '-t')
COMMAND = 'modulecmd.tcl'
@@ -1255,12 +1599,13 @@ class EnvironmentModulesTcl(EnvironmentModulesC):
COMMAND_SHELL = ['tclsh']
VERSION_OPTION = ''
REQ_VERSION = None
+ DEPR_VERSION = '9999.9'
VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s'
def set_path_env_var(self, key, paths):
"""Set environment variable with given name to the given list of paths."""
super(EnvironmentModulesTcl, self).set_path_env_var(key, paths)
- # for Tcl environment modules, we need to make sure the _modshare env var is kept in sync
+ # for Tcl Environment Modules, we need to make sure the _modshare env var is kept in sync
setvar('%s_modshare' % key, ':1:'.join(paths), verbose=False)
def run_module(self, *args, **kwargs):
@@ -1323,13 +1668,15 @@ def remove_module_path(self, path, set_mod_paths=True):
self.set_mod_paths()
-class EnvironmentModules(EnvironmentModulesTcl):
- """Interface to environment modules 4.0+"""
+class EnvironmentModules(ModulesTool):
+ """Interface to Environment Modules 4.0+"""
NAME = "Environment Modules"
COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl')
COMMAND_ENVIRONMENT = 'MODULES_CMD'
- REQ_VERSION = '4.0.0'
+ REQ_VERSION = '4.3.0'
+ DEPR_VERSION = '4.3.0'
MAX_VERSION = None
+ REQ_VERSION_TCL_CHECK_GROUP = '4.6.0'
VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s'
SHOW_HIDDEN_OPTION = '--all'
@@ -1356,6 +1703,10 @@ def __init__(self, *args, **kwargs):
setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False)
super(EnvironmentModules, self).__init__(*args, **kwargs)
+ version = LooseVersion(self.version)
+ self.supports_tcl_getenv = True
+ self.supports_tcl_check_group = version >= LooseVersion(self.REQ_VERSION_TCL_CHECK_GROUP)
+ self.supports_safe_auto_load = True
def check_module_function(self, allow_mismatch=False, regex=None):
"""Check whether selected module tool matches 'module' function definition."""
@@ -1369,7 +1720,8 @@ def check_module_function(self, allow_mismatch=False, regex=None):
out, ec = None, 1
else:
cmd = "type _module_raw"
- out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False)
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False)
+ out, ec = res.output, res.exit_code
if regex is None:
regex = r".*%s" % os.path.basename(self.cmd)
@@ -1399,7 +1751,7 @@ def available(self, mod_name=None, extra_args=None):
if extra_args is None:
extra_args = []
# make hidden modules visible (requires Environment Modules 4.6.0)
- if StrictVersion(self.version) >= StrictVersion('4.6.0'):
+ if LooseVersion(self.version) >= LooseVersion('4.6.0'):
extra_args.append(self.SHOW_HIDDEN_OPTION)
return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args)
@@ -1417,23 +1769,53 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name):
# - line starts with 'setenv'
# - whitespace (spaces & tabs) around variable name
# - curly braces around value if it contain spaces
- value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name,
- var_name=var_name)
+ regex = re.compile(r'^setenv\s+%s\s+(?P.+)' % var_name, re.M)
+ value = self.get_value_from_modulefile(mod_name, regex, strict=False)
if value:
- value = value.strip('{}')
+ value = value.strip(' {}')
return value
+ def remove_module_path(self, path, set_mod_paths=True):
+ """
+ Remove specified module path (using 'module unuse').
+
+ :param path: path to remove from $MODULEPATH via 'unuse'
+ :param set_mod_paths: (re)set self.mod_paths
+ """
+ # remove module path via 'module use' and make sure self.mod_paths is synced
+ # Environment Modules <5.0 keeps track of how often a path was added via 'module use',
+ # so we need to check to make sure it's really removed
+ path = normalize_path(path)
+ while True:
+ try:
+ # Unuse the path that is actually present in the environment
+ module_path = next(p for p in curr_module_paths() if normalize_path(p) == path)
+ except StopIteration:
+ break
+ self.unuse(module_path)
+ if set_mod_paths:
+ self.set_mod_paths()
+
+ def update(self):
+ """Update after new modules were added."""
+
+ version = LooseVersion(self.version)
+ if build_option('update_modules_tool_cache') and version >= LooseVersion('5.3.0'):
+ out = self.run_module('cachebuild', return_stderr=True, check_output=False)
+
+ if self.testing:
+ return out
+
class Lmod(ModulesTool):
"""Interface to Lmod."""
NAME = "Lmod"
COMMAND = 'lmod'
COMMAND_ENVIRONMENT = 'LMOD_CMD'
- REQ_VERSION = '6.5.1'
- DEPR_VERSION = '7.0.0'
- REQ_VERSION_DEPENDS_ON = '7.6.1'
+ REQ_VERSION = '8.0.0'
+ DEPR_VERSION = '8.0.0'
VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s"
SHOW_HIDDEN_OPTION = '--show-hidden'
@@ -1453,11 +1835,11 @@ def __init__(self, *args, **kwargs):
setvar('LMOD_TERSE_DECORATIONS', 'no', verbose=False)
super(Lmod, self).__init__(*args, **kwargs)
- version = StrictVersion(self.version)
+ version = LooseVersion(self.version)
- self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON
+ self.supports_depends_on = True
# See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html
- if version >= '8.7.12':
+ if version >= LooseVersion('8.7.12'):
self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod')
else:
self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
@@ -1522,14 +1904,16 @@ def update(self):
if build_option('update_modules_tool_cache'):
spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider')
- cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']]
- self.log.debug("Running command '%s'..." % ' '.join(cmd))
+ cmd_list = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']]
+ cmd = ' '.join(cmd_list)
+ self.log.debug("Running command '%s'...", cmd)
- proc = subprocess_popen_text(cmd, env=os.environ)
- (stdout, stderr) = proc.communicate()
+ res = run_shell_cmd(cmd_list, env=os.environ, fail_on_error=False, use_bash=False, split_stderr=True,
+ hidden=True)
+ stdout, stderr = res.output, res.stderr
if stderr:
- raise EasyBuildError("An error occurred when running '%s': %s", ' '.join(cmd), stderr)
+ raise EasyBuildError("An error occurred when running '%s': %s", cmd, stderr)
if self.testing:
# don't actually update local cache when testing, just return the cache contents
@@ -1537,7 +1921,7 @@ def update(self):
else:
suffix = build_option('module_cache_suffix') or ''
cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT%s.lua' % suffix)
- self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd)))
+ self.log.debug("Updating Lmod spider cache %s with output from '%s'", cache_fp, cmd)
cache_dir = os.path.dirname(cache_fp)
if not os.path.exists(cache_dir):
mkdir(cache_dir, parents=True)
@@ -1612,13 +1996,9 @@ def module_wrapper_exists(self, mod_name):
Determine whether a module wrapper with specified name exists.
First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax).
"""
- res = None
-
- # first consider .modulerc.lua with Lmod 7.8 (or newer)
- if StrictVersion(self.version) >= StrictVersion('7.8'):
- mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$'
- res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua',
- mod_wrapper_regex_template=mod_wrapper_regex_template)
+ mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$'
+ res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua',
+ mod_wrapper_regex_template=mod_wrapper_regex_template)
# fall back to checking for .modulerc in Tcl syntax
if res is None:
@@ -1670,7 +2050,7 @@ def get_software_root(name, with_env_var=False):
return res
-def get_software_libdir(name, only_one=True, fs=None):
+def get_software_libdir(name, only_one=True, fs=None, full_path=False):
"""
Find library subdirectories for the specified software package.
@@ -1681,51 +2061,57 @@ def get_software_libdir(name, only_one=True, fs=None):
:param name: name of the software package
:param only_one: indicates whether only one lib path is expected to be found
:param fs: only retain library subdirs that contain one of the files in this list
+ :param full_path: Include the software root in the returned path, or just return the subfolder found
"""
lib_subdirs = ['lib', 'lib64']
root = get_software_root(name)
- res = []
- if root:
- for lib_subdir in lib_subdirs:
- lib_dir_path = os.path.join(root, lib_subdir)
- if os.path.exists(lib_dir_path):
- # take into account that lib64 could be a symlink to lib (or vice versa)
- # see https://github.com/easybuilders/easybuild-framework/issues/3139
- if any(os.path.samefile(lib_dir_path, os.path.join(root, x)) for x in res):
- _log.debug("%s is the same as one of the other paths, so skipping it", lib_dir_path)
-
- elif fs is None or any(os.path.exists(os.path.join(lib_dir_path, f)) for f in fs):
- _log.debug("Retaining library subdir '%s' (found at %s)", lib_subdir, lib_dir_path)
- res.append(lib_subdir)
-
- elif build_option('extended_dry_run'):
- res.append(lib_subdir)
- break
-
- # if no library subdir was found, return None
- if not res:
- return None
- if only_one:
- if len(res) == 1:
- res = res[0]
- else:
- if fs is None and len(res) == 2:
- # if both lib and lib64 were found, check if only one (exactly) has libraries;
- # this is needed for software with library archives in lib64 but other files/directories in lib
- lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]]
- has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res]
- if has_libs[0] and not has_libs[1]:
- return res[0]
- elif has_libs[1] and not has_libs[0]:
- return res[1]
-
- raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s",
- name, root, ', '.join(res))
- return res
- else:
+ if not root:
# return None if software package root could not be determined
return None
+ found_subdirs = []
+ for lib_subdir in lib_subdirs:
+ lib_dir_path = os.path.join(root, lib_subdir)
+ if os.path.exists(lib_dir_path):
+ # take into account that lib64 could be a symlink to lib (or vice versa)
+ # see https://github.com/easybuilders/easybuild-framework/issues/3139
+ if any(os.path.samefile(lib_dir_path, os.path.join(root, x)) for x in found_subdirs):
+ _log.debug("%s is the same as one of the other paths, so skipping it", lib_dir_path)
+
+ elif fs is None or any(os.path.exists(os.path.join(lib_dir_path, f)) for f in fs):
+ _log.debug("Retaining library subdir '%s' (found at %s)", lib_subdir, lib_dir_path)
+ found_subdirs.append(lib_subdir)
+
+ elif build_option('extended_dry_run'):
+ found_subdirs.append(lib_subdir)
+ break
+
+ # if no library subdir was found, return None
+ if not found_subdirs:
+ return None
+ if full_path:
+ res = [os.path.join(root, subdir) for subdir in found_subdirs]
+ else:
+ res = found_subdirs
+ if only_one:
+ if len(res) == 1:
+ res = res[0]
+ else:
+ if fs is None and len(res) == 2:
+ # if both lib and lib64 were found, check if only one (exactly) has libraries;
+ # this is needed for software with library archives in lib64 but other files/directories in lib
+ lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]]
+ has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob)
+ for subdir in found_subdirs]
+ if has_libs[0] and not has_libs[1]:
+ return res[0]
+ if has_libs[1] and not has_libs[0]:
+ return res[1]
+
+ raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s",
+ name, root, ', '.join(found_subdirs))
+ return res
+
def get_software_version_env_var_name(name):
"""Return name of environment variable for software root."""
@@ -1780,7 +2166,7 @@ def avail_modules_tools():
def modules_tool(mod_paths=None, testing=False):
"""
- Return interface to modules tool (environment modules (C, Tcl), or Lmod)
+ Return interface to modules tool (EnvironmentModules, Lmod, ...)
"""
# get_modules_tool might return none (e.g. if config was not initialized yet)
modules_tool = get_modules_tool()
diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py
index 99ee949e56..e8881275c9 100644
--- a/easybuild/tools/multidiff.py
+++ b/easybuild/tools/multidiff.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -291,8 +291,8 @@ def multidiff(base, files, colored=True):
offset -= 1
# construct the multi-diff based on the constructed dict
- for line_no in local_diff:
- for (line, filename) in local_diff[line_no]:
+ for line_no, line_infos in local_diff.items():
+ for (line, filename) in line_infos:
mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip())
return str(mdiff)
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index fd22e384e5..4068aa1ad6 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -51,16 +51,15 @@
from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog
from easybuild.base.fancylogger import setLogLevel
from easybuild.base.generaloption import GeneralOption
-from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, FETCH_STEP, EasyBlock
+from easybuild.framework.easyblock import MODULE_ONLY_STEPS, EXTRACT_STEP, FETCH_STEP, EasyBlock
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8
from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION
from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
-from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION
from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for
from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM
from easybuild.tools import LooseVersion, build_log, run # build_log should always stay there, to ensure EasyBuildLog
-from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
+from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, EasyBuildExit
from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror
from easybuild.tools.config import CHECKSUM_PRIORITY_CHOICES, DEFAULT_CHECKSUM_PRIORITY
from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES
@@ -68,25 +67,27 @@
from easybuild.tools.config import DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES
from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND
from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
-from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
+from easybuild.tools.config import DEFAULT_MAX_PARALLEL, DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS
+from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
-from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
-from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
+from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, MOD_SEARCH_PATH_HEADERS
+from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN, build_option
from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
from easybuild.tools.config import BuildOptions, ConfigurationVariables
+from easybuild.tools.config import PYTHON_SEARCH_PATH_TYPES, PYTHONPATH
from easybuild.tools.configobj import ConfigObj, ConfigObjError
from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
from easybuild.tools.docs import list_easyblocks, list_toolchains
from easybuild.tools.environment import restore_env, unset_env_vars
-from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc
-from easybuild.tools.filetools import move_file, which
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd
+from easybuild.tools.filetools import install_fake_vsc, move_file, which, is_parent_path
from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED
from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS
from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
@@ -98,15 +99,16 @@
from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
from easybuild.tools.modules import Lmod
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.robot import det_robot_path
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.package.utilities import avail_package_naming_schemes
from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
+from easybuild.tools.toolchain.toolchain import DEFAULT_SEARCH_PATH_CPP_HEADERS, DEFAULT_SEARCH_PATH_LINKER, SEARCH_PATH
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
from easybuild.tools.repository.repository import avail_repositories
-from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
-from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info
+from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
+from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info
+from easybuild.tools.utilities import flatten
from easybuild.tools.version import this_is_easybuild
@@ -125,14 +127,17 @@ def terminal_supports_colors(stream):
CONFIG_ENV_VAR_PREFIX = 'EASYBUILD'
XDG_CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), ".config"))
-XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc').split(os.pathsep)
-DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))]
+XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc/xdg').split(os.pathsep)
+DEFAULT_SYS_CFGFILES = [[f for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))]
+ for d in XDG_CONFIG_DIRS]
DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg')
DEFAULT_LIST_PR_STATE = GITHUB_PR_STATE_OPEN
DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED
DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC
+RPATH_DEFAULT = False if get_os_type() == DARWIN else True
+
_log = fancylogger.getLogger('options', fname=False)
@@ -212,7 +217,17 @@ class EasyBuildOptions(GeneralOption):
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
- DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:]
+ # https://specifications.freedesktop.org/basedir-spec/latest/
+ # says precedence should be
+ # XDG_CONFIG_HOME > 1st entry of XDG_CONFIG_DIRS > 2nd entry ...
+ # EasyBuild parses this list backwards, gives priority to last entry
+ DEFAULT_CONFIGFILES = flatten(DEFAULT_SYS_CFGFILES[::-1])
+ if 'XDG_CONFIG_DIRS' not in os.environ:
+ old_etc_location = os.path.join('/etc', 'easybuild.d')
+ if os.path.isdir(old_etc_location) and glob.glob(os.path.join(old_etc_location, '*.cfg')):
+ _log.deprecated(f"Using {old_etc_location} is deprecated. Please use "
+ "/etc/xdg/easybuild.d instead or add /etc to XDG_CONFIG_DIRS", '6.0')
+
if os.path.exists(DEFAULT_USER_CFGFILE):
DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE)
@@ -286,7 +301,7 @@ def basic_options(self):
'skip': ("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step",
- 'choice', 'store_or_None', SOURCE_STEP, 's', all_stops),
+ 'choice', 'store_or_None', EXTRACT_STEP, 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store', WARN, strictness_options),
})
@@ -341,19 +356,16 @@ def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
- all_deprecations = ('python2', 'Lmod6', 'easyconfig', 'toolchain')
+ all_deprecations = ('easyconfig', 'toolchain')
opts = OrderedDict({
- 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]",
- 'strlist', 'store', []),
'accept-eula-for': ("Accept EULA for specified software", 'strlist', 'store', []),
- 'add-dummy-to-minimal-toolchains': ("Include dummy toolchain in minimal toolchain searches "
- "[DEPRECATED, use --add-system-to-minimal-toolchains instead!]",
- None, 'store_true', False),
'add-system-to-minimal-toolchains': ("Include system toolchain in minimal toolchain searches",
None, 'store_true', False),
'allow-loaded-modules': ("List of software names for which to allow loaded modules in initial environment",
'strlist', 'store', DEFAULT_ALLOW_LOADED_MODULES),
+ 'allow-unresolved-templates': ("Don't error out when templates such as %(name)s in EasyConfigs "
+ "could not be resolved", None, 'store_true', False),
'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'allow-use-as-root-and-accept-consequences': ("Allow using of EasyBuild as root (NOT RECOMMENDED!)",
@@ -389,7 +401,7 @@ def override_options(self):
"for example: 3.5,5.0,7.2", 'strlist', 'extend', None),
'debug-lmod': ("Run Lmod modules tool commands in debug module", None, 'store_true', False),
'default-opt-level': ("Specify default optimisation level", 'choice', 'store', DEFAULT_OPT_LEVEL,
- Compiler.COMPILER_OPT_FLAGS),
+ Compiler.COMPILER_OPT_OPTIONS),
'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'detect-loaded-modules': ("Detect loaded EasyBuild-generated modules, act accordingly; "
@@ -410,6 +422,8 @@ def override_options(self):
'strlist', 'extend', None),
"extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig",
"urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}),
+ 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true',
+ False),
'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
"implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "
@@ -452,6 +466,7 @@ def override_options(self):
'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
None, 'store_true', False),
'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
+ 'keep-debug-symbols': ("Sets default value of debug toolchain option", None, 'store_true', False),
'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing",
None, 'store_true', True),
'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries",
@@ -460,6 +475,8 @@ def override_options(self):
None, 'store_true', True),
'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions",
'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS),
+ 'max-parallel': ("Specify maximum level of parallelism that should be used during build procedure",
+ 'int', 'store', DEFAULT_MAX_PARALLEL),
'minimal-build-env': ("Minimal build environment to define when using system toolchain, "
"specified as a comma-separated list that defines a mapping between name of "
"environment variable and its value separated by a colon (':')",
@@ -481,12 +498,18 @@ def override_options(self):
'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
"with fallback to basic colored output",
'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
- 'parallel': ("Specify (maximum) level of parallelism used during build procedure",
+ 'parallel': ("Specify level of parallelism that should be used during build procedure, "
+ "(bypasses auto-detection of number of available cores; "
+ "actual value is determined by this value + 'max_parallel' easyconfig parameter)",
'int', 'store', None),
'parallel-extensions-install': ("Install list of extensions in parallel (if supported)",
None, 'store_true', False),
'pre-create-installdir': ("Create installation directory before submitting build jobs",
None, 'store_true', True),
+ 'prefer-python-search-path': (("Prefer using specified environment variable when possible to specify where"
+ " Python packages were installed; see also "
+ "https://docs.easybuild.io/python-search-path"),
+ 'choice', 'store_or_None', PYTHONPATH, PYTHON_SEARCH_PATH_TYPES),
'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
None, 'store_true', False, 'p'),
'read-only-installdir': ("Set read-only permissions on installation directory after installation",
@@ -497,7 +520,7 @@ def override_options(self):
'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) "
"which must be linked in all installed binaries/libraries",
'strlist', 'extend', None),
- 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False),
+ 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', RPATH_DEFAULT),
'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None),
'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)",
None, 'store', None),
@@ -521,9 +544,12 @@ def override_options(self):
"Git commit to use for the target software build (robot capabilities are automatically disabled)",
None, 'store', None),
'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
+ 'strict-rpath-sanity-check': ("Perform strict RPATH sanity check, which involces unsetting "
+ "$LD_LIBRARY_PATH before checking whether all required libraries are found",
+ None, 'store_true', False),
'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",
None, 'store', None),
- 'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'),
+ 'trace': ("Provide more information in output to stdout on progress", None, 'store_true', True, 'T'),
'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file",
@@ -537,10 +563,6 @@ def override_options(self):
None, 'store_true', False),
'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents",
None, 'store_true', False),
- 'wait-on-lock': ("Wait for lock to be released; 0 implies no waiting (exit with an error if the lock "
- "already exists), non-zero value specified waiting interval [DEPRECATED: "
- "use --wait-on-lock-interval and --wait-on-lock-limit instead]",
- int, 'store_or_None', None),
'wait-on-lock-interval': ("Wait interval (in seconds) to use when waiting for existing lock to be removed",
int, 'store', DEFAULT_WAIT_ON_LOCK_INTERVAL),
'wait-on-lock-limit': ("Maximum amount of time (in seconds) to wait until lock is released (0 means no "
@@ -573,6 +595,12 @@ def config_options(self):
[DEFAULT_ENVVAR_USERS_MODULES]),
'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata "
"for external modules (INI format)", 'strlist', 'store', None),
+ 'failed-install-build-dirs-path': ("Location where build directories are copied if installation fails; "
+ "an empty value disables copying of build directories",
+ None, 'store', None, {'metavar': "PATH"}),
+ 'failed-install-logs-path': ("Location where log files are copied if installation fails; "
+ "an empty value disables copying of log files",
+ None, 'store', None, {'metavar': "PATH"}),
'hooks': ("Location of Python module with hook implementations", 'str', 'store', None),
'ignore-dirs': ("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
@@ -594,10 +622,13 @@ def config_options(self):
'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}),
'module-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules "
"(implies recursive unloading of modules).",
- None, 'store_true', False),
+ None, 'store_true', True),
'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)",
- None, 'store_true', False),
+ None, 'store_true', True),
'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS),
+ 'module-search-path-headers': ("Environment variable set by modules on load with search paths "
+ "to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS,
+ sorted(MOD_SEARCH_PATH_HEADERS.keys())),
'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX,
sorted(avail_module_generators().keys())),
'moduleclasses': (("Extend supported module classes "
@@ -624,6 +655,10 @@ def config_options(self):
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."),
'strlist', 'store', self.default_repositorypath),
+ 'search-path-cpp-headers': ("Search path used at build time for include directories", 'choice',
+ 'store', DEFAULT_SEARCH_PATH_CPP_HEADERS, [*SEARCH_PATH["cpp_headers"]]),
+ 'search-path-linker': ("Search path used at build time by the linker for libraries", 'choice',
+ 'store', DEFAULT_SEARCH_PATH_LINKER, [*SEARCH_PATH["linker"]]),
'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', mk_full_default_path('sourcepath')),
'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),
@@ -845,7 +880,7 @@ def job_options(self):
'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD),
'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0),
'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24),
- 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()),
+ 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', get_cwd()),
'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0),
'target-resource': ("Target resource for jobs", None, 'store', None),
})
@@ -916,7 +951,10 @@ def validate(self):
error_msgs.append(error_msg % (cuda_cc_regex.pattern, ', '.join(faulty_cuda_ccs)))
if error_msgs:
- raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs))
+ raise EasyBuildError(
+ "Found problems validating the options: %s", '\n'.join(error_msgs),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
@@ -941,10 +979,6 @@ def postprocess(self):
# set tmpdir
self.tmpdir = set_tmpdir(self.options.tmpdir)
- # early check for opt-in to installing extensions in parallel (experimental feature)
- if self.options.parallel_extensions_install:
- self.log.experimental("installing extensions in parallel")
-
# take --include options into account (unless instructed otherwise)
if self.with_include:
self._postprocess_include()
@@ -1004,15 +1038,20 @@ def _postprocess_optarch(self):
n_parts = len(optarch_parts)
map_char_cnts = [p.count(OPTARCH_MAP_CHAR) for p in optarch_parts]
if (n_parts > 1 and any(c != 1 for c in map_char_cnts)) or (n_parts == 1 and map_char_cnts[0] > 1):
- raise EasyBuildError("The optarch option has an incorrect syntax: %s", self.options.optarch)
+ raise EasyBuildError(
+ "The optarch option has an incorrect syntax: %s", self.options.optarch,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
# if there are options for different compilers, we set up a dict
if OPTARCH_MAP_CHAR in optarch_parts[0]:
optarch_dict = {}
for compiler, compiler_opt in [p.split(OPTARCH_MAP_CHAR) for p in optarch_parts]:
if compiler in optarch_dict:
- raise EasyBuildError("The optarch option contains duplicated entries for compiler %s: %s",
- compiler, self.options.optarch)
+ raise EasyBuildError(
+ "The optarch option contains duplicated entries for compiler %s: %s",
+ compiler, self.options.optarch, exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
optarch_dict[compiler] = compiler_opt
self.options.optarch = optarch_dict
@@ -1024,13 +1063,17 @@ def _postprocess_optarch(self):
def _postprocess_close_pr_reasons(self):
"""Postprocess --close-pr-reasons options"""
if self.options.close_pr_msg:
- raise EasyBuildError("Please either specify predefined reasons with --close-pr-reasons or " +
- "a custom message with--close-pr-msg")
+ raise EasyBuildError(
+ "Please either select a reason with --close-pr-reasons or add a custom message with--close-pr-msg",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
reasons = self.options.close_pr_reasons.split(',')
if any([reason not in VALID_CLOSE_PR_REASONS.keys() for reason in reasons]):
- raise EasyBuildError("Argument to --close-pr_reasons must be a comma separated list of valid reasons " +
- "among %s" % VALID_CLOSE_PR_REASONS.keys())
+ raise EasyBuildError(
+ "Argument to --close-pr_reasons must be a comma separated list of valid reasons among %s",
+ VALID_CLOSE_PR_REASONS.keys(), exit_code=EasyBuildExit.OPTION_ERROR
+ )
self.options.close_pr_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in reasons])
def _postprocess_list_prs(self):
@@ -1039,18 +1082,30 @@ def _postprocess_list_prs(self):
nparts = len(list_pr_parts)
if nparts > 3:
- raise EasyBuildError("Argument to --list-prs must be in the format 'state[,order[,direction]]")
+ raise EasyBuildError(
+ "Argument to --list-prs must be in the format 'state[,order[,direction]]",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
list_pr_state = list_pr_parts[0]
list_pr_order = list_pr_parts[1] if nparts > 1 else DEFAULT_LIST_PR_ORDER
list_pr_direc = list_pr_parts[2] if nparts > 2 else DEFAULT_LIST_PR_DIREC
if list_pr_state not in GITHUB_PR_STATES:
- raise EasyBuildError("1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES)
+ raise EasyBuildError(
+ "1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if list_pr_order not in GITHUB_PR_ORDERS:
- raise EasyBuildError("2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS)
+ raise EasyBuildError(
+ "2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if list_pr_direc not in GITHUB_PR_DIRECTIONS:
- raise EasyBuildError("3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS)
+ raise EasyBuildError(
+ "3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
self.options.list_prs = (list_pr_state, list_pr_order, list_pr_direc)
@@ -1072,41 +1127,62 @@ def _postprocess_checks(self):
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.include_easyblocks_from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
- raise EasyBuildError("Required support for using GitHub API is not available (see warnings)")
+ raise EasyBuildError(
+ "Required support for using GitHub API is not available (see warnings)",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# using Lua module syntax only makes sense when modules tool being used is Lmod
if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
error_msg = "Generating Lua module files requires Lmod as modules tool; "
mod_syntaxes = ', '.join(sorted(avail_module_generators().keys()))
error_msg += "use --module-syntax to specify a different module syntax to use (%s)" % mod_syntaxes
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
# check whether specified action --detect-loaded-modules is valid
if self.options.detect_loaded_modules not in LOADED_MODULES_ACTIONS:
error_msg = "Unknown action specified to --detect-loaded-modules: %s (known values: %s)"
- raise EasyBuildError(error_msg % (self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS)))
+ raise EasyBuildError(
+ error_msg, self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
- raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available")
+ raise EasyBuildError(
+ "Python 'keyring' module required for obtaining GitHub token is not available",
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
if self.options.github_user is None:
- raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token")
+ raise EasyBuildError(
+ "No GitHub user name provided, required for fetching GitHub token",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
token = fetch_github_token(self.options.github_user)
if token is None:
- raise EasyBuildError("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)
+ raise EasyBuildError(
+ "Failed to obtain required GitHub token for user '%s'", self.options.github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# make sure autopep8 is available when it needs to be
if self.options.dump_autopep8:
if not HAVE_AUTOPEP8:
- raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")
+ raise EasyBuildError(
+ "Python 'autopep8' module required to reformat dumped easyconfigs as requested",
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
# if a path is specified to --sysroot, it must exist
if self.options.sysroot:
if os.path.exists(self.options.sysroot):
self.log.info("Specified sysroot '%s' exists: OK", self.options.sysroot)
else:
- raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot)
+ raise EasyBuildError(
+ "Specified sysroot '%s' does not exist!", self.options.sysroot,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# 'software_commit' is specific to a particular software package and cannot be used with 'robot'
if self.options.software_commit:
@@ -1131,14 +1207,17 @@ def _ensure_abs_path(self, opt_name):
opt_val = getattr(self.options, opt_name)
if opt_val:
- if isinstance(opt_val, string_type):
+ if isinstance(opt_val, str):
setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val))
elif isinstance(opt_val, list):
abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val]
setattr(self.options, opt_name, abs_paths)
else:
error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)"
- raise EasyBuildError(error_msg, opt_name, type(opt_val))
+ raise EasyBuildError(
+ error_msg, opt_name, type(opt_val),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
def _postprocess_config(self):
"""Postprocessing of configuration options"""
@@ -1147,22 +1226,23 @@ def _postprocess_config(self):
# to avoid incorrect paths being used when EasyBuild changes the current working directory
# (see https://github.com/easybuilders/easybuild-framework/issues/3619);
# ensuring absolute paths for 'robot' is handled separately below,
- # because we need to be careful with the argument pass to --robot;
+ # because we need to be careful with the argument passed to --robot;
# note: repositorypath is purposely not listed here, because it's a special case:
# - the value could consist of a 2-tuple (, );
# - the could also specify the location of a *remote* (Git( repository,
# which can be done in variety of formats (git@:/), https://, etc.)
# (see also https://github.com/easybuilders/easybuild-framework/issues/3892);
- path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
- 'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
- 'robot_paths', 'sourcepath']
+ path_opt_names = ['buildpath', 'containerpath', 'failed_install_build_dirs_path', 'failed_install_logs_path',
+ 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software',
+ 'prefix', 'packagepath', 'robot_paths', 'sourcepath']
for opt_name in path_opt_names:
self._ensure_abs_path(opt_name)
if self.options.prefix is not None:
- # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
- # in the legacy-style configuration, repository is initialised in configuration file itself
+ # prefix applies to selected path configuration options;
+ # repository has to be reinitialised to take new repositorypath in account;
+ # in the legacy-style configuration, repository is initialised in configuration file itself;
path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath',
'sourcepath']
for dest in path_opts:
@@ -1187,18 +1267,21 @@ def _postprocess_config(self):
# which makes it susceptible to 'eating' the following argument/option;
# for example: with 'eb -r foo', 'foo' must be an existing directory (or 'eb foo -r' should be used);
# when multiple directories are specified, we deliberately do not enforce that all of them exist;
- # if a single argument is passed to --robot/-r that ends with '.eb' or '.yeb', we assume it's an easyconfig
+ # if a single argument is passed to --robot/-r that ends with '.eb' we assume it's an easyconfig
if len(self.options.robot) == 1:
robot_arg = self.options.robot[0]
if not os.path.isdir(robot_arg):
- if robot_arg.endswith(EB_FORMAT_EXTENSION) or robot_arg.endswith(YEB_FORMAT_EXTENSION):
+ if robot_arg.endswith(EB_FORMAT_EXTENSION):
info_msg = "Sole --robot argument %s is not an existing directory, "
info_msg += "promoting it to a stand-alone argument since it looks like an easyconfig file name"
self.log.info(info_msg, robot_arg)
self.args.append(robot_arg)
self.options.robot = []
else:
- raise EasyBuildError("Argument passed to --robot is not an existing directory: %s", robot_arg)
+ raise EasyBuildError(
+ "Argument passed to --robot is not an existing directory: %s", robot_arg,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# paths specified to --robot have preference over --robot-paths
# keep both values in sync if robot is enabled, which implies enabling dependency resolver
@@ -1220,6 +1303,20 @@ def _postprocess_config(self):
if self.options.inject_checksums or self.options.inject_checksums_to_json:
self.options.pre_create_installdir = False
+ # Prevent that build directories and logs for failed installations are copied to location for build directories
+ if self.options.buildpath and self.options.failed_install_logs_path:
+ if is_parent_path(self.options.buildpath, self.options.failed_install_logs_path):
+ raise EasyBuildError(
+ f"The --failed-install-logs-path ('{self.options.failed_install_logs_path}') "
+ f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')"
+ )
+ if self.options.buildpath and self.options.failed_install_build_dirs_path:
+ if is_parent_path(self.options.buildpath, self.options.failed_install_build_dirs_path):
+ raise EasyBuildError(
+ f"The --failed-install-build-dirs-path ('{self.options.failed_install_build_dirs_path}') "
+ f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')"
+ )
+
def _postprocess_list_avail(self):
"""Create all the additional info that can be requested (exit at the end)"""
msg = ''
@@ -1335,10 +1432,11 @@ def show_default_configfiles(self):
'',
"* user-level: %s" % os.path.join('${XDG_CONFIG_HOME:-$HOME/.config}', 'easybuild', 'config.cfg'),
" -> %s => %s" % (DEFAULT_USER_CFGFILE, ('not found', 'found')[os.path.exists(DEFAULT_USER_CFGFILE)]),
- "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc}', 'easybuild.d', '*.cfg'),
- " -> %s => %s" % (system_cfg_glob_paths, ', '.join(DEFAULT_SYS_CFGFILES) or "(no matches)"),
+ "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc/xdg}', 'easybuild.d', '*.cfg'),
+ " -> %s => %s" % (system_cfg_glob_paths, ', '.join(flatten(DEFAULT_SYS_CFGFILES)) or "(no matches)"),
'',
- "Default list of existing configuration files (%d): %s" % (found_cfgfile_cnt, found_cfgfile_list),
+ "Default list of existing configuration files (%d, most important last):" % found_cfgfile_cnt,
+ found_cfgfile_list,
]
return '\n'.join(lines)
@@ -1387,9 +1485,9 @@ def show_system_info(self):
'',
"* GPU:",
])
- for vendor in gpu_info:
+ for vendor, vendor_gpu in gpu_info.items():
lines.append(" -> %s" % vendor)
- for gpu, num in gpu_info[vendor].items():
+ for gpu, num in vendor_gpu.items():
lines.append(" -> %sx %s" % (num, gpu))
lines.extend([
@@ -1409,7 +1507,8 @@ def show_config(self):
# options that should never/always be printed
ignore_opts = ['show_config', 'show_full_config']
- include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', 'sourcepath']
+ include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths',
+ 'rpath', 'sourcepath']
cmdline_opts_dict = self.dict_by_prefix()
def reparse_cfg(args=None, withcfg=True):
@@ -1523,7 +1622,11 @@ def parse_options(args=None, with_include=True):
go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror,
with_include=with_include)
except EasyBuildError as err:
- raise EasyBuildError("Failed to parse configuration options: %s" % err)
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.OPTION_ERROR
+ raise EasyBuildError("Failed to parse configuration options: %s", err, exit_code=exit_code)
return eb_go
@@ -1533,12 +1636,15 @@ def check_options(options):
Check configuration options, some combinations are not allowed.
"""
if options.from_commit and options.from_pr:
- raise EasyBuildError("--from-commit and --from-pr should not be used together, pick one")
+ raise EasyBuildError(
+ "--from-commit and --from-pr should not be used together, pick one",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if options.include_easyblocks_from_commit and options.include_easyblocks_from_pr:
error_msg = "--include-easyblocks-from-commit and --include-easyblocks-from-pr "
error_msg += "should not be used together, pick one"
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
def check_root_usage(allow_use_as_root=False):
@@ -1572,6 +1678,7 @@ def check_included_multiple(included_easyblocks_from, source):
print_warning(warning_msg)
if options.include_easyblocks_from_pr or options.include_easyblocks_from_commit:
+ terse = build_option('terse')
if options.include_easyblocks:
# check if you are including the same easyblock twice
@@ -1582,7 +1689,10 @@ def check_included_multiple(included_easyblocks_from, source):
try:
easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr]
except ValueError:
- raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s")
+ raise EasyBuildError(
+ "Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
for easyblock_pr in easyblock_prs:
easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr)
@@ -1593,7 +1703,7 @@ def check_included_multiple(included_easyblocks_from, source):
included_easyblocks |= included_from_pr
for easyblock in included_from_pr:
- print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log)
+ print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log, silent=terse)
include_easyblocks(options.tmpdir, easyblocks_from_pr)
@@ -1606,7 +1716,8 @@ def check_included_multiple(included_easyblocks_from, source):
check_included_multiple(included_from_commit, "commit %s" % easyblock_commit)
for easyblock in included_from_commit:
- print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log)
+ print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit),
+ log=log, silent=terse)
include_easyblocks(options.tmpdir, easyblocks_from_commit)
@@ -1677,12 +1788,18 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r
try:
from_prs = [int(x) for x in eb_go.options.from_pr]
except ValueError:
- raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
+ raise EasyBuildError(
+ "Argument to --from-pr must be a comma separated list of PR #s.",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
try:
review_pr = (lambda x: int(x) if x else None)(eb_go.options.review_pr)
except ValueError:
- raise EasyBuildError("Argument to --review-pr must be an integer PR #.")
+ raise EasyBuildError(
+ "Argument to --review-pr must be an integer PR #.",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
@@ -1857,7 +1974,10 @@ def parse_external_modules_metadata(cfgs):
paths.extend(res)
else:
# if there are no matches, we report an error to avoid silently ignores faulty paths
- raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", cfg)
+ raise EasyBuildError(
+ "Specified path for file with external modules metadata does not exist: %s", cfg,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
cfgs = paths
# use external modules metadata configuration files that are available by default, unless others are specified
@@ -1889,7 +2009,10 @@ def parse_external_modules_metadata(cfgs):
try:
parsed_metadata.merge(ConfigObj(cfg))
except ConfigObjError as err:
- raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err)
+ raise EasyBuildError(
+ "Failed to parse %s with external modules metadata: %s", cfg, err,
+ exit_code=EasyBuildExit.MODULE_ERROR
+ )
known_metadata_keys = ['name', 'prefix', 'version']
unknown_keys = {}
@@ -1902,7 +2025,7 @@ def parse_external_modules_metadata(cfgs):
unknown_keys.setdefault(mod, []).append(key)
for key in ['name', 'version']:
- if isinstance(entry.get(key), string_type):
+ if isinstance(entry.get(key), str):
entry[key] = [entry[key]]
_log.debug("Transformed external module metadata value %s for %s into a single-value list: %s",
key, mod, entry[key])
@@ -1910,14 +2033,17 @@ def parse_external_modules_metadata(cfgs):
# if both names and versions are available, lists must be of same length
names, versions = entry.get('name'), entry.get('version')
if names is not None and versions is not None and len(names) != len(versions):
- raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: "
- "names: %s; versions: %s", mod, names, versions)
+ raise EasyBuildError(
+ "Different length for lists of names/versions in metadata for external module %s: ; "
+ "names: %s; versions: %s", mod, names, versions,
+ exit_code=EasyBuildExit.MODULE_ERROR
+ )
if unknown_keys:
error_msg = "Found metadata entries with unknown keys:"
for mod in sorted(unknown_keys.keys()):
error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(unknown_keys[mod])))
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MODULE_ERROR)
_log.debug("External modules metadata: %s", parsed_metadata)
return parsed_metadata
@@ -1968,8 +2094,9 @@ def set_tmpdir(tmpdir=None, raise_error=False):
fd, tmptest_file = tempfile.mkstemp()
os.close(fd)
os.chmod(tmptest_file, 0o700)
- if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False,
- stream_output=False, with_hooks=False, with_sysroot=False):
+ res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False,
+ with_hooks=False)
+ if res.exit_code != EasyBuildExit.SUCCESS:
msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
msg += "This can cause problems in the build process, consider using --tmpdir."
if raise_error:
@@ -2002,6 +2129,7 @@ def opts_dict_to_eb_opts(args_dict):
:return: a list of strings representing command-line options for the 'eb' command
"""
+ allow_multiple_calls = ['amend', 'try-amend']
_log.debug("Converting dictionary %s to argument list" % args_dict)
args = []
for arg in sorted(args_dict):
@@ -2011,14 +2139,18 @@ def opts_dict_to_eb_opts(args_dict):
prefix = '--'
option = prefix + str(arg)
value = args_dict[arg]
- if isinstance(value, (list, tuple)):
- value = ','.join(str(x) for x in value)
- if value in [True, None]:
+ if str(arg) in allow_multiple_calls:
+ if not isinstance(value, (list, tuple)):
+ value = [value]
+ args.extend(option + '=' + str(x) for x in value)
+ elif value in [True, None]:
args.append(option)
elif value is False:
args.append('--disable-' + option[2:])
elif value is not None:
+ if isinstance(value, (list, tuple)):
+ value = ','.join(str(x) for x in value)
args.append(option + '=' + str(value))
_log.debug("Converted dictionary %s to argument list %s" % (args_dict, args))
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index a8f98480f9..8439d8f08b 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# #
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,6 +33,7 @@
"""
import functools
from collections import OrderedDict
+import sys
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
@@ -328,9 +329,7 @@ def print_checks(checks_data):
if use_rich():
console = Console()
- # don't use console.print, which causes SyntaxError in Python 2
- console_print = getattr(console, 'print') # noqa: B009
- console_print('')
+ console.print('')
for section in checks_data:
section_checks = checks_data[section]
@@ -382,11 +381,25 @@ def print_checks(checks_data):
lines.append('')
if use_rich():
- console_print(table)
+ console.print(table)
else:
print('\n'.join(lines))
+def print_error(error_msg, rich_highlight=True):
+ """
+ Print error message, using a Rich Console instance if possible.
+ Newlines before/after message are automatically added.
+
+ :param rich_highlight: boolean indicating whether automatic highlighting by Rich should be enabled
+ """
+ if use_rich():
+ console = Console(stderr=True)
+ console.print('\n\n' + error_msg + '\n', highlight=rich_highlight)
+ else:
+ sys.stderr.write('\n' + error_msg + '\n\n')
+
+
# this constant must be defined at the end, since functions used as values need to be defined
PROGRESS_BAR_TYPES = {
PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar,
diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
index 033687fac1..d106a46554 100644
--- a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
+++ b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- vim: set fileencoding=utf-8
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
index 7a53a86101..5136a5d89e 100644
--- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
+++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py
index 8683ae8d37..fbc503be72 100644
--- a/easybuild/tools/package/package_naming_scheme/pns.py
+++ b/easybuild/tools/package/package_naming_scheme/pns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py
index bf55fdcff1..5f55983f05 100644
--- a/easybuild/tools/package/utilities.py
+++ b/easybuild/tools/package/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,13 +39,13 @@
import pprint
from easybuild.base import fancylogger
+from easybuild.base.wrapper import create_base_metaclass
from easybuild.tools.config import PKG_TOOL_FPM, PKG_TYPE_RPM, Singleton
from easybuild.tools.config import build_option, get_package_naming_scheme, log_path
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import change_dir, which
from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme
-from easybuild.tools.py2vs3 import create_base_metaclass
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.utilities import get_subclasses, import_available_modules
@@ -145,7 +145,7 @@ def package_with_fpm(easyblock):
])
cmd = ' '.join(cmdlist)
_log.debug("The flattened cmdlist looks like: %s", cmd)
- run_cmd(cmdlist, log_all=True, simple=True, shell=False)
+ run_shell_cmd(cmdlist, use_bash=False)
_log.info("Created %s package(s) in %s", pkgtype, workdir)
diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py
index 3ab31e1efc..94e971e7a6 100644
--- a/easybuild/tools/parallelbuild.py
+++ b/easybuild/tools/parallelbuild.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,6 +33,7 @@
* Toon Willems (Ghent University)
* Kenneth Hoste (Ghent University)
* Stijn De Weirdt (Ghent University)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
"""
import math
import os
@@ -43,8 +44,9 @@
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_repository, get_repositorypath
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
-from easybuild.tools.job.backend import job_backend
+from easybuild.tools.job.backend import job_backend, JobBackend
from easybuild.tools.repository.repository import init_repository
@@ -56,7 +58,8 @@ def _to_key(dep):
return ActiveMNS().det_full_module_name(dep)
-def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
+def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', testing=False,
+ prepare_first=True, tweak_map=None, try_opts=''):
"""
Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
Return list of jobs submitted.
@@ -68,11 +71,14 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
:param build_command: build command to use
:param easyconfigs: list of easyconfig files
:param output_dir: output directory
+ :param testing: If `True`, skip actual job submission
:param prepare_first: prepare by runnning fetch step first for each easyconfig
+ :param tweak_map: Mapping from tweaked to original easyconfigs
+ :param try_opts: --try-* options to pass if the easyconfig is tweaked
"""
_log.info("going to build these easyconfigs in parallel: %s", [os.path.basename(ec['spec']) for ec in easyconfigs])
- active_job_backend = job_backend()
+ active_job_backend = JobBackend() if testing else job_backend()
if active_job_backend is None:
raise EasyBuildError("Can not use --job if no job backend is available.")
@@ -92,12 +98,17 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
- if prepare_first:
+ if prepare_first and not testing:
prepare_easyconfig(easyconfig)
+ # convert to to avoid needing a shared tmpdir
+ spec = easyconfig['spec']
+ if spec in (tweak_map or {}):
+ spec = tweak_map[spec] + try_opts
+
# the new job will only depend on already submitted jobs
- _log.info("creating job for ec: %s" % os.path.basename(easyconfig['spec']))
- new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir)
+ _log.info("creating job for ec: %s using %s" % (os.path.basename(easyconfig['spec']), spec))
+ new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir, spec=spec)
# filter out dependencies marked as external modules
deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)]
@@ -115,24 +126,27 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
active_job_backend.complete()
- return jobs
+ return build_command if testing else jobs
-def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
+def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True, tweak_map=None):
"""
Submit jobs.
:param ordered_ecs: list of easyconfigs, in the order they should be processed
:param cmd_line_opts: list of command line options (in 'longopt=value' form)
:param testing: If `True`, skip actual job submission
:param prepare_first: prepare by runnning fetch step first for each easyconfig
+ :param tweak_map: Mapping from tweaked to original easyconfigs
"""
- curdir = os.getcwd()
+ curdir = get_cwd()
- # regex pattern for options to ignore (help options can't reach here)
+ # regex patterns for options to ignore and tweak options (help options can't reach here)
ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$')
+ try_opts_re = re.compile('^--try-.*$')
# generate_cmd_line returns the options in form --longopt=value
opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])]
+ try_opts = [o for o in cmd_line_opts if try_opts_re.match(o.split('=')[0])]
# add --disable-job to make sure the submitted job doesn't submit a job itself,
# resulting in an infinite cycle of jobs;
@@ -142,6 +156,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
# compose string with command line options, properly quoted and with '%' characters escaped
opts_str = ' '.join(opts).replace('%', '%%')
+ try_opts_str = ' ' + ' '.join(try_opts).replace('%', '%%')
eb_cmd = build_option('job_eb_cmd')
@@ -153,12 +168,11 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
_log.info("Command template for jobs: %s", command)
if testing:
_log.debug("Skipping actual submission of jobs since testing mode is enabled")
- return command
- else:
- return build_easyconfigs_in_parallel(command, ordered_ecs, prepare_first=prepare_first)
+ return build_easyconfigs_in_parallel(command, ordered_ecs, testing=testing, prepare_first=prepare_first,
+ tweak_map=tweak_map, try_opts=try_opts_str)
-def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
+def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build', spec=''):
"""
Creates a job to build a *single* easyconfig.
@@ -166,6 +180,7 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
+ :param spec: untweaked easyconfig name with optional --try-* options
returns the job
"""
@@ -182,7 +197,7 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
- 'spec': easyconfig['spec'],
+ 'spec': spec or easyconfig['spec'],
}
# just use latest build stats
diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py
index 5d45e11dc3..c0cc7a61f4 100644
--- a/easybuild/tools/py2vs3/__init__.py
+++ b/easybuild/tools/py2vs3/__init__.py
@@ -1,5 +1,5 @@
#
-# Copyright 2019-2024 Ghent University
+# Copyright 2019-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -24,38 +24,23 @@
#
import sys
-# all functionality provided by the py2 and py3 modules is made available via the easybuild.tools.py2vs3 namespace
-if sys.version_info[0] >= 3:
- from easybuild.tools.py2vs3.py3 import * # noqa
-else:
- from easybuild.tools.py2vs3.py2 import * # noqa
+from easybuild.base import fancylogger
+from easybuild.base.wrapper import create_base_metaclass # noqa
-# based on six's 'with_metaclass' function
-# see also https://stackoverflow.com/questions/18513821/python-metaclass-understanding-the-with-metaclass
-def create_base_metaclass(base_class_name, metaclass, *bases):
- """Create new class with specified metaclass based on specified base class(es)."""
- return metaclass(base_class_name, bases, {})
+# all functionality provided by the py3 modules is made available via the easybuild.tools.py2vs3 namespace
+from easybuild.tools.py2vs3.py3 import * # noqa
+
+
+_log = fancylogger.getLogger('py2vs3', fname=False)
+_log.deprecated("Using py2vs3 is deprecated, since EasyBuild no longer runs on Python 2.", '6.0')
def python2_is_deprecated():
"""
- Print warning when using Python 2, since the support for running EasyBuild with it is deprecated.
+ Exit with an error when using Python 2, since EasyBuild does not support it.
+ We preserve the function name here in here EB5, to maintain the API, even though it now exits.
"""
if sys.version_info[0] == 2:
- full_py_ver = '.'.join(str(x) for x in sys.version_info[:3])
- warning_lines = [
- "Running EasyBuild with Python v2.x is deprecated, found Python v%s." % full_py_ver,
- "Support for running EasyBuild with Python v2.x will be removed in EasyBuild v5.0.",
- '',
- "It is strongly recommended to start using Python v3.x for running EasyBuild,",
- "see https://docs.easybuild.io/en/latest/Python-2-3-compatibility.html for more information.",
- ]
- max_len = max(len(x) for x in warning_lines)
- for i in range(len(warning_lines)):
- line_len = len(warning_lines[i])
- warning_lines[i] = '!!! ' + warning_lines[i] + ' ' * (max_len - line_len) + ' !!!'
- max_len = max(len(x) for x in warning_lines)
- warning_lines.insert(0, '!' * max_len)
- warning_lines.append('!' * max_len)
- sys.stderr.write('\n\n' + '\n'.join(warning_lines) + '\n\n\n')
+ sys.stderr.write('\n\nEasyBuild v5.0+ is not compatible with Python v2. Use Python >= 3.6.\n\n\n')
+ sys.exit(1)
diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py
deleted file mode 100644
index b52f55a1a2..0000000000
--- a/easybuild/tools/py2vs3/py2.py
+++ /dev/null
@@ -1,129 +0,0 @@
-#
-# Copyright 2019-2024 Ghent University
-#
-# This file is part of EasyBuild,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/easybuilders/easybuild
-#
-# EasyBuild is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation v2.
-#
-# EasyBuild is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with EasyBuild. If not, see .
-#
-"""
-Functionality to facilitate keeping code compatible with Python 2 & Python 3.
-
-Implementations for Python 2.
-
-Authors:
-
-* Kenneth Hoste (Ghent University)
-"""
-# these are not used here, but imported from here in other places
-import ConfigParser as configparser # noqa
-import imp
-import json
-import os
-import subprocess
-import time
-import urllib2 as std_urllib # noqa
-from collections import Mapping, OrderedDict # noqa
-from HTMLParser import HTMLParser # noqa
-from string import letters as ascii_letters # noqa
-from string import lowercase as ascii_lowercase # noqa
-from StringIO import StringIO # noqa
-from urllib import urlencode # noqa
-from urllib2 import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa
-
-# Use the safe version. In Python 3.2+ this is the default already
-ConfigParser = configparser.SafeConfigParser
-
-
-# reload function (built-in in Python 2)
-reload = reload # noqa: F821
-
-# string type that can be used in 'isinstance' calls
-string_type = basestring
-
-# trivial wrapper for json.loads (Python 3 version is less trivial)
-json_loads = json.loads
-
-
-def load_source(filename, path):
- """Load Python module"""
- return imp.load_source(filename, path)
-
-
-def subprocess_popen_text(cmd, **kwargs):
- """Call subprocess.Popen with specified named arguments."""
- kwargs.setdefault('stderr', subprocess.PIPE)
- return subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs)
-
-
-def subprocess_terminate(proc, timeout):
- """Terminate the subprocess if it hasn't finished after the given timeout"""
- res = None
- for pipe in (proc.stdout, proc.stderr, proc.stdin):
- if pipe:
- pipe.close()
- while timeout > 0:
- res = proc.poll()
- if res is not None:
- break
- delay = min(timeout, 0.1)
- time.sleep(delay)
- timeout -= delay
- if res is None:
- proc.terminate()
-
-
-# Wrapped in exec to avoid invalid syntax warnings for Python 3
-exec('''
-def raise_with_traceback(exception_class, message, traceback):
- """Raise exception of specified class with given message and traceback."""
- raise exception_class, message, traceback # noqa: E999
-''')
-
-
-def extract_method_name(method_func):
- """Extract method name from lambda function."""
- return '_'.join(method_func.func_code.co_names)
-
-
-def mk_wrapper_baseclass(metaclass):
-
- class WrapperBase(object):
- """
- Wrapper class that provides proxy access to an instance of some internal instance.
- """
- __metaclass__ = metaclass
- __wraps__ = None
-
- return WrapperBase
-
-
-def sort_looseversions(looseversions):
- """Sort list of (values including) LooseVersion instances."""
- # with Python 2, we can safely use 'sorted' on LooseVersion instances
- # (but we can't in Python 3, see https://bugs.python.org/issue14894)
- return sorted(looseversions)
-
-
-def makedirs(name, mode=0o777, exist_ok=False):
- try:
- os.makedirs(name, mode)
- except OSError:
- if not exist_ok or not os.path.isdir(name):
- raise
diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py
index 62e7a93723..01c5a3a397 100644
--- a/easybuild/tools/py2vs3/py3.py
+++ b/easybuild/tools/py2vs3/py3.py
@@ -1,5 +1,5 @@
#
-# Copyright 2019-2024 Ghent University
+# Copyright 2019-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,10 +34,8 @@
# these are not used here, but imported from here in other places
import configparser # noqa
import json
-import subprocess
import sys
import urllib.request as std_urllib # noqa
-from collections import OrderedDict # noqa
from collections.abc import Mapping # noqa
from functools import cmp_to_key
from importlib.util import spec_from_file_location, module_from_spec
@@ -61,10 +59,14 @@
except ImportError:
HAVE_DISTUTILS = False
+from easybuild.base.wrapper import mk_wrapper_baseclass # noqa
+from easybuild.tools.run import subprocess_popen_text, subprocess_terminate # noqa
+
# string type that can be used in 'isinstance' calls
string_type = str
+# note: also available in easybuild.tools.filetools, should be imported from there!
def load_source(filename, path):
"""Load file as Python module"""
spec = spec_from_file_location(filename, path)
@@ -85,24 +87,6 @@ def json_loads(body):
return json.loads(body)
-def subprocess_popen_text(cmd, **kwargs):
- """Call subprocess.Popen in text mode with specified named arguments."""
- # open stdout/stderr in text mode in Popen when using Python 3
- kwargs.setdefault('stderr', subprocess.PIPE)
- return subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, **kwargs)
-
-
-def subprocess_terminate(proc, timeout):
- """Terminate the subprocess if it hasn't finished after the given timeout"""
- try:
- proc.communicate(timeout=timeout)
- except subprocess.TimeoutExpired:
- for pipe in (proc.stdout, proc.stderr, proc.stdin):
- if pipe:
- pipe.close()
- proc.terminate()
-
-
def raise_with_traceback(exception_class, message, traceback):
"""Raise exception of specified class with given message and traceback."""
raise exception_class(message).with_traceback(traceback)
@@ -113,17 +97,6 @@ def extract_method_name(method_func):
return '_'.join(method_func.__code__.co_names)
-def mk_wrapper_baseclass(metaclass):
-
- class WrapperBase(object, metaclass=metaclass):
- """
- Wrapper class that provides proxy access to an instance of some internal instance.
- """
- __wraps__ = None
-
- return WrapperBase
-
-
def safe_cmp_looseversions(v1, v2):
"""Safe comparison function for two (values containing) LooseVersion instances."""
diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py
index b18c3c6159..0a40cce578 100644
--- a/easybuild/tools/repository/filerepo.py
+++ b/easybuild/tools/repository/filerepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -43,7 +43,6 @@
from easybuild.framework.easyconfig.easyconfig import EasyConfig
from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION
-from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION, is_yeb_format
from easybuild.framework.easyconfig.tools import stats_to_str
from easybuild.tools.filetools import copy_file, mkdir, read_file, write_file
from easybuild.tools.repository.repository import Repository
@@ -85,14 +84,8 @@ def add_easyconfig(self, cfg, name, version, stats, previous):
# create directory for eb file
full_path = os.path.join(self.wc, self.subdir, name)
- yeb_format = is_yeb_format(cfg, None)
- if yeb_format:
- extension = YEB_FORMAT_EXTENSION
- prefix = "buildstats: ["
-
- else:
- extension = EB_FORMAT_EXTENSION
- prefix = "buildstats = ["
+ extension = EB_FORMAT_EXTENSION
+ prefix = "buildstats = ["
# destination
dest = os.path.join(full_path, "%s-%s%s" % (name, version, extension))
@@ -109,10 +102,10 @@ def add_easyconfig(self, cfg, name, version, stats, previous):
if previous:
statstxt = statscomment + statsprefix + '\n'
for entry in previous + [stats]:
- statstxt += stats_to_str(entry, isyeb=yeb_format) + ',\n'
+ statstxt += stats_to_str(entry) + ',\n'
statstxt += statssuffix
else:
- statstxt = statscomment + statsprefix + stats_to_str(stats, isyeb=yeb_format) + statssuffix
+ statstxt = statscomment + statsprefix + stats_to_str(stats) + statssuffix
txt += statstxt
write_file(dest, txt)
diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py
index feaf8cb2ad..1e343c2375 100644
--- a/easybuild/tools/repository/gitrepo.py
+++ b/easybuild/tools/repository/gitrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py
index 83a7890cc3..a3e678df4a 100644
--- a/easybuild/tools/repository/hgrepo.py
+++ b/easybuild/tools/repository/hgrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py
index e84c9d3cc5..23f4f53cb8 100644
--- a/easybuild/tools/repository/repository.py
+++ b/easybuild/tools/repository/repository.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,7 +38,6 @@
"""
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import get_subclasses, import_available_modules
_log = fancylogger.getLogger('repository', fname=False)
@@ -159,10 +158,10 @@ def init_repository(repository, repository_path):
inited_repo = None
if isinstance(repository, Repository):
inited_repo = repository
- elif isinstance(repository, string_type):
+ elif isinstance(repository, str):
repo = avail_repositories().get(repository)
try:
- if isinstance(repository_path, string_type):
+ if isinstance(repository_path, str):
inited_repo = repo(repository_path)
elif isinstance(repository_path, (tuple, list)) and len(repository_path) <= 2:
inited_repo = repo(*repository_path)
diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py
index 79f593b1ee..63e04b6b89 100644
--- a/easybuild/tools/repository/svnrepo.py
+++ b/easybuild/tools/repository/svnrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -46,7 +46,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import remove_dir
+from easybuild.tools.filetools import get_cwd, remove_dir
from easybuild.tools.repository.filerepo import FileRepository
from easybuild.tools.utilities import only_if_module_is_available
@@ -145,7 +145,7 @@ def stage_file(self, path):
"""
if self.client and not self.client.status(path)[0].is_versioned:
# add it to version control
- self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, os.getcwd()))
+ self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, get_cwd()))
self.client.add(path)
def add_easyconfig(self, cfg, name, version, stats, previous_stats):
diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py
index 5189769929..1d0546de8e 100644
--- a/easybuild/tools/robot.py
+++ b/easybuild/tools/robot.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -43,9 +43,9 @@
from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig
from easybuild.framework.easyconfig.easyconfig import robot_find_easyconfig, verify_easyconfig_filename
from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit
from easybuild.tools.config import build_option
-from easybuild.tools.filetools import det_common_path_prefix, search_file
+from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file
from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.utilities import flatten, nub
@@ -242,7 +242,7 @@ def dry_run(easyconfigs, modtool, short=False):
:param short: use short format for overview: use a variable for common prefixes
"""
lines = []
- if build_option('robot_path') is None:
+ if build_option('robot') is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
@@ -325,7 +325,7 @@ def raise_error_missing_deps(missing_deps, extra_msg=None):
error_msg = "Missing dependencies: %s" % mod_names
if extra_msg:
error_msg += ' (%s)' % extra_msg
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MISSING_DEPENDENCY)
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True):
@@ -491,7 +491,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con
"""
search_path = build_option('robot_path')
if not search_path:
- search_path = [os.getcwd()]
+ search_path = [get_cwd()]
extra_search_paths = build_option('search_paths')
# If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths
if extra_search_paths and consider_extra_paths:
diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index f8d034f981..2e118e0860 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,36 +35,45 @@
* Toon Willems (Ghent University)
* Ward Poelmans (Ghent University)
"""
-import contextlib
import functools
+import inspect
+import locale
import os
import re
-import signal
+import shlex
+import shutil
+import string
import subprocess
-import sys
import tempfile
import time
+from collections import namedtuple
from datetime import datetime
-import easybuild.tools.asyncprocess as asyncprocess
+# import deprecated functions so they can still be imported from easybuild.tools.run, for now
+from easybuild._deprecated import check_async_cmd, check_log_for_errors, complete_cmd, extract_errors_from_log # noqa
+from easybuild._deprecated import get_output_from_process, parse_cmd_output, parse_log_for_error # noqa
+from easybuild._deprecated import run_cmd, run_cmd_qa # noqa
+
+try:
+ # get_native_id is only available in Python >= 3.8
+ from threading import get_native_id as get_thread_id
+except ImportError:
+ # get_ident is available in Python >= 3.3
+ from threading import get_ident as get_thread_id
+
from easybuild.base import fancylogger
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since
-from easybuild.tools.config import ERROR, IGNORE, WARN, build_option
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
+from easybuild.tools.build_log import dry_run_msg, time_str_since
+from easybuild.tools.config import build_option
from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook
-from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.utilities import nub, trace_msg
+from easybuild.tools.output import COLOR_RED, COLOR_YELLOW, colorize, print_error
+from easybuild.tools.utilities import trace_msg
_log = fancylogger.getLogger('run', fname=False)
-errors_found_in_log = 0
-
-# default strictness level
-strictness = WARN
-
-
-CACHED_COMMANDS = [
+CACHED_COMMANDS = (
"sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X)
"sysctl -n hw.memsize", # used in get_total_memory (OS X)
"sysctl -n hw.ncpu", # used in get_avail_core_count (OS X)
@@ -73,10 +82,99 @@
"type module", # used in ModulesTool.check_module_function
"type _module_raw", # used in EnvironmentModules.check_module_function
"ulimit -u", # used in det_parallelism
-]
+)
+
+RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir',
+ 'out_file', 'err_file', 'cmd_sh', 'thread_id', 'task_id'))
+RunShellCmdResult.__doc__ = """A namedtuple that represents the result of a call to run_shell_cmd,
+with the following fields:
+- cmd: the command that was executed;
+- exit_code: the exit code of the command (zero if it was successful, non-zero if not);
+- output: output of the command (stdout+stderr combined, only stdout if stderr was caught separately);
+- stderr: stderr output produced by the command, if caught separately (None otherwise);
+- work_dir: the working directory of the command;
+- out_file: path to file with output of command (stdout+stderr combined, only stdout if stderr was caught separately);
+- err_file: path to file with stderr output of command, if caught separately (None otherwise);
+- cmd_sh: path to script to set up interactive shell with environment in which command was executed;
+- thread_id: thread ID of command that was executed (None unless asynchronous mode was enabled for running command);
+- task_id: task ID of command, if it was specified (None otherwise);
+"""
+
+
+class RunShellCmdError(BaseException):
+
+ def __init__(self, cmd_result, caller_info, *args, **kwargs):
+ """Constructor for RunShellCmdError."""
+ self.cmd = cmd_result.cmd
+ self.cmd_name = os.path.basename(self.cmd.split(' ')[0])
+ self.exit_code = cmd_result.exit_code
+ self.work_dir = cmd_result.work_dir
+ self.output = cmd_result.output
+ self.out_file = cmd_result.out_file
+ self.stderr = cmd_result.stderr
+ self.err_file = cmd_result.err_file
+ self.cmd_sh = cmd_result.cmd_sh
+
+ self.caller_info = caller_info
+
+ msg = f"Shell command '{self.cmd_name}' failed!"
+ super(RunShellCmdError, self).__init__(msg, *args, **kwargs)
+
+ def print(self):
+ """
+ Report failed shell command for this RunShellCmdError instance
+ """
+
+ def pad_4_spaces(msg, color=None):
+ padded_msg = ' ' * 4 + msg
+ if color:
+ return colorize(padded_msg, color)
+ else:
+ return padded_msg
+
+ caller_file_name, caller_line_nr, caller_function_name = self.caller_info
+ called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})"
+
+ error_info = [
+ colorize("ERROR: Shell command failed!", COLOR_RED),
+ pad_4_spaces(f"full command -> {self.cmd}"),
+ pad_4_spaces(f"exit code -> {self.exit_code}"),
+ pad_4_spaces(f"called from -> {called_from_info}"),
+ pad_4_spaces(f"working directory -> {self.work_dir}"),
+ ]
+
+ if self.out_file is not None:
+ # if there's no separate file for error/warnings, then out_file includes both stdout + stderr
+ out_info_msg = "output (stdout + stderr)" if self.err_file is None else "output (stdout) "
+ error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}", color=COLOR_YELLOW))
+ if self.err_file is not None:
+ error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}", color=COLOR_YELLOW))
-def run_cmd_cache(func):
+ if self.cmd_sh is not None:
+ error_info.append(pad_4_spaces(f"interactive shell script -> {self.cmd_sh}", color=COLOR_YELLOW))
+
+ print_error('\n'.join(error_info), rich_highlight=False)
+
+
+def raise_run_shell_cmd_error(cmd_res):
+ """
+ Raise RunShellCmdError for failed shell command, after collecting additional caller info
+ """
+
+ # figure out where failing command was run
+ # need to go 3 levels down:
+ # 1) this function
+ # 2) run_shell_cmd function
+ # 3) run_shell_cmd_cache decorator
+ # 4) actual caller site
+ frameinfo = inspect.getouterframes(inspect.currentframe())[3]
+ caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function)
+
+ raise RunShellCmdError(cmd_res, caller_info)
+
+
+def run_shell_cmd_cache(func):
"""Function decorator to cache (and retrieve cached) results of running commands."""
cache = {}
@@ -84,9 +182,9 @@ def run_cmd_cache(func):
def cache_aware_func(cmd, *args, **kwargs):
"""Retrieve cached result of selected commands, or run specified and collect & cache result."""
# cache key is combination of command and input provided via stdin
- key = (cmd, kwargs.get('inp', None))
+ key = (cmd, kwargs.get('stdin', None))
# fetch from cache if available, cache it if it's not, but only on cmd strings
- if isinstance(cmd, string_type) and key in cache:
+ if isinstance(cmd, str) and key in cache:
_log.debug("Using cached value for command '%s': %s", cmd, cache[key])
return cache[key]
else:
@@ -102,725 +200,488 @@ def cache_aware_func(cmd, *args, **kwargs):
return cache_aware_func
-def get_output_from_process(proc, read_size=None, asynchronous=False):
+def fileprefix_from_cmd(cmd, allowed_chars=False):
"""
- Get output from running process (that was opened with subprocess.Popen).
+ Simplify the cmd to only the allowed_chars we want in a filename
- :param proc: process to get output from
- :param read_size: number of bytes of output to read (if None: read all output)
- :param asynchronous: get output asynchronously
+ :param cmd: the cmd (string)
+ :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-")
"""
+ if not allowed_chars:
+ allowed_chars = f"{string.ascii_letters}{string.digits}_-"
- if asynchronous:
- # e=False is set to avoid raising an exception when command has completed;
- # that's needed to ensure we get all output,
- # see https://github.com/easybuilders/easybuild-framework/issues/3593
- output = asyncprocess.recv_some(proc, e=False)
- elif read_size:
- output = proc.stdout.read(read_size)
- else:
- output = proc.stdout.read()
+ return ''.join([c for c in cmd if c in allowed_chars])
- # need to be careful w.r.t. encoding since we want to obtain a string value,
- # and the output may include non UTF-8 characters
- # * in Python 2, .decode() returns a value of type 'unicode',
- # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors)
- # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read()
- output = str(output.decode('ascii', 'ignore'))
- return output
+def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file):
+ """
+ Create helper scripts for specified command in specified directory:
+ - env.sh which can be sourced to define environment in which command was run;
+ - cmd.sh to create interactive (bash) shell session with working directory and environment,
+ and with the command in shell history;
+ """
+ # Save environment variables in env.sh which can be sourced to restore environment
+ if env is None:
+ env = os.environ.copy()
+
+ # Decode any declared bash functions
+ proc = subprocess.Popen('declare -f', stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+ env=env, shell=True, executable='bash')
+ (bash_functions, _) = proc.communicate()
+
+ env_fp = os.path.join(tmpdir, 'env.sh')
+ with open(env_fp, 'w') as fid:
+ # unset all environment variables in current environment first to start from a clean slate;
+ # we need to be careful to filter out functions definitions, so first undefine those
+ fid.write('\n'.join([
+ 'for var in $(compgen -e); do',
+ ' unset "$var"',
+ 'done',
+ ]) + '\n')
+ # also unset any bash functions
+ fid.write('\n'.join([
+ 'for func in $(compgen -A function); do',
+ ' if [[ $func != _* ]]; then',
+ ' unset -f "$func"',
+ ' fi',
+ 'done',
+ ]) + '\n')
+
+ # excludes bash functions (environment variables ending with %)
+ fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(env.items())
+ if not key.endswith('%')) + '\n')
+
+ fid.write(bash_functions.decode(errors='ignore') + '\n')
+
+ fid.write('\n\nPS1="eb-shell> "')
+
+ # define $EB_CMD_OUT_FILE to contain path to file with command output
+ fid.write(f'\nEB_CMD_OUT_FILE="{out_file}"')
+ # define $EB_CMD_ERR_FILE to contain path to file with command stderr output (if available)
+ if err_file:
+ fid.write(f'\nEB_CMD_ERR_FILE="{err_file}"')
+
+ # also change to working directory (to ensure that working directory is correct for interactive bash shell)
+ fid.write(f'\ncd "{work_dir}"')
+
+ # reset shell history to only include executed command
+ fid.write(f'\nhistory -s {shlex.quote(cmd_str)}')
+
+ # Make script that sets up bash shell with specified environment and working directory
+ cmd_fp = os.path.join(tmpdir, 'cmd.sh')
+ with open(cmd_fp, 'w') as fid:
+ fid.write('#!/usr/bin/env bash\n')
+ fid.write('# Run this script to set up a shell environment that EasyBuild used to run the shell command\n')
+ fid.write('\n'.join([
+ 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )',
+ f'echo "# Shell for the command: \'"{shlex.quote(cmd_str)}"\'"',
+ 'echo "# Use command history, exit to stop"',
+ # using -i to force interactive shell, so env.sh is also sourced when -c is used to run commands
+ 'bash --rcfile $EB_SCRIPT_DIR/env.sh -i "$@"',
+ ]))
+ os.chmod(cmd_fp, 0o775)
+
+ return cmd_fp
+
+
+def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns):
+ """
+ Private helper function to try and answer questions raised in interactive shell commands.
+ """
+ match_found = False
+
+ space_line_break_pattern = r'[\s\n]+'
+ space_line_break_regex = re.compile(space_line_break_pattern)
+
+ stdout_end = stdout.decode(errors='ignore')[-1000:]
+ for question, answers in qa_patterns:
+ # first replace hard spaces by regular spaces, since they would mess up the join/split below
+ question = question.replace(r'\ ', ' ')
+ # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks,
+ # and allow extra whitespace at the end
+ question = space_line_break_pattern.join(space_line_break_regex.split(question)) + r'[\s\n]*$'
+ _log.debug(f"Checking for question pattern '{question}'...")
+ regex = re.compile(question.encode())
+ res = regex.search(stdout)
+ if res:
+ _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout_end}")
+ # if answer is specified as a list, we take the first item as current answer,
+ # and add it to the back of the list (so we cycle through answers)
+ if isinstance(answers, list):
+ answer = answers.pop(0)
+ answers.append(answer)
+ elif isinstance(answers, str):
+ answer = answers
+ else:
+ raise EasyBuildError(f"Unknown type of answers encountered for question ({question}): {answers}")
+ # answer may need to be completed via pattern extracted from question
+ _log.debug(f"Raw answer for question pattern '{question}': {answer}")
+ answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()}
+ answer += '\n'
+ _log.info(f"Found match for question pattern '{question}', replying with: {answer}")
-@run_cmd_cache
-def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
- force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
- with_hooks=True, with_sysroot=True):
+ try:
+ os.write(proc.stdin.fileno(), answer.encode())
+ except OSError as err:
+ raise EasyBuildError("Failed to answer question raised by interactive command: %s", err)
+
+ match_found = True
+ break
+ else:
+ _log.debug(f"No match for question pattern '{question}' at end of stdout: {stdout_end}")
+ else:
+ _log.info("No match found for question patterns, considering question wait patterns")
+ # if no match was found among question patterns,
+ # take into account patterns for non-questions (qa_wait_patterns)
+ for pattern in qa_wait_patterns:
+ # first replace hard spaces by regular spaces, since they would mess up the join/split below
+ pattern = pattern.replace(r'\ ', ' ')
+ # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks,
+ # and allow extra whitespace at the end
+ pattern = space_line_break_pattern.join(space_line_break_regex.split(pattern)) + r'[\s\n]*$'
+ regex = re.compile(pattern.encode())
+ _log.debug(f"Checking for question wait pattern '{pattern}'...")
+ if regex.search(stdout):
+ _log.info(f"Found match for question wait pattern '{pattern}'")
+ _log.debug(f"Found match for question wait pattern '{pattern}' at end of stdout: {stdout_end}")
+ match_found = True
+ break
+ else:
+ _log.debug(f"No match for question wait pattern '{pattern}' at end of stdout: {stdout_end}")
+ else:
+ _log.info("No match found for question wait patterns")
+ _log.debug(f"No match found in question (wait) patterns at end of stdout: {stdout_end}")
+
+ return match_found
+
+
+@run_shell_cmd_cache
+def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
+ hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True,
+ output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True,
+ qa_patterns=None, qa_wait_patterns=None, qa_timeout=100):
"""
- Run specified command (in a subshell)
- :param cmd: command to run
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param inp: the input given to the command via stdin
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param log_output: indicate whether all output of command should be logged to a separate temporary logfile
- :param path: path to execute the command in; current working directory is used if unspecified
- :param force_in_dry_run: force running the command during dry run
- :param verbose: include message on running the command in dry run output
- :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
- :param trace: print command being executed as part of trace output
- :param stream_output: enable streaming command output to stdout
- :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
+ Run specified (interactive) shell command, and capture output + exit code.
+
+ :param fail_on_error: fail on non-zero exit code (enabled by default)
+ :param split_stderr: split of stderr from stdout output
+ :param stdin: input to be sent to stdin (nothing if set to None)
+ :param env: environment to use to run command (if None, inherit current process environment)
+ :param hidden: do not show command in terminal output (when using --trace, or with --extended-dry-run / -x)
+ :param in_dry_run: also run command in dry run mode
+ :param verbose_dry_run: show that command is run in dry run mode (overrules 'hidden')
+ :param work_dir: working directory to run command in (current working directory if None)
+ :param use_bash: execute command through bash shell (enabled by default)
+ :param output_file: collect command output in temporary output file
+ :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None)
+ :param asynchronous: indicate that command is being run asynchronously
+ :param task_id: task ID for specified shell command (included in return value)
:param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
- :param with_sysroot: prepend sysroot to exec_cmd (if defined)
+ :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers
+ :param qa_wait_patterns: list of strings with patterns for non-questions
+ :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question
+
+ :return: Named tuple with:
+ - output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise
+ - exit_code: exit code of command (integer)
+ - stderr: stderr output if split_stderr is enabled, None otherwise
"""
- cwd = os.getcwd()
+ def to_cmd_str(cmd):
+ """
+ Helper function to create string representation of specified command.
+ """
+ if isinstance(cmd, str):
+ cmd_str = cmd.strip()
+ elif isinstance(cmd, list):
+ cmd_str = ' '.join(cmd)
+ else:
+ raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}")
- if isinstance(cmd, string_type):
- cmd_msg = cmd.strip()
- elif isinstance(cmd, list):
- cmd_msg = ' '.join(cmd)
- else:
- raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
-
- if shell is None:
- shell = True
- if isinstance(cmd, list):
- raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! "
- "Note that all elements of the list but the first are treated as arguments "
- "to the shell and NOT to the command to be executed!")
-
- if log_output or (trace and build_option('trace')):
- # collect output of running command in temporary log file, if desired
- fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
- os.close(fd)
- try:
- cmd_log = open(cmd_log_fn, 'w')
- except IOError as err:
- raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
- _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
- else:
- cmd_log_fn, cmd_log = None, None
+ return cmd_str
- # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
- if stream_output is None and build_option('logtostdout'):
- _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
- stream_output = True
+ # make sure that qa_patterns is a list of 2-tuples (not a dict, or something else)
+ if qa_patterns:
+ if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns):
+ raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!")
- if stream_output:
- print_msg("(streaming) output for command '%s':" % cmd_msg)
+ interactive = bool(qa_patterns)
- start_time = datetime.now()
- if trace:
- trace_txt = "running command:\n"
- trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
- trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
- if inp:
- trace_txt += "\t[input: %s]\n" % inp
- trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
- trace_msg(trace_txt + '\t' + cmd_msg)
-
- # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
- if not force_in_dry_run and build_option('extended_dry_run'):
- if path is None:
- path = cwd
- if verbose:
- dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
- dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
-
- # make sure we get the type of the return value right
- if simple:
- return True
- else:
- # output, exit code
- return ('', 0)
+ if qa_wait_patterns is None:
+ qa_wait_patterns = []
+ # keep path to current working dir in case we need to come back to it
try:
- if path:
- os.chdir(path)
-
- _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
- except OSError as err:
- _log.warning("Failed to change to %s: %s" % (path, err))
- _log.info("running cmd %s in non-existing directory, might fail!", cmd)
-
- if cmd_log:
- cmd_log.write("# output for command: %s\n\n" % cmd_msg)
-
- exec_cmd = "/bin/bash"
-
- # if EasyBuild is configured to use an alternate sysroot,
- # we should also run shell commands using the bash shell provided in there,
- # since /bin/bash may not be compatible with the alternate sysroot
- if with_sysroot:
- sysroot = build_option('sysroot')
- if sysroot:
- sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash')
- if os.path.exists(sysroot_bin_bash):
- exec_cmd = sysroot_bin_bash
-
- if not shell:
- if isinstance(cmd, list):
- exec_cmd = None
- cmd.insert(0, '/usr/bin/env')
- elif isinstance(cmd, string_type):
- cmd = '/usr/bin/env %s' % cmd
- else:
- raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
+ initial_work_dir = os.getcwd()
+ except FileNotFoundError:
+ raise EasyBuildError(CWD_NOTFOUND_ERROR)
- _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd)
+ if work_dir is None:
+ work_dir = initial_work_dir
if with_hooks:
hooks = load_hooks(build_option('hooks'))
- hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})
- if isinstance(hook_res, string_type):
+ kwargs = {
+ 'interactive': interactive,
+ 'work_dir': work_dir,
+ }
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs)
+ if hook_res:
cmd, old_cmd = hook_res, cmd
_log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
- _log.info('running cmd: %s ' % cmd)
- try:
- proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
- except OSError as err:
- raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
-
- if inp:
- proc.stdin.write(inp.encode())
- proc.stdin.close()
+ cmd_str = to_cmd_str(cmd)
+ thread_id = None
if asynchronous:
- return (proc, cmd, cwd, start_time, cmd_log)
- else:
- return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple,
- regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks)
+ thread_id = get_thread_id()
+ _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}")
+ # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
+ if stream_output is None and build_option('logtostdout'):
+ _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled")
+ stream_output = True
-def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
- """
- Check status of command that was started asynchronously.
+ # temporary output file(s) for command output, along with helper scripts
+ if output_file:
+ toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output')
+ os.makedirs(toptmpdir, exist_ok=True)
+ cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0]))
+ tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-')
- :param proc: subprocess.Popen instance representing asynchronous command
- :param cmd: command being run
- :param owd: original working directory
- :param start_time: start time of command (datetime instance)
- :param cmd_log: log file to print command output to
- :param fail_on_error: raise EasyBuildError when command exited with an error
- :param output_read_size: number of bytes to read from output
- :param output: already collected output for this command
-
- :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
- """
- # use small read size, to avoid waiting for a long time until sufficient output is produced
- if output_read_size:
- if not isinstance(output_read_size, int) or output_read_size < 0:
- raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)")
- add_out = get_output_from_process(proc, read_size=output_read_size)
- _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
- output += add_out
-
- exit_code = proc.poll()
- if exit_code is None:
- _log.debug("Asynchronous command '%s' still running..." % cmd)
- done = False
+ _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {tmpdir}')
+
+ cmd_out_fp = os.path.join(tmpdir, 'out.txt')
+ _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}')
+ if split_stderr:
+ cmd_err_fp = os.path.join(tmpdir, 'err.txt')
+ _log.info(f'run_shell_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}')
+ else:
+ cmd_err_fp = None
+
+ cmd_sh = create_cmd_scripts(cmd_str, work_dir, env, tmpdir, cmd_out_fp, cmd_err_fp)
else:
- _log.debug("Asynchronous command '%s' completed!", cmd)
- output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
- simple=False, trace=False, log_ok=fail_on_error)
- done = True
-
- res = {
- 'done': done,
- 'exit_code': exit_code,
- 'output': output,
- }
- return res
+ tmpdir, cmd_out_fp, cmd_err_fp, cmd_sh = None, None, None, None
+ interactive_msg = 'interactive ' if interactive else ''
-def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
- regexp=True, stream_output=None, trace=True, output='', with_hook=True):
- """
- Complete running of command represented by passed subprocess.Popen instance.
+ # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled)
+ if not in_dry_run and build_option('extended_dry_run'):
+ if not hidden or verbose_dry_run:
+ silent = build_option('silent')
+ msg = f" running {interactive_msg}shell command \"{cmd_str}\"\n"
+ msg += f" (in {work_dir})"
+ dry_run_msg(msg, silent=silent)
- :param proc: subprocess.Popen instance representing running command
- :param cmd: command being run
- :param owd: original working directory
- :param start_time: start time of command (datetime instance)
- :param cmd_log: log file to print command output to
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param stream_output: enable streaming command output to stdout
- :param trace: print command being executed as part of trace output
- :param with_hook: trigger post run_shell_cmd hooks (if defined)
- """
- # use small read size when streaming output, to make it stream more fluently
- # read size should not be too small though, to avoid too much overhead
- if stream_output:
- read_size = 128
+ return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir,
+ out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh,
+ thread_id=thread_id, task_id=task_id)
+
+ start_time = datetime.now()
+ if not hidden:
+ _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, tmpdir, thread_id, interactive=interactive)
+
+ # use bash as shell instead of the default /bin/sh used by subprocess.run
+ # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh)
+ # stick to None (default value) when not running command via a shell
+ if use_bash:
+ bash = shutil.which('bash')
+ _log.info(f"Path to bash that will be used to run shell commands: {bash}")
+ executable, shell = bash, True
else:
- read_size = 1024 * 8
+ executable, shell = None, False
- stdouterr = output
+ stderr_handle = subprocess.PIPE if split_stderr else subprocess.STDOUT
+ stdin_handle = subprocess.PIPE if stdin or qa_patterns else subprocess.DEVNULL
- try:
- ec = proc.poll()
- while ec is None:
- # need to read from time to time.
- # - otherwise the stdout/stderr buffer gets filled and it all stops working
- output = get_output_from_process(proc, read_size=read_size)
- if cmd_log:
- cmd_log.write(output)
- if stream_output:
- sys.stdout.write(output)
- stdouterr += output
- ec = proc.poll()
-
- # read remaining data (all of it)
- output = get_output_from_process(proc)
- finally:
- proc.stdout.close()
-
- if cmd_log:
- cmd_log.write(output)
- cmd_log.close()
- if stream_output:
- sys.stdout.write(output)
- stdouterr += output
-
- if with_hook:
- hooks = load_hooks(build_option('hooks'))
- run_hook_kwargs = {
- 'exit_code': ec,
- 'output': stdouterr,
- 'work_dir': os.getcwd(),
- }
- run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+ log_msg = f"Running {interactive_msg}shell command '{cmd_str}' in {work_dir}"
+ if thread_id:
+ log_msg += f" (via thread with ID {thread_id})"
+ _log.info(log_msg)
- if trace:
- trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr_handle, stdin=stdin_handle,
+ cwd=work_dir, env=env, shell=shell, executable=executable)
- try:
- os.chdir(owd)
- except OSError as err:
- raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err)
+ # 'input' value fed to subprocess.run must be a byte sequence
+ if stdin:
+ stdin = stdin.encode()
- return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
+ if stream_output or qa_patterns:
+ # enable non-blocking access to stdout, stderr, stdin
+ for channel in (proc.stdout, proc.stdin, proc.stderr):
+ if channel is not None:
+ os.set_blocking(channel.fileno(), False)
+ if stdin:
+ proc.stdin.write(stdin)
+ proc.stdin.flush()
+ if not qa_patterns:
+ proc.stdin.close()
-def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
- maxhits=50, trace=True):
- """
- Run specified interactive command (in a subshell)
- :param cmd: command to run
- :param qa: dictionary which maps question to answers
- :param no_qa: list of patters that are not questions
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param std_qa: dictionary which maps question regex patterns to answers
- :param path: path to execute the command is; current working directory is used if unspecified
- :param maxhits: maximum number of cycles (seconds) without being able to find a known question
- :param trace: print command being executed as part of trace output
- """
- cwd = os.getcwd()
+ exit_code = None
+ stdout, stderr = b'', b''
+ check_interval_secs = 0.1
+ time_no_match = 0
+ prev_stdout = ''
+
+ while exit_code is None:
+ # collect output line by line, while checking for questions to answer (if qa_patterns is provided)
+ for line in iter(proc.stdout.readline, b''):
+ _log.debug(f"Captured stdout: {line.decode(errors='ignore').rstrip()}")
+ stdout += line
+
+ # note: we assume that there won't be any questions in stderr output
+ if split_stderr:
+ for line in iter(proc.stderr.readline, b''):
+ stderr += line
+
+ if qa_patterns:
+ # only check for question patterns if additional output is available
+ # compared to last time a question was answered;
+ # use empty list of question patterns if no extra output (except for whitespace) is available
+ # we do always need to check for wait patterns though!
+ active_qa_patterns = qa_patterns if stdout.strip() != prev_stdout else []
+
+ if _answer_question(stdout, proc, active_qa_patterns, qa_wait_patterns):
+ time_no_match = 0
+ prev_stdout = stdout.strip()
+ else:
+ # this will only run if the for loop above was *not* stopped by the break statement
+ time_no_match += check_interval_secs
+ if time_no_match > qa_timeout:
+ error_msg = "No matching questions found for current command output, "
+ error_msg += f"giving up after {qa_timeout} seconds!"
+ raise EasyBuildError(error_msg)
+ _log.debug(f"{time_no_match:0.1f} seconds without match in output of interactive shell command")
+
+ time.sleep(check_interval_secs)
+
+ exit_code = proc.poll()
+
+ # collect last bit of output once processed has exited
+ for line in iter(proc.stdout.readline, b''):
+ _log.debug(f"Captured stdout: {line.decode(errors='ignore').rstrip()}")
+ stdout += line
+ if split_stderr:
+ stderr += proc.stderr.read() or b''
+ else:
+ (stdout, stderr) = proc.communicate(input=stdin)
- if not isinstance(cmd, string_type) and len(cmd) > 1:
- # We use shell=True and hence we should really pass the command as a string
- # When using a list then every element past the first is passed to the shell itself, not the command!
- raise EasyBuildError("The command passed must be a string!")
+ # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out)
+ # getpreferredencoding normally gives 'utf-8' but can be ASCII (ANSI_X3.4-1968)
+ # for Python 3.6 and older with LC_ALL=C
+ encoding = locale.getpreferredencoding(False)
+ output = stdout.decode(encoding, 'ignore')
+ stderr = stderr.decode(encoding, 'ignore') if split_stderr else None
- if log_all or (trace and build_option('trace')):
- # collect output of running command in temporary log file, if desired
- fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
- os.close(fd)
+ # store command output to temporary file(s)
+ if output_file:
try:
- cmd_log = open(cmd_log_fn, 'w')
+ with open(cmd_out_fp, 'w') as fp:
+ fp.write(output)
+ if split_stderr:
+ with open(cmd_err_fp, 'w') as fp:
+ fp.write(stderr)
except IOError as err:
- raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
- _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
+ raise EasyBuildError(f"Failed to dump command output to temporary file: {err}")
+
+ res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr,
+ work_dir=work_dir, out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh,
+ thread_id=thread_id, task_id=task_id)
+
+ # always log command output
+ cmd_name = cmd_str.split(' ')[0]
+ if split_stderr:
+ _log.info(f"Output of '{cmd_name} ...' shell command (stdout only):\n{res.output}")
+ _log.info(f"Warnings and errors of '{cmd_name} ...' shell command (stderr only):\n{res.stderr}")
else:
- cmd_log_fn, cmd_log = None, None
+ _log.info(f"Output of '{cmd_name} ...' shell command (stdout + stderr):\n{res.output}")
- start_time = datetime.now()
- if trace:
- trace_txt = "running interactive command:\n"
- trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
- trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
- trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
- trace_msg(trace_txt + '\t' + cmd.strip())
-
- # early exit in 'dry run' mode, after printing the command that would be run
- if build_option('extended_dry_run'):
- if path is None:
- path = cwd
- dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
- dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
- if cmd_log:
- cmd_log.close()
- if simple:
- return True
- else:
- # output, exit code
- return ('', 0)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ _log.info(f"Shell command completed successfully (see output above): {cmd_str}")
+ else:
+ _log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}")
+ if fail_on_error:
+ raise_run_shell_cmd_error(res)
+ # check that we still are in a sane environment after command execution
+ # safeguard against commands that deleted the work dir or missbehaving filesystems
try:
- if path:
- os.chdir(path)
-
- _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
- except OSError as err:
- _log.warning("Failed to change to %s: %s" % (path, err))
- _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
-
- # Part 1: process the QandA dictionary
- # given initial set of Q and A (in dict), return dict of reg. exp. and A
- #
- # make regular expression that matches the string with
- # - replace whitespace
- # - replace newline
-
- def escape_special(string):
- return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
-
- split = r'[\s\n]+'
- regSplit = re.compile(r"" + split)
-
- def process_QA(q, a_s):
- splitq = [escape_special(x) for x in regSplit.split(q)]
- regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
- # add optional split at the end
- for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
- a_s[i] += '\n'
- regQ = re.compile(r"" + regQtxt)
- if regQ.search(q):
- return (a_s, regQ)
- else:
- raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
-
- def check_answers_list(answers):
- """Make sure we have a list of answers (as strings)."""
- if isinstance(answers, string_type):
- answers = [answers]
- elif not isinstance(answers, list):
- if cmd_log:
- cmd_log.close()
- raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
- question, type(answers), answers)
- # list is manipulated when answering matching question, so return a copy
- return answers[:]
-
- new_qa = {}
- _log.debug("new_qa: ")
- for question, answers in qa.items():
- answers = check_answers_list(answers)
- (answers, regQ) = process_QA(question, answers)
- new_qa[regQ] = answers
- _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
-
- new_std_qa = {}
- if std_qa:
- for question, answers in std_qa.items():
- regQ = re.compile(r"" + question + r"[\s\n]*$")
- answers = check_answers_list(answers)
- for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
- answers[i] += '\n'
- new_std_qa[regQ] = answers
- _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
-
- new_no_qa = []
- if no_qa:
- # simple statements, can contain wildcards
- new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
-
- _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
-
- # Part 2: Run the command and answer questions
- # - this needs asynchronous stdout
-
- hooks = load_hooks(build_option('hooks'))
- run_hook_kwargs = {
- 'interactive': True,
- 'work_dir': os.getcwd(),
- }
- hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
- if isinstance(hook_res, string_type):
- cmd, old_cmd = hook_res, cmd
- _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')",
- RUN_SHELL_CMD, cmd, old_cmd)
-
- # # Log command output
- if cmd_log:
- cmd_log.write("# output for interactive command: %s\n\n" % cmd)
-
- # Make sure we close the proc handles and the cmd_log file
- @contextlib.contextmanager
- def get_proc():
+ os.getcwd()
+ except FileNotFoundError:
+ _log.warning(
+ f"Shell command `{cmd_str}` completed successfully but left the system in an unknown working directory. "
+ f"Changing back to initial working directory: {initial_work_dir}"
+ )
try:
- proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT,
- stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash')
+ os.chdir(initial_work_dir)
except OSError as err:
- if cmd_log:
- cmd_log.close()
- raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
- try:
- yield proc
- finally:
- if proc.stdout:
- proc.stdout.close()
- if proc.stdin:
- proc.stdin.close()
- if cmd_log:
- cmd_log.close()
-
- with get_proc() as proc:
- ec = proc.poll()
- stdout_err = ''
- old_len_out = -1
- hit_count = 0
-
- while ec is None:
- # need to read from time to time.
- # - otherwise the stdout/stderr buffer gets filled and it all stops working
- try:
- out = get_output_from_process(proc, asynchronous=True)
-
- if cmd_log:
- cmd_log.write(out)
- stdout_err += out
- # recv_some used by get_output_from_process for getting asynchronous output may throw exception
- except (IOError, Exception) as err:
- _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
- out = None
-
- hit = False
- for question, answers in new_qa.items():
- res = question.search(stdout_err)
- if out and res:
- fa = answers[0] % res.groupdict()
- # cycle through list of answers
- last_answer = answers.pop(0)
- answers.append(last_answer)
- _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
-
- _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
- asyncprocess.send_all(proc, fa)
- hit = True
- break
- if not hit:
- for question, answers in new_std_qa.items():
- res = question.search(stdout_err)
- if out and res:
- fa = answers[0] % res.groupdict()
- # cycle through list of answers
- last_answer = answers.pop(0)
- answers.append(last_answer)
- _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
-
- _log.debug("run_cmd_qa answer %s std question %s out %s",
- fa, question.pattern, stdout_err[-50:])
- asyncprocess.send_all(proc, fa)
- hit = True
- break
- if not hit:
- if len(stdout_err) > old_len_out:
- old_len_out = len(stdout_err)
- else:
- noqa = False
- for r in new_no_qa:
- if r.search(stdout_err):
- _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
- noqa = True
- if not noqa:
- hit_count += 1
- else:
- hit_count = 0
- else:
- hit_count = 0
-
- if hit_count > maxhits:
- # explicitly kill the child process before exiting
- try:
- os.killpg(proc.pid, signal.SIGKILL)
- os.kill(proc.pid, signal.SIGKILL)
- except OSError as err:
- _log.debug("run_cmd_qa exception caught when killing child process: %s", err)
- _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
- raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
- cmd, maxhits, stdout_err[-500:])
-
- # the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
- time.sleep(1)
- ec = proc.poll()
-
- # Process stopped. Read all remaining data
- try:
- if proc.stdout:
- out = get_output_from_process(proc)
- stdout_err += out
- if cmd_log:
- cmd_log.write(out)
- except IOError as err:
- _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
+ raise EasyBuildError(f"Failed to return to {initial_work_dir} after executing command `{cmd_str}`: {err}")
- run_hook_kwargs.update({
- 'interactive': True,
- 'exit_code': ec,
- 'output': stdout_err,
- })
- run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
-
- if trace:
- trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+ if with_hooks:
+ run_hook_kwargs = {
+ 'exit_code': res.exit_code,
+ 'interactive': interactive,
+ 'output': res.output,
+ 'stderr': res.stderr,
+ 'work_dir': res.work_dir,
+ }
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
- try:
- os.chdir(cwd)
- except OSError as err:
- raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
+ if not hidden:
+ time_since_start = time_str_since(start_time)
+ trace_msg(f"command completed: exit {res.exit_code}, ran in {time_since_start}")
- return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
+ return res
-def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
+def _cmd_trace_msg(cmd, start_time, work_dir, stdin, tmpdir, thread_id, interactive=False):
"""
- Parse command output and construct return value.
- :param cmd: executed command
- :param stdouterr: combined stdout/stderr of executed command
- :param ec: exit code of executed command
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param log_all: always log command output and exit code
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- """
- if strictness == IGNORE:
- check_ec = False
- fail_on_error_match = False
- elif strictness == WARN:
- check_ec = True
- fail_on_error_match = False
- elif strictness == ERROR:
- check_ec = True
- fail_on_error_match = True
- else:
- raise EasyBuildError("invalid strictness setting: %s", strictness)
-
- # allow for overriding the regexp setting
- if not regexp:
- fail_on_error_match = False
-
- if ec and (log_all or log_ok):
- # We don't want to error if the user doesn't care
- if check_ec:
- raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr)
- else:
- _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
- elif not ec:
- if log_all:
- _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
- else:
- _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
-
- # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
- if fail_on_error_match or regexp:
- res = parse_log_for_error(stdouterr, regexp, stdout=False)
- if res:
- errors = "\n\t" + "\n\t".join([r[0] for r in res])
- error_str = "error" if len(res) == 1 else "errors"
- if fail_on_error_match:
- raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
- else:
- _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
- len(res), error_str, cmd, errors)
-
- if simple:
- if ec:
- # If the user does not care -> will return true
- return not check_ec
- else:
- return True
- else:
- # Because we are not running in simple mode, we return the output and ec to the user
- return (stdouterr, ec)
+ Helper function to construct and print trace message for command being run
-
-def parse_log_for_error(txt, regExp=None, stdout=True, msg=None):
- """
- txt is multiline string.
- - in memory
- regExp is a one-line regular expression
- - default
+ :param cmd: command being run
+ :param start_time: datetime object indicating when command was started
+ :param work_dir: path of working directory in which command is run
+ :param stdin: stdin input value for command
+ :param tmpdir: path to temporary output directory for command
+ :param thread_id: thread ID (None when not running shell command asynchronously)
+ :param interactive: boolean indicating whether it is an interactive command, or not
"""
- global errors_found_in_log
+ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
- if regExp and isinstance(regExp, bool):
- regExp = r"(? 0:
- core_cnt = int(out)
+ if int(res.output) > 0:
+ core_cnt = int(res.output)
except ValueError:
pass
@@ -313,10 +313,9 @@ def get_total_memory():
elif os_type == DARWIN:
cmd = "sysctl -n hw.memsize"
_log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
- with_sysroot=False)
- if ec == 0:
- memtotal = int(out.strip()) // (1024**2)
+ res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ memtotal = int(res.output.strip()) // (1024**2)
if memtotal is None:
memtotal = UNKNOWN
@@ -396,18 +395,18 @@ def get_cpu_vendor():
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
- with_hooks=False, with_sysroot=False)
- out = out.strip()
- if ec == 0 and out in VENDOR_IDS:
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ out = res.output.strip()
+ if res.exit_code == EasyBuildExit.SUCCESS and out in VENDOR_IDS:
vendor = VENDOR_IDS[out]
_log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
else:
cmd = "sysctl -n machdep.cpu.brand_string"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
- with_hooks=False, with_sysroot=False)
- out = out.strip().split(' ')[0]
- if ec == 0 and out in CPU_VENDORS:
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ out = res.output.strip().split(' ')[0]
+ if res.exit_code == EasyBuildExit.SUCCESS and out in CPU_VENDORS:
vendor = out
_log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
@@ -508,10 +507,9 @@ def get_cpu_model():
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
- with_sysroot=False)
- if ec == 0:
- model = out.strip()
+ res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ model = res.output.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
if model is None:
@@ -554,11 +552,10 @@ def get_cpu_speed():
elif os_type == DARWIN:
cmd = "sysctl -n hw.cpufrequency_max"
_log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
- with_sysroot=False)
- out = out.strip()
+ res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False)
+ out = res.output.strip()
cpu_freq = None
- if ec == 0 and out:
+ if res.exit_code == EasyBuildExit.SUCCESS and out:
# returns clock frequency in cycles/sec, but we want MHz
cpu_freq = float(out) // (1000 ** 2)
@@ -603,10 +600,10 @@ def get_cpu_features():
for feature_set in ['extfeatures', 'features', 'leaf7_features']:
cmd = "sysctl -n machdep.cpu.%s" % feature_set
_log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
- with_hooks=False, with_sysroot=False)
- if ec == 0:
- cpu_feat.extend(out.strip().lower().split())
+ res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False,
+ output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ cpu_feat.extend(res.output.strip().lower().split())
cpu_feat.sort()
@@ -631,36 +628,68 @@ def get_gpu_info():
try:
cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
_log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
- trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
- if ec == 0:
- for line in out.strip().split('\n'):
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ for line in res.output.strip().split('\n'):
nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
nvidia_gpu_info.setdefault(line, 0)
nvidia_gpu_info[line] += 1
else:
- _log.debug("None zero exit (%s) from nvidia-smi: %s", ec, out)
- except Exception as err:
+ _log.debug("None zero exit (%s) from nvidia-smi: %s", res.exit_code, res.output)
+ except EasyBuildError as err:
_log.debug("Exception was raised when running nvidia-smi: %s", err)
_log.info("No NVIDIA GPUs detected")
+ amdgpu_checked = False
+ if not which('amd-smi', on_error=IGNORE):
+ _log.info("amd-smi not found. Trying to detect AMD GPUs via rocm-smi")
+ else:
+ try:
+ cmd = "amd-smi static --driver --board --asic --csv"
+ _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd)
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ csv_reader = csv.DictReader(io.StringIO(res.output.strip()))
+
+ for row in csv_reader:
+ amd_card_series = row['product_name']
+ amd_card_device_id = row['device_id']
+ amd_card_gfx = row['target_graphics_version']
+ amd_card_driver = row['version']
+
+ amd_gpu = ("%s (device id: %s, gfx: %s, driver: %s)" %
+ (amd_card_series, amd_card_device_id, amd_card_gfx, amd_card_driver))
+ amd_gpu_info = gpu_info.setdefault('AMD', {})
+ amd_gpu_info.setdefault(amd_gpu, 0)
+ amd_gpu_info[amd_gpu] += 1
+ amdgpu_checked = True
+ else:
+ _log.debug("None zero exit (%s) from amd-smi: %s.", res.exit_code, res.output)
+ except EasyBuildError as err:
+ _log.debug("Exception was raised when running amd-smi: %s", err)
+ _log.info("No AMD GPUs detected via amd-smi.")
+ except KeyError as err:
+ _log.warning("Failed to extract AMD GPU info from amd-smi output: %s.", err)
+
if not which('rocm-smi', on_error=IGNORE):
_log.info("rocm-smi not found. Cannot detect AMD GPUs")
- else:
+ elif not amdgpu_checked:
try:
cmd = "rocm-smi --showdriverversion --csv"
_log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
- trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
- if ec == 0:
- amd_driver = out.strip().split('\n')[1].split(',')[1]
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ amd_driver = res.output.strip().split('\n')[1].split(',')[1]
cmd = "rocm-smi --showproductname --csv"
_log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
- trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
- if ec == 0:
- for line in out.strip().split('\n')[1:]:
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False,
+ output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ for line in res.output.strip().split('\n')[1:]:
amd_card_series = line.split(',')[1]
amd_card_model = line.split(',')[2]
amd_gpu = "%s (model: %s, driver: %s)" % (amd_card_series, amd_card_model, amd_driver)
@@ -668,8 +697,8 @@ def get_gpu_info():
amd_gpu_info.setdefault(amd_gpu, 0)
amd_gpu_info[amd_gpu] += 1
else:
- _log.debug("None zero exit (%s) from rocm-smi: %s", ec, out)
- except Exception as err:
+ _log.debug("None zero exit (%s) from rocm-smi: %s", res.exit_code, res.output)
+ except EasyBuildError as err:
_log.debug("Exception was raised when running rocm-smi: %s", err)
_log.info("No AMD GPUs detected")
@@ -868,16 +897,17 @@ def check_os_dependency(dep):
for pkg_cmd in pkg_cmds:
if which(pkg_cmd):
- cmd = [
+ cmd = ' '.join([
# unset $LD_LIBRARY_PATH to avoid broken rpm command due to loaded dependencies
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/4179
'unset LD_LIBRARY_PATH &&',
pkg_cmd,
pkg_cmd_flag.get(pkg_cmd),
dep,
- ]
- found = run_cmd(' '.join(cmd), simple=True, log_all=False, log_ok=False,
- force_in_dry_run=True, trace=False, stream_output=False)
+ ])
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True,
+ output_file=False, stream_output=False)
+ found = res.exit_code == EasyBuildExit.SUCCESS
if found:
break
@@ -888,10 +918,10 @@ def check_os_dependency(dep):
# try locate if it's available
if not found and which('locate'):
cmd = 'locate -c --regexp "/%s$"' % dep
- out, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False, force_in_dry_run=True, trace=False,
- stream_output=False)
+ res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True,
+ output_file=False, stream_output=False)
try:
- found = (ec == 0 and int(out.strip()) > 0)
+ found = (res.exit_code == EasyBuildExit.SUCCESS and int(res.output.strip()) > 0)
except ValueError:
# Returned something else than an int -> Error
found = False
@@ -904,41 +934,41 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False):
Get output of running version option for specific command line tool.
Output is returned as a single-line string (newlines are replaced by '; ').
"""
- out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True,
- trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
- if not ignore_ec and ec:
- _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out))
+ res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True,
+ hidden=True, with_hooks=False, output_file=False, stream_output=False)
+ if not ignore_ec and res.exit_code != EasyBuildExit.SUCCESS:
+ _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output))
return UNKNOWN
else:
- return '; '.join(out.split('\n'))
+ return '; '.join(res.output.split('\n'))
def get_gcc_version():
"""
Process `gcc --version` and return the GCC version.
"""
- out, ec = run_cmd('gcc --version', simple=False, log_ok=False, force_in_dry_run=True, verbose=False, trace=False,
- stream_output=False)
- res = None
- if ec:
- _log.warning("Failed to determine the version of GCC: %s", out)
- res = UNKNOWN
+ res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True,
+ output_file=False, stream_output=False)
+ gcc_ver = None
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ _log.warning("Failed to determine the version of GCC: %s", res.output)
+ gcc_ver = UNKNOWN
# Fedora: gcc (GCC) 5.1.1 20150618 (Red Hat 5.1.1-4)
# Debian: gcc (Debian 4.9.2-10) 4.9.2
- find_version = re.search(r"^gcc\s+\([^)]+\)\s+(?P[^\s]+)\s+", out)
+ find_version = re.search(r"^gcc\s+\([^)]+\)\s+(?P[^\s]+)\s+", res.output)
if find_version:
- res = find_version.group('version')
- _log.debug("Found GCC version: %s from %s", res, out)
+ gcc_ver = find_version.group('version')
+ _log.debug("Found GCC version: %s from %s", res, res.output)
else:
# Apple likes to install clang but call it gcc.
if get_os_type() == DARWIN:
_log.warning("On recent version of Mac OS, gcc is actually clang, returning None as GCC version")
- res = None
+ gcc_ver = None
else:
- raise EasyBuildError("Failed to determine the GCC version from: %s", out)
+ raise EasyBuildError("Failed to determine the GCC version from: %s", res.output)
- return res
+ return gcc_ver
def get_glibc_version():
@@ -974,9 +1004,9 @@ def get_linked_libs_raw(path):
or None for other types of files.
"""
- file_cmd_out, ec = run_cmd("file %s" % path, simple=False, trace=False)
- if ec:
- fail_msg = "Failed to run 'file %s': %s" % (path, file_cmd_out)
+ res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ fail_msg = "Failed to run 'file %s': %s" % (path, res.output)
_log.warning(fail_msg)
os_type = get_os_type()
@@ -987,7 +1017,7 @@ def get_linked_libs_raw(path):
# /usr/bin/ls: ELF 64-bit LSB executable, x86-64, ..., dynamically linked (uses shared libs), ...
# example output for shared libraries:
# /lib64/libc-2.17.so: ELF 64-bit LSB shared object, x86-64, ..., dynamically linked (uses shared libs), ...
- if "dynamically linked" in file_cmd_out:
+ if "dynamically linked" in res.output:
# determine linked libraries via 'ldd'
linked_libs_cmd = "ldd %s" % path
else:
@@ -999,7 +1029,7 @@ def get_linked_libs_raw(path):
# example output for shared libraries:
# /usr/lib/libz.dylib: Mach-O 64-bit dynamically linked shared library x86_64
bin_lib_regex = re.compile('(Mach-O .* executable)|(dynamically linked)', re.M)
- if bin_lib_regex.search(file_cmd_out):
+ if bin_lib_regex.search(res.output):
linked_libs_cmd = "otool -L %s" % path
else:
return None
@@ -1009,12 +1039,12 @@ def get_linked_libs_raw(path):
# take into account that 'ldd' may fail for strange reasons,
# like printing 'not a dynamic executable' when not enough memory is available
# (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111)
- out, ec = run_cmd(linked_libs_cmd, simple=False, trace=False, log_ok=False, log_all=False)
- if ec == 0:
- linked_libs_out = out
+ res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
+ if res.exit_code == EasyBuildExit.SUCCESS:
+ linked_libs_out = res.output
else:
- fail_msg = "Determining linked libraries for %s via '%s' failed! Output: '%s'" % (path, linked_libs_cmd, out)
- print_warning(fail_msg)
+ fail_msg = "Determining linked libraries for %s via '%s' failed! Output: '%s'"
+ print_warning(fail_msg % (path, linked_libs_cmd, res.output))
linked_libs_out = None
return linked_libs_out
@@ -1033,12 +1063,12 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None)
if required_patterns is None:
required_regexs = []
else:
- required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns]
+ required_regexs = [re.compile(p) if isinstance(p, str) else p for p in required_patterns]
if banned_patterns is None:
banned_regexs = []
else:
- banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns]
+ banned_regexs = [re.compile(p) if isinstance(p, str) else p for p in banned_patterns]
# resolve symbolic links (unless they're broken)
if os.path.islink(path) and os.path.exists(path):
@@ -1190,20 +1220,22 @@ def get_default_parallelism():
except AttributeError:
# No cache -> Calculate value from current system values
par = get_avail_core_count()
- # check ulimit -u
- out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False)
+ # determine max user processes via ulimit -u
+ res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False, stream_output=False)
try:
- if out.startswith("unlimited"):
+ if res.output.startswith("unlimited"):
maxuserproc = 2 ** 32 - 1
else:
- maxuserproc = int(out)
+ maxuserproc = int(res.output)
except ValueError as err:
- raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err)
+ raise EasyBuildError(
+ "Failed to determine max user processes (%s, %s): %s", res.exit_code, res.output, err
+ )
# assume 6 processes per build thread + 15 overhead
par_guess = (maxuserproc - 15) // 6
if par_guess < par:
par = par_guess
- _log.info("Limit parallel builds to %s because max user processes is %s", par, out)
+ _log.info("Limit parallel builds to %s because max user processes is %s", par, res.output)
# Cache value
det_parallelism._default_parallelism = par
return par
@@ -1217,6 +1249,8 @@ def get_default_parallelism():
raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
if maxpar is not None and maxpar < par:
+ if maxpar is False:
+ maxpar = 1
_log.info("Limiting parallelism from %s to %s", par, maxpar)
par = maxpar
@@ -1249,17 +1283,13 @@ def check_python_version():
python_ver = '%d.%d' % (python_maj_ver, python_min_ver)
_log.info("Found Python version %s", python_ver)
- if python_maj_ver == 2:
- if python_min_ver < 7:
- raise EasyBuildError("Python 2.7 is required when using Python 2, found Python %s", python_ver)
- else:
- _log.info("Running EasyBuild with Python 2 (version %s)", python_ver)
-
- elif python_maj_ver == 3:
- if python_min_ver < 5:
- raise EasyBuildError("Python 3.5 or higher is required when using Python 3, found Python %s", python_ver)
+ if python_maj_ver == 3:
+ if python_min_ver < 6:
+ raise EasyBuildError("Python 3.6 or higher is required, found Python %s", python_ver)
else:
_log.info("Running EasyBuild with Python 3 (version %s)", python_ver)
+ elif python_maj_ver < 3:
+ raise EasyBuildError("EasyBuild is not compatible with Python %s", python_ver)
else:
raise EasyBuildError("EasyBuild is not compatible (yet) with Python %s", python_ver)
@@ -1320,7 +1350,7 @@ def pick_dep_version(dep_version):
result = None
else:
result = pick_system_specific_value("version", dep_version)
- if not isinstance(result, string_type) and result is not False:
+ if not isinstance(result, str) and result is not False:
typ = type(dep_version)
raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version)
@@ -1373,9 +1403,9 @@ def extract_version(tool):
python_version = extract_version(sys.executable)
opt_dep_versions = {}
- for key in EASYBUILD_OPTIONAL_DEPENDENCIES:
+ for key, opt_dep in EASYBUILD_OPTIONAL_DEPENDENCIES.items():
- pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0]
+ pkg = opt_dep[0]
if pkg is None:
pkg = key.lower()
@@ -1401,8 +1431,8 @@ def extract_version(tool):
opt_deps_key = "Optional dependencies"
checks_data[opt_deps_key] = {}
- for key in opt_dep_versions:
- checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1])
+ for key, version in opt_dep_versions.items():
+ checks_data[opt_deps_key][key] = (version, EASYBUILD_OPTIONAL_DEPENDENCIES[key][1])
sys_tools_key = "System tools"
checks_data[sys_tools_key] = {}
diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
index 881788337e..2f0b0c8a6f 100644
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -47,7 +47,7 @@
from easybuild.framework.easyconfig.tools import skip_available
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
-from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file
+from easybuild.tools.filetools import find_easyconfigs, get_cwd, mkdir, read_file, write_file
from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue
from easybuild.tools.jenkins import aggregate_xml_in_dirs
from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
@@ -67,7 +67,7 @@ def regtest(easyconfig_paths, modtool, build_specs=None):
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
- cur_dir = os.getcwd()
+ cur_dir = get_cwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
@@ -321,8 +321,8 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
gpu_info = get_gpu_info()
gpu_str = ""
if gpu_info:
- for vendor in gpu_info:
- for gpu, num in gpu_info[vendor].items():
+ for vendor, vendor_gpu in gpu_info.items():
+ for gpu, num in vendor_gpu.items():
gpu_str += ", %s x %s %s" % (num, vendor, gpu)
os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py
index cf755a8e40..928a2252af 100644
--- a/easybuild/tools/toolchain/__init__.py
+++ b/easybuild/tools/toolchain/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py
index faf8f03d88..5619ecc733 100644
--- a/easybuild/tools/toolchain/compiler.py
+++ b/easybuild/tools/toolchain/compiler.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,13 +32,12 @@
* Damian Alvarez (Forschungszentrum Juelich GmbH)
"""
from easybuild.tools import systemtools
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.config import build_option
-from easybuild.tools.py2vs3 import string_type
from easybuild.tools.toolchain.constants import COMPILER_VARIABLES
from easybuild.tools.toolchain.toolchain import Toolchain
-# default optimization 'level' (see COMPILER_SHARED_OPTION_MAP/COMPILER_OPT_FLAGS)
+# default optimization 'level' (see COMPILER_SHARED_OPTION_MAP/COMPILER_OPT_OPTIONS)
DEFAULT_OPT_LEVEL = 'defaultopt'
# 'GENERIC' can be used to enable generic compilation instead of optimized compilation (which is the default)
@@ -83,18 +82,19 @@ class Compiler(Toolchain):
'loose': (False, "Loose precision"),
'veryloose': (False, "Very loose precision"),
'verbose': (False, "Verbose output"),
- 'debug': (False, "Enable debug"),
+ 'debug': (None, "Keep debug symbols"), # default value set by build option keep-debug-symbols
'i8': (False, "Integers are 8 byte integers"), # fortran only -> no: MKL and icc give -DMKL_ILP64
'r8': (False, "Real is 8 byte real"), # fortran only
'unroll': (False, "Unroll loops"),
'cstd': (None, "Specify C standard"),
'shared': (False, "Build shared library"),
'static': (False, "Build static library"),
- '32bit': (False, "Compile 32bit target"), # LA, FFTW
'openmp': (False, "Enable OpenMP"),
'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"),
'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly
'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"),
+ 'search-path-cpp-headers': (None, "Search path used at build time for include directories"),
+ 'search-path-linker': (None, "Search path used at build time by the linker for libraries"),
'extra_cflags': (None, "Specify extra CFLAGS options."),
'extra_cxxflags': (None, "Specify extra CXXFLAGS options."),
'extra_fflags': (None, "Specify extra FFLAGS options."),
@@ -104,19 +104,18 @@ class Compiler(Toolchain):
COMPILER_UNIQUE_OPTION_MAP = None
COMPILER_SHARED_OPTION_MAP = {
- DEFAULT_OPT_LEVEL: 'O2',
- '32bit': 'm32',
- 'cstd': 'std=%(value)s',
- 'debug': 'g',
- 'lowopt': 'O1',
- 'noopt': 'O0',
- 'openmp': 'fopenmp',
- 'opt': 'O3',
- 'pic': 'fPIC',
- 'shared': 'shared',
- 'static': 'static',
- 'unroll': 'unroll',
- 'verbose': 'v',
+ DEFAULT_OPT_LEVEL: '-O2',
+ 'cstd': '-std=%(value)s',
+ 'debug': '-g',
+ 'lowopt': '-O1',
+ 'noopt': '-O0',
+ 'openmp': '-fopenmp',
+ 'opt': '-O3',
+ 'pic': '-fPIC',
+ 'shared': '-shared',
+ 'static': '-static',
+ 'unroll': '-unroll',
+ 'verbose': '-v',
'extra_cflags': '%(value)s',
'extra_cxxflags': '%(value)s',
'extra_fflags': '%(value)s',
@@ -127,20 +126,20 @@ class Compiler(Toolchain):
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None
COMPILER_GENERIC_OPTION = None
- COMPILER_FLAGS = ['debug', 'ieee', 'openmp', 'pic', 'shared', 'static', 'unroll', 'verbose'] # any compiler
- COMPILER_OPT_FLAGS = ['noopt', 'lowopt', DEFAULT_OPT_LEVEL, 'opt'] # optimisation args, ordered !
- COMPILER_PREC_FLAGS = ['strict', 'precise', 'defaultprec', 'loose', 'veryloose'] # precision flags, ordered !
+ COMPILER_OPTIONS = ['debug', 'ieee', 'openmp', 'pic', 'shared', 'static', 'unroll', 'verbose'] # any compiler
+ COMPILER_OPT_OPTIONS = ['noopt', 'lowopt', DEFAULT_OPT_LEVEL, 'opt'] # optimisation args, ordered !
+ COMPILER_PREC_OPTIONS = ['strict', 'precise', 'defaultprec', 'loose', 'veryloose'] # precision flags, ordered !
COMPILER_CC = None
COMPILER_CXX = None
- COMPILER_C_FLAGS = ['cstd']
- COMPILER_C_UNIQUE_FLAGS = []
+ COMPILER_C_OPTIONS = ['cstd']
+ COMPILER_C_UNIQUE_OPTIONS = []
COMPILER_F77 = None
COMPILER_F90 = None
COMPILER_FC = None
- COMPILER_F_FLAGS = ['i8', 'r8']
- COMPILER_F_UNIQUE_FLAGS = []
+ COMPILER_F_OPTIONS = ['i8', 'r8']
+ COMPILER_F_UNIQUE_OPTIONS = []
LINKER_TOGGLE_STATIC_DYNAMIC = None
LINKER_TOGGLE_START_STOP_GROUP = {
@@ -178,6 +177,10 @@ def set_variables(self):
def _set_compiler_toolchainoptions(self):
"""Set the compiler related toolchain options"""
+ # Initialize default value of debug symbols based on global build option
+ if self.COMPILER_SHARED_OPTS and 'debug' in self.COMPILER_SHARED_OPTS:
+ _, desc = self.COMPILER_SHARED_OPTS['debug']
+ self.COMPILER_SHARED_OPTS['debug'] = (build_option('keep_debug_symbols'), desc)
self.options.add_options(self.COMPILER_SHARED_OPTS, self.COMPILER_SHARED_OPTION_MAP)
# always include empty infix first for non-prefixed compilers (e.g., GCC, Intel, ...)
@@ -190,10 +193,6 @@ def _set_compiler_toolchainoptions(self):
def _set_compiler_vars(self):
"""Set the compiler variables"""
- is32bit = self.options.get('32bit', None)
- if is32bit:
- self.log.debug("_set_compiler_vars: 32bit set: changing compiler definitions")
-
comp_var_tmpl_dict = {}
# always include empty infix first for non-prefixed compilers (e.g., GCC, Intel, ...)
@@ -215,8 +214,6 @@ def _set_compiler_vars(self):
self.log.warning("_set_compiler_vars: %s compiler variable %s undefined", infix, var)
self.variables[pref_var] = value
- if is32bit:
- self.variables.nappend_el(pref_var, self.options.option('32bit'))
# update dictionary to complete compiler variable template
# to produce e.g. 'nvcc -ccbin=icpc' from 'nvcc -ccbin=%(CXX_base)'
@@ -246,23 +243,34 @@ def _set_compiler_vars(self):
def _set_compiler_flags(self):
"""Collect the flags set, and add them as variables too"""
-
- flags = [self.options.option(x) for x in self.COMPILER_FLAGS if self.options.get(x, False)]
- cflags = [self.options.option(x) for x in self.COMPILER_C_FLAGS + self.COMPILER_C_UNIQUE_FLAGS
+ variants = ['', '_F', '_F_UNIQUE', '_C', '_C_UNIQUE', '_OPT', '_PREC']
+ for variant in variants:
+ old_var = getattr(self, f'COMPILER{variant}_FLAGS', None)
+ if old_var is not None:
+ self.log.deprecated(f'COMPILER{variant}_FLAGS has been renamed to COMPILER{variant}_OPTIONS.', '6.0')
+ setattr(self, f'COMPILER{variant}_OPTIONS', old_var)
+
+ flags = [self.options.option(x) for x in self.COMPILER_OPTIONS if self.options.get(x, False)]
+ cflags = [self.options.option(x) for x in self.COMPILER_C_OPTIONS + self.COMPILER_C_UNIQUE_OPTIONS
if self.options.get(x, False)]
- fflags = [self.options.option(x) for x in self.COMPILER_F_FLAGS + self.COMPILER_F_UNIQUE_FLAGS
+ fflags = [self.options.option(x) for x in self.COMPILER_F_OPTIONS + self.COMPILER_F_UNIQUE_OPTIONS
if self.options.get(x, False)]
# Allow a user-defined default optimisation
default_opt_level = build_option('default_opt_level')
- if default_opt_level not in self.COMPILER_OPT_FLAGS:
+ if default_opt_level not in self.COMPILER_OPT_OPTIONS:
raise EasyBuildError("Unknown value for default optimisation: %s (possibilities are %s)" %
- (default_opt_level, self.COMPILER_OPT_FLAGS))
+ (default_opt_level, self.COMPILER_OPT_OPTIONS))
# 1st one is the one to use. add default at the end so len is at least 1
- optflags = ([self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] +
+ optflags = ([self.options.option(x) for x in self.COMPILER_OPT_OPTIONS if self.options.get(x, False)] +
[self.options.option(default_opt_level)])[:1]
+ # Normal compiler flags need to include "-" starting with EB 5.0, check the first as a sanity check.
+ # Avoiding all flags as there may be legitimate use for flags that lack -
+ if optflags and optflags[0] and not optflags[0][0].startswith('-'):
+ print_warning(f'Compiler flag "{optflags[0][0]}" does not start with a dash. See changes in EasyBuild 5.')
+
# only apply if the vectorize toolchainopt is explicitly set
# otherwise the individual compiler toolchain file should make sure that
# vectorization is disabled for noopt and lowopt, and enabled otherwise.
@@ -282,7 +290,7 @@ def _set_compiler_flags(self):
elif self.options.get('optarch', False):
optarchflags.append(self.options.option('optarch'))
- precflags = [self.options.option(x) for x in self.COMPILER_PREC_FLAGS if self.options.get(x, False)] + \
+ precflags = [self.options.option(x) for x in self.COMPILER_PREC_OPTIONS if self.options.get(x, False)] + \
[self.options.option('defaultprec')]
self.variables.nextend('OPTFLAGS', optflags + optarchflags)
@@ -301,7 +309,7 @@ def _set_compiler_flags(self):
extraflags = self.options.option(extra)
if not extraflags or extraflags[0] != '-':
raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, extraflags))
- self.variables.nappend_el(var, extraflags[1:])
+ self.variables.nappend_el(var, extraflags)
def _set_optimal_architecture(self, default_optarch=None):
"""
@@ -311,7 +319,7 @@ def _set_optimal_architecture(self, default_optarch=None):
(--optarch and --optarch=GENERIC still override this value)
"""
ec_optarch = self.options.get('optarch', False)
- if isinstance(ec_optarch, string_type):
+ if isinstance(ec_optarch, str):
if OPTARCH_MAP_CHAR in ec_optarch:
error_msg = "When setting optarch in the easyconfig (found %s), " % ec_optarch
error_msg += "the syntax is not allowed. " % OPTARCH_MAP_CHAR
@@ -338,7 +346,7 @@ def _set_optimal_architecture(self, default_optarch=None):
self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.",
current_compiler)
- if isinstance(optarch, string_type):
+ if isinstance(optarch, str):
use_generic = (optarch == OPTARCH_GENERIC)
elif optarch is None:
use_generic = False
@@ -358,6 +366,11 @@ def _set_optimal_architecture(self, default_optarch=None):
optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(self.arch, self.cpu_family)]
if optarch is not None:
+ if optarch and not optarch.startswith('-'):
+ self.log.deprecated(f'Specifying optarch "{optarch}" without initial dash is deprecated.', '6.0')
+ # Add flags for backwards compatibility
+ optarch = '-' + optarch
+
optarch_log_str = optarch or 'no flags'
self.log.info("_set_optimal_architecture: using %s as optarch for %s/%s.",
optarch_log_str, self.arch, self.cpu_family)
@@ -366,9 +379,9 @@ def _set_optimal_architecture(self, default_optarch=None):
optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic]
error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family)
error_msg += "Use --optarch='' to override (see "
- error_msg += "http://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html "
+ error_msg += "https://docs.easybuild.io/controlling-compiler-optimization-flags/ "
error_msg += "for details) and consider contributing your settings back (see "
- error_msg += "http://easybuild.readthedocs.io/en/latest/Contributing.html)."
+ error_msg += "https://docs.easybuild.io/contributing/)."
raise EasyBuildError(error_msg)
def comp_family(self, prefix=None):
diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py
index 643e8b4d21..47f06c7113 100644
--- a/easybuild/tools/toolchain/constants.py
+++ b/easybuild/tools/toolchain/constants.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,9 +31,9 @@
* Kenneth Hoste (Ghent University)
"""
-from easybuild.tools.variables import AbsPathList
-from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs
-from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths
+from easybuild.tools.toolchain.variables import CommaSharedLibs, CommaStaticLibs
+from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths
+from easybuild.tools.variables import AbsPathList, StrList
COMPILER_VARIABLES = [
@@ -53,32 +53,36 @@
]
COMPILER_MAP_CLASS = {
- FlagList: [
+ StrList: [
('OPTFLAGS', 'Optimization flags'),
('PRECFLAGS', 'FP precision flags'),
- ] + COMPILER_FLAGS,
+ ] + COMPILER_FLAGS + COMPILER_VARIABLES,
LibraryList: [
- ('LIBS', 'Libraries'), # TODO: where are these used? ld?
- ('FLIBS', 'Fortran libraries'), # TODO: where are these used? gfortran only?
+ ('LIBS', 'Libraries'), # -l options to pass to the linker (C/C++/Fortran)
+ ('FLIBS', 'Fortran libraries'), # linker flags (e.g. -L and -l) for Fortran libraries
],
LinkLibraryPaths: [
- ('LDFLAGS', 'Flags passed to linker'), # TODO: overridden by command line?
+ ('LDFLAGS', 'Linker flags'),
],
IncludePaths: [
- ('CPPFLAGS', 'Precompiler flags'),
+ ('CPPFLAGS', 'Preprocessor flags'),
+ ],
+ SearchPaths: [
+ ('CPATH', 'Location of C/C++ header files'),
+ ('C_INCLUDE_PATH', 'Location of C header files'),
+ ('CPLUS_INCLUDE_PATH', 'Location of C++ header files'),
+ ('OBJC_INCLUDE_PATH', 'Location of Objective C header files'),
+ ('LIBRARY_PATH', 'Location of linker files'),
],
- CommandFlagList: COMPILER_VARIABLES,
}
CO_COMPILER_MAP_CLASS = {
- CommandFlagList: [
+ StrList: [
('CUDA_CC', 'CUDA C compiler command'),
('CUDA_CXX', 'CUDA C++ compiler command'),
('CUDA_F77', 'CUDA Fortran 77 compiler command'),
('CUDA_F90', 'CUDA Fortran 90 compiler command'),
('CUDA_FC', 'CUDA Fortran 77/90 compiler command'),
- ],
- FlagList: [
('CUDA_CFLAGS', 'CUDA C compiler flags'),
('CUDA_CXXFLAGS', 'CUDA C++ compiler flags'),
('CUDA_FCFLAGS', 'CUDA Fortran 77/90 compiler flags'),
@@ -103,7 +107,7 @@
('MPI_LIB_DIR', 'MPI library directory'),
('MPI_INC_DIR', 'MPI include directory'),
],
- CommandFlagList: MPI_COMPILER_VARIABLES + SEQ_COMPILER_VARIABLES,
+ StrList: MPI_COMPILER_VARIABLES + SEQ_COMPILER_VARIABLES,
}
BLAS_MAP_CLASS = {
diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py
index b7191d5a26..53bce4cd8c 100644
--- a/easybuild/tools/toolchain/fft.py
+++ b/easybuild/tools/toolchain/fft.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py
index fd74615c4a..bbf158d9e5 100644
--- a/easybuild/tools/toolchain/linalg.py
+++ b/easybuild/tools/toolchain/linalg.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py
index 0ea2714490..aaaf4e7078 100644
--- a/easybuild/tools/toolchain/mpi.py
+++ b/easybuild/tools/toolchain/mpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -193,10 +193,6 @@ def set_variables(self):
def _set_mpi_compiler_variables(self):
"""Set the MPI compiler variables"""
- is32bit = self.options.get('32bit', None)
- if is32bit:
- self.log.debug("_set_mpi_compiler_variables: 32bit set: changing compiler definitions")
-
for var_tuple in COMPILER_VARIABLES:
c_var = var_tuple[0] # [1] is the description
var = MPI_COMPILER_TEMPLATE % {'c_var': c_var}
@@ -214,9 +210,6 @@ def _set_mpi_compiler_variables(self):
self.variables.nappend_el(var, self.options.option('_opt_%s' % var, templatedict=templatedict))
- if is32bit:
- self.variables.nappend_el(var, self.options.option('32bit'))
-
if self.options.get('usempi', None):
var_seq = SEQ_COMPILER_TEMPLATE % {'c_var': c_var}
self.log.debug("usempi set: defining %s as %s", var_seq, self.variables[c_var])
@@ -238,9 +231,7 @@ def _set_mpi_variables(self):
lib_dir = ['lib']
incl_dir = ['include']
- suffix = None
- if not self.options.get('32bit', None):
- suffix = '64'
+ suffix = '64'
# take into account that MPI_MODULE_NAME could be None (see Cray toolchains)
for root in self.get_software_root(self.MPI_MODULE_NAME or []):
diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py
index 391a2831e0..cc0e515301 100644
--- a/easybuild/tools/toolchain/options.py
+++ b/easybuild/tools/toolchain/options.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,7 +39,6 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
class ToolchainOptions(dict):
@@ -105,7 +104,7 @@ def option(self, name, templatedict=None):
'value': value,
})
- if isinstance(res, string_type):
+ if isinstance(res, str):
# allow for template
res = self.options_map[name] % templatedict
elif isinstance(res, (list, tuple,)):
diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
index f928215fd4..c13f9e10d7 100644
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -69,17 +69,11 @@
from easybuild.tools.systemtools import LINUX, get_os_type
from easybuild.tools.toolchain.options import ToolchainOptions
from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables
-from easybuild.tools.utilities import nub, trace_msg
+from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg
_log = fancylogger.getLogger('tools.toolchain', fname=False)
-# name/version for dummy toolchain
-# if name==DUMMY_TOOLCHAIN_NAME and version==DUMMY_TOOLCHAIN_VERSION, do not load dependencies
-# NOTE: use of 'dummy' toolchain is deprecated, replaced by 'system' toolchain (which always loads dependencies)
-DUMMY_TOOLCHAIN_NAME = 'dummy'
-DUMMY_TOOLCHAIN_VERSION = 'dummy'
-
SYSTEM_TOOLCHAIN_NAME = 'system'
CCACHE = 'ccache'
@@ -101,11 +95,27 @@
TOOLCHAIN_CAPABILITY_LAPACK_FAMILY,
TOOLCHAIN_CAPABILITY_MPI_FAMILY,
]
+# modes to handle header and linker search paths
+# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html
+# supported on Linux by: GCC, GFortran, oneAPI C/C++ Compilers, oneAPI Fortran Compiler, LLVM-based
+SEARCH_PATH = {
+ "cpp_headers": {
+ "flags": ["CPPFLAGS"],
+ "cpath": ["CPATH"],
+ "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+ },
+ "linker": {
+ "flags": ["LDFLAGS"],
+ "library_path": ["LIBRARY_PATH"],
+ },
+}
+DEFAULT_SEARCH_PATH_CPP_HEADERS = "flags"
+DEFAULT_SEARCH_PATH_LINKER = "flags"
def is_system_toolchain(tc_name):
"""Return whether toolchain with specified name is a system toolchain or not."""
- return tc_name in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]
+ return tc_name in [SYSTEM_TOOLCHAIN_NAME]
def env_vars_external_module(name, version, metadata):
@@ -197,10 +207,6 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
if name is None:
raise EasyBuildError("Toolchain init: no name provided")
self.name = name
- if self.name == DUMMY_TOOLCHAIN_NAME:
- self.log.deprecated("Use of 'dummy' toolchain is deprecated, use 'system' toolchain instead", '5.0',
- silent=build_option('silent'))
- self.name = SYSTEM_TOOLCHAIN_NAME
if version is None:
version = self.VERSION
@@ -224,6 +230,11 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
self.use_rpath = False
+ self.search_path = {
+ "cpp_headers": DEFAULT_SEARCH_PATH_CPP_HEADERS,
+ "linker": DEFAULT_SEARCH_PATH_LINKER,
+ }
+
self.mns = mns
self.mod_full_name = None
self.mod_short_name = None
@@ -237,7 +248,7 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
self.init_modpaths = self.mns.det_init_modulepaths(tc_dict)
def is_system_toolchain(self):
- """Return boolean to indicate whether this toolchain is a system(/dummy) toolchain."""
+ """Return boolean to indicate whether this toolchain is a system toolchain."""
return is_system_toolchain(self.name)
def set_minimal_build_env(self):
@@ -371,9 +382,11 @@ def get_variable(self, name, typ=str):
return res
def set_variables(self):
- """Do nothing? Everything should have been set by others
- Needs to be defined for super() relations
"""
+ No generic toolchain variables set.
+ Post-process variables set by child Toolchain classes.
+ """
+
if self.options.option('packed-linker-options'):
self.log.devel("set_variables: toolchain variables. packed-linker-options.")
self.variables.try_function_on_element('set_packed_linker_options')
@@ -564,16 +577,6 @@ def _check_dependencies(self, dependencies):
return deps
- def add_dependencies(self, dependencies):
- """
- [DEPRECATED] Verify if the given dependencies exist, and return them.
-
- This method is deprecated.
- You should pass the dependencies to the 'prepare' method instead, via the 'deps' named argument.
- """
- self.log.deprecated("use of 'Toolchain.add_dependencies' method", '4.0')
- self.dependencies = self._check_dependencies(dependencies)
-
def is_required(self, name):
"""Determine whether this is a required toolchain element."""
# default: assume every element is required
@@ -608,8 +611,8 @@ def _simulated_load_dependency_module(self, name, version, metadata, verbose=Fal
self.log.debug("Defining $EB* environment variables for software named %s", name)
env_vars = env_vars_external_module(name, version, metadata)
- for key in env_vars:
- setvar(key, env_vars[key], verbose=verbose)
+ for var, value in env_vars.items():
+ setvar(var, value, verbose=verbose)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
@@ -768,6 +771,27 @@ def _verify_toolchain(self):
raise EasyBuildError("List of toolchain dependency modules and toolchain definition do not match "
"(found %s vs expected %s)", self.toolchain_dep_mods, toolchain_definition)
+ def _validate_search_path(self):
+ """
+ Validate search path toolchain options.
+ Toolchain option has precedence over build option
+ """
+ for search_path in self.search_path:
+ sp_build_opt = f"search_path_{search_path}"
+ sp_toolchain_opt = sp_build_opt.replace("_", "-")
+ if self.options.get(sp_toolchain_opt) is not None:
+ self.search_path[search_path] = self.options.option(sp_toolchain_opt)
+ elif build_option(sp_build_opt) is not None:
+ self.search_path[search_path] = build_option(sp_build_opt)
+
+ if self.search_path[search_path] not in SEARCH_PATH[search_path]:
+ raise EasyBuildError(
+ "Unknown value selected for toolchain option %s: %s. Choose one of: %s",
+ sp_toolchain_opt, self.search_path[search_path], ", ".join(SEARCH_PATH[search_path])
+ )
+
+ self.log.debug("%s toolchain option set to: %s", sp_toolchain_opt, self.search_path[search_path])
+
def symlink_commands(self, paths):
"""
Create a symlink for each command to binary/script at specified path.
@@ -847,7 +871,6 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
self._load_modules(silent=silent)
if self.is_system_toolchain():
-
# define minimal build environment when using system toolchain;
# this is mostly done to try controlling which compiler commands are being used,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/3398
@@ -860,6 +883,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
self._verify_toolchain()
# Generate the variables to be set
+ self._validate_search_path()
self.set_variables()
# set the variables
@@ -870,7 +894,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
else:
self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
- # add LDFLAGS and CPPFLAGS from dependencies to self.vars
+ # add linker and preprocessor paths of dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod, verbose=not silent)
@@ -1048,7 +1072,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
def handle_sysroot(self):
"""
- Extra stuff to be done when alternate system root is specified via --sysroot EasyBuild configuration option.
+ Extra stuff to be done when alternative system root is specified via --sysroot EasyBuild configuration option.
* Update $PKG_CONFIG_PATH to include sysroot location to pkg-config files (*.pc).
"""
@@ -1069,41 +1093,54 @@ def handle_sysroot(self):
setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path))
def _add_dependency_variables(self, names=None, cpp=None, ld=None):
- """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
- names should be a list of strings containing the name of the dependency
"""
- cpp_paths = ['include']
- ld_paths = ['lib']
- if not self.options.get('32bit', None):
- ld_paths.insert(0, 'lib64')
-
- if cpp is not None:
- for p in cpp:
- if p not in cpp_paths:
- cpp_paths.append(p)
- if ld is not None:
- for p in ld:
- if p not in ld_paths:
- ld_paths.append(p)
-
- if not names:
- deps = self.dependencies
- else:
- deps = [{'name': name} for name in names if name is not None]
+ Add linker and preprocessor paths of dependencies to self.variables
+ :names: list of strings containing the name of the dependency
+ """
+ # collect dependencies
+ dependencies = self.dependencies if names is None else [{"name": name} for name in names if name]
# collect software install prefixes for dependencies
- roots = []
- for dep in deps:
- if dep.get('external_module', False):
+ dependency_roots = []
+ for dep in dependencies:
+ if dep.get("external_module", False):
# for software names provided via external modules, install prefix may be unknown
- names = dep['external_module_metadata'].get('name', [])
- roots.extend([root for root in self.get_software_root(names) if root is not None])
+ names = dep["external_module_metadata"].get("name", [])
+ dependency_roots.extend([root for root in self.get_software_root(names) if root is not None])
else:
- roots.extend(self.get_software_root(dep['name']))
+ dependency_roots.extend(self.get_software_root(dep["name"]))
+
+ for root in dependency_roots:
+ self._add_dependency_cpp_headers(root, extra_dirs=cpp)
+ self._add_dependency_linker_paths(root, extra_dirs=ld)
+
+ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
+ """
+ Append prepocessor paths for given dependency root directory
+ """
+ if extra_dirs is None:
+ extra_dirs = ()
+
+ header_dirs = ["include"]
+ header_dirs = unique_ordered_extend(header_dirs, extra_dirs)
+
+ for env_var in SEARCH_PATH["cpp_headers"][self.search_path["cpp_headers"]]:
+ self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root)
+ self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs)
+
+ def _add_dependency_linker_paths(self, dep_root, extra_dirs=None):
+ """
+ Append linker paths for given dependency root directory
+ """
+ if extra_dirs is None:
+ extra_dirs = ()
+
+ lib_dirs = ["lib64", "lib"]
+ lib_dirs = unique_ordered_extend(lib_dirs, extra_dirs)
- for root in roots:
- self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
- self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+ for env_var in SEARCH_PATH["linker"][self.search_path["linker"]]:
+ self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root)
+ self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs)
def _setenv_variables(self, donotset=None, verbose=True):
"""Actually set the environment variables"""
@@ -1136,9 +1173,9 @@ def _setenv_variables(self, donotset=None, verbose=True):
def get_flag(self, name):
"""Get compiler flag(s) for a certain option."""
if isinstance(self.options.option(name), list):
- return " ".join("-%s" % x for x in list(self.options.option(name)))
+ return " ".join(self.options.option(name))
else:
- return "-%s" % self.options.option(name)
+ return self.options.option(name)
def toolchain_family(self):
"""Return toolchain family for this toolchain."""
diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py
index dbb5686040..a5a78ab9fd 100644
--- a/easybuild/tools/toolchain/toolchainvariables.py
+++ b/easybuild/tools/toolchain/toolchainvariables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,7 +33,8 @@
from easybuild.tools.variables import Variables, join_map_class
from easybuild.tools.toolchain.constants import ALL_MAP_CLASSES
-from easybuild.tools.toolchain.variables import LinkerFlagList, FlagList
+from easybuild.tools.toolchain.variables import LinkerFlagList
+from easybuild.tools.variables import StrList
class ToolchainVariables(Variables):
@@ -42,7 +43,7 @@ class ToolchainVariables(Variables):
in context of compilers (i.e. the generated string are e.g. compiler options or link flags)
"""
MAP_CLASS = join_map_class(ALL_MAP_CLASSES) # join_map_class strips explanation
- DEFAULT_CLASS = FlagList
+ DEFAULT_CLASS = StrList
LINKER_TOGGLE_START_STOP_GROUP = None
LINKER_TOGGLE_STATIC_DYNAMIC = None
diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py
index 8047778391..90a0c99583 100644
--- a/easybuild/tools/toolchain/utilities.py
+++ b/easybuild/tools/toolchain/utilities.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py
index 482f989480..16487de7ff 100644
--- a/easybuild/tools/toolchain/variables.py
+++ b/easybuild/tools/toolchain/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList):
PREFIX = '-L'
+class SearchPaths(AbsPathList):
+ """Colon-separated list of absolute paths"""
+ SEPARATOR = ':'
+
+
class FlagList(StrList):
"""Flag list"""
PREFIX = "-"
diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py
index 8f6407e55d..ae570f3dc0 100644
--- a/easybuild/tools/utilities.py
+++ b/easybuild/tools/utilities.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,12 +34,11 @@
import os
import re
import sys
-from string import digits
+from string import ascii_letters, digits
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option
-from easybuild.tools.py2vs3 import ascii_letters, string_type
_log = fancylogger.getLogger('tools.utilities')
@@ -72,7 +71,7 @@ def quote_str(val, escape_newline=False, prefer_single_quotes=False, escape_back
:param tcl: Boolean for whether we are quoting for Tcl syntax
"""
- if isinstance(val, string_type):
+ if isinstance(val, str):
# escape backslashes
if escape_backslash:
val = val.replace('\\', '\\\\')
@@ -164,7 +163,7 @@ def only_if_module_is_available(modnames, pkgname=None, url=None):
if pkgname and url is None:
url = 'https://pypi.python.org/pypi/%s' % pkgname
- if isinstance(modnames, string_type):
+ if isinstance(modnames, str):
modnames = (modnames,)
def wrap(orig):
@@ -205,23 +204,35 @@ def trace_msg(message, silent=False):
def nub(list_):
- """Returns the unique items of a list of hashables, while preserving order of
- the original list, i.e. the first unique element encoutered is
- retained.
+ """Returns the unique items of a list of hashables, while preserving order of the original list,
+ i.e. the first unique element encoutered is retained.
- Code is taken from
- http://stackoverflow.com/questions/480214/how-do-you-remove-duplicates-from-a-list-in-python-whilst-preserving-order
+ Code is taken from https://www.peterbe.com/plog/fastest-way-to-uniquify-a-list-in-python-3.6
- Supposedly, this is one of the fastest ways to determine the
- unique elements of a list.
+ Supposedly, this is one of the fastest ways to determine the unique elements of a list.
@type list_: a list :-)
:return: a new list with each element from `list` appearing only once (cfr. Michelle Dubois).
"""
- seen = set()
- seen_add = seen.add
- return [x for x in list_ if x not in seen and not seen_add(x)]
+ return list(dict.fromkeys(list_))
+
+
+def unique_ordered_extend(base, affix):
+ """Extend base list with elements of affix list keeping order and without duplicates"""
+ if isinstance(affix, str):
+ # avoid extending with strings, as iterables generate wrong result without error
+ raise EasyBuildError(f"given affix list is a string: {affix}")
+
+ try:
+ ext_base = base.copy()
+ ext_base.extend(affix)
+ except TypeError as err:
+ raise EasyBuildError(f"given affix list is not iterable: {affix}") from err
+ except AttributeError as err:
+ raise EasyBuildError(f"given base cannot be extended: {base}") from err
+
+ return nub(ext_base) # remove duplicates
def get_class_for(modulepath, class_name):
diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py
index cef771f3d8..423f7adb36 100644
--- a/easybuild/tools/variables.py
+++ b/easybuild/tools/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 681233536b..7b1716a497 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,7 +45,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.9.4')
+VERSION = LooseVersion('5.0.0')
UNKNOWN = 'UNKNOWN'
UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
@@ -93,8 +93,7 @@ def get_git_revision():
def this_is_easybuild():
"""Standard starting message"""
- top_version = max(FRAMEWORK_VERSION, EASYBLOCKS_VERSION)
- # !!! bootstrap_eb.py script checks hard on the string below, so adjust with sufficient care !!!
+ top_version = max(FRAMEWORK_VERSION, LooseVersion(EASYBLOCKS_VERSION))
msg = "This is EasyBuild %s (framework: %s, easyblocks: %s) on host %s."
msg = msg % (top_version, FRAMEWORK_VERSION, EASYBLOCKS_VERSION, gethostname())
diff --git a/eb b/eb
index 0a510c70ad..132cf92ecf 100755
--- a/eb
+++ b/eb
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,6 +32,7 @@
# @author: Kenneth Hoste (Ghent University)
# @author: Pieter De Baets (Ghent University)
# @author: Jens Timmerman (Ghent University)
+# @author: Simon Branford (University of Birmingham)
keyboard_interrupt() {
echo "Keyboard interrupt!"
@@ -40,9 +41,8 @@ keyboard_interrupt() {
trap keyboard_interrupt SIGINT
-# Python 2.7+ or 3.5+ required
-REQ_MIN_PY2VER=7
-REQ_MIN_PY3VER=5
+# Python 3.6+ required
+REQ_MIN_PY3VER=6
EASYBUILD_MAIN='easybuild.main'
@@ -60,7 +60,7 @@ PYTHON=
# - EB_INSTALLPYTHON is set when EasyBuild is installed as a module (by EasyBuild). It is set to the PYTHON
# used during that installation (for example, you could override PYTHON using EB_PYTHON at installation
# time, this variable preserves that choice).
-for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python' 'python3' 'python2'; do
+for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python3' 'python'; do
# Only consider non-empty values, i.e. continue if e.g. $EB_PYTHON is not set
[ -n "${python_cmd}" ] || continue
@@ -76,10 +76,7 @@ for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python' 'python3' 'pytho
pyver_maj=$(echo "${pyver}" | cut -f1 -d'.')
pyver_min=$(echo "${pyver}" | cut -f2 -d'.')
- if [ "${pyver_maj}" -eq 2 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY2VER}" ]; then
- verbose "'${python_cmd}' version: ${pyver}, which matches Python 2 version requirement (>= 2.${REQ_MIN_PY2VER})"
- PYTHON="${python_cmd}"
- elif [ "${pyver_maj}" -eq 3 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY3VER}" ]; then
+ if [ "${pyver_maj}" -eq 3 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY3VER}" ]; then
verbose "'${python_cmd}' version: ${pyver}, which matches Python 3 version requirement (>= 3.${REQ_MIN_PY3VER})"
PYTHON="${python_cmd}"
fi
@@ -106,7 +103,7 @@ done
if [ -z "${PYTHON}" ]; then
echo -n "ERROR: No compatible 'python' command found via \$PATH " >&2
- echo "(EasyBuild requires Python 2.${REQ_MIN_PY2VER}+ or 3.${REQ_MIN_PY3VER}+)" >&2
+ echo "(EasyBuild requires Python 3.${REQ_MIN_PY3VER}+)" >&2
exit 1
else
verbose "Selected Python command: ${python_cmd} ($(command -v "${python_cmd}"))"
diff --git a/requirements.txt b/requirements.txt
index d2ce2ccdc1..699cc90372 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,10 @@
# keyring is required to provide GitHub token to EasyBuild;
# for recent versions of keyring, keyrings.alt must be installed too
-# 19.0 dropped Python 2 support
-keyring<19.0; python_version < '3.0'
-keyring; python_version >= '3.0'
+keyring
keyrings.alt
-# GitPython 3.1.15 deprecates Python 3.5
-GitPython<3.1.15; python_version >= '3.0' and python_version < '3.6'
-GitPython; python_version >= '3.6' or python_version <= '3.0'
-
-# autopep8
-# stick to older autopep8 with Python 2.7, since autopep8 1.7.0 requires pycodestyle>=2.9.1 (which is Python 3 only)
-autopep8<1.7.0; python_version < '3.0'
-autopep8; python_version >= '3.0'
-
-# PyYAML
+GitPython
+autopep8
PyYAML
# optional Python packages for EasyBuild
@@ -22,18 +12,11 @@ PyYAML
# flake8 is a superset of pycodestyle
flake8
-# 2.6.7 uses invalid Python 2 syntax
-GC3Pie!=2.6.7; python_version < '3.0'
-GC3Pie; python_version >= '3.0' and python_version < '3.11'
+GC3Pie; python_version < '3.11'
python-graph-dot
python-hglib
requests
archspec
-# cryptography 3.4.0 no longer supports Python 2.7
-cryptography==3.3.2; python_version == '2.7'
-cryptography; python_version >= '3.5' and python_version < '3.11'
-
-# rich is only supported for Python 3.6+
-rich; python_version >= '3.6'
+rich
diff --git a/setup.py b/setup.py
index 82bae3aa2f..c305735e13 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -24,16 +24,17 @@
##
"""
This script can be used to install easybuild-framework, e.g. using:
- easy_install --user .
-or
- python setup.py --prefix=$HOME/easybuild
+ python setup.py install --prefix=$HOME/easybuild
@author: Kenneth Hoste (Ghent University)
"""
import glob
import os
-from distutils import log
-from distutils.core import setup
+import logging
+try:
+ from distutils.core import setup
+except ImportError:
+ from setuptools import setup
from easybuild.tools.version import VERSION
@@ -45,8 +46,10 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
-# log levels: 0 = WARN (default), 1 = INFO, 2 = DEBUG
-log.set_verbosity(1)
+log = logging.getLogger("EasyBuild")
+
+# log levels: NOTSET (default), DEBUG, INFO, WARNING, ERROR, CRITICAL
+log.setLevel(logging.INFO)
log.info("Installing version %s (API version %s)" % (VERSION, API_VERSION))
@@ -96,7 +99,6 @@ def find_rel_test():
'eb_bash_completion.bash',
'eb_bash_completion_local.bash',
# utility scripts
- 'easybuild/scripts/bootstrap_eb.py',
'easybuild/scripts/install_eb_dep.sh',
],
data_files=[
@@ -111,14 +113,14 @@ def find_rel_test():
"Intended Audience :: System Administrators",
"License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
"Operating System :: POSIX :: Linux",
- "Programming Language :: Python :: 2.7",
- "Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
"Topic :: Software Development :: Build Tools",
],
platforms="Linux",
diff --git a/test/__init__.py b/test/__init__.py
index 20c9130dcc..939c300f79 100644
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/__init__.py b/test/framework/__init__.py
index fae73ca9e6..df6adea40d 100644
--- a/test/framework/__init__.py
+++ b/test/framework/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py
index c7eced6c48..09045e0e01 100644
--- a/test/framework/asyncprocess.py
+++ b/test/framework/asyncprocess.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,7 +35,7 @@
import easybuild.tools.asyncprocess as p
from easybuild.tools.asyncprocess import Popen
-from easybuild.tools.py2vs3 import subprocess_terminate
+from easybuild.tools.run import subprocess_terminate
class AsyncProcessTest(EnhancedTestCase):
diff --git a/test/framework/build_log.py b/test/framework/build_log.py
index 15f6984099..4b29a2af26 100644
--- a/test/framework/build_log.py
+++ b/test/framework/build_log.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -116,11 +116,12 @@ def test_easybuildlog(self):
stderr = self.get_stderr()
self.mock_stderr(False)
- more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
+ more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information"
+ common_warning = "\nWARNING: Deprecated functionality, will no longer work in"
expected_stderr = '\n\n'.join([
- "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
- "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning",
- "\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning",
+ common_warning + " EasyBuild v10000001: anotherwarning; " + more_info,
+ common_warning + " EasyBuild v2.0: onemorewarning",
+ common_warning + " EasyBuild v2.0: lastwarning",
]) + '\n\n'
self.assertEqual(stderr, expected_stderr)
@@ -139,8 +140,8 @@ def test_easybuildlog(self):
r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*",
r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*",
r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*thisisnotprinted.*",
- r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput",
- r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
+ r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput",
+ r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): err: msg: %s",
r"fancyroot.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops",
'',
])
@@ -168,7 +169,7 @@ def test_easybuildlog(self):
r"fancyroot.test_easybuildlog \[WARNING\] :: bleh",
r"fancyroot.test_easybuildlog \[INFO\] :: 4\+2 = 42",
r"fancyroot.test_easybuildlog \[DEBUG\] :: this is just a test",
- r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz",
+ r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): foo baz baz",
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
@@ -183,7 +184,7 @@ def test_easybuildlog(self):
self.mock_stderr(False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
- "[WARNING] :: Deprecated functionality, will no longer work in v10000001: ",
+ "[WARNING] :: Deprecated functionality, will no longer work in EasyBuild v10000001: ",
"this is just a test",
"(see URLGOESHERE for more information)",
])
@@ -208,7 +209,7 @@ def test_log_levels(self):
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
- log.warn('this is a warning')
+ log.warning('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
@@ -223,7 +224,7 @@ def test_log_levels(self):
info_msg = r"%s \[INFO\] :: fyi" % prefix
warning_msg = r"%s \[WARNING\] :: this is a warning" % prefix
deprecated_msg = r"%s \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % prefix
- error_msg = r"%s \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % prefix
+ error_msg = r"%s \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput" % prefix
expected_logtxt = '\n'.join([
error_msg,
diff --git a/test/framework/config.py b/test/framework/config.py
index 55234d31b9..9614d4fb31 100644
--- a/test/framework/config.py
+++ b/test/framework/config.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,20 +33,19 @@
import shutil
import sys
import tempfile
+from importlib import reload
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
import easybuild.tools.options as eboptions
-from easybuild.tools import run
from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables
from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath
from easybuild.tools.config import install_path, log_file_format, log_path, source_paths
from easybuild.tools.config import update_build_option, update_build_options
-from easybuild.tools.config import BuildOptions, ConfigurationVariables
from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options
from easybuild.tools.filetools import copy_dir, mkdir, write_file
from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX
-from easybuild.tools.py2vs3 import reload
class EasyBuildConfigTest(EnhancedTestCase):
@@ -449,7 +448,7 @@ def test_XDG_CONFIG_env_vars(self):
# $XDG_CONFIG_HOME not set, multiple directories listed in $XDG_CONFIG_DIRS
del os.environ['XDG_CONFIG_HOME'] # unset, so should become default
- os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir1, dir2, dir3])
+ os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir3, dir2, dir1])
cfg_files = [
os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
@@ -580,9 +579,9 @@ def test_flex_robot_paths(self):
def test_strict(self):
"""Test use of --strict."""
# check default
- self.assertEqual(build_option('strict'), run.WARN)
+ self.assertEqual(build_option('strict'), WARN)
- for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]:
+ for strict_str, strict_val in [('error', ERROR), ('ignore', IGNORE), ('warn', WARN)]:
options = init_config(args=['--strict=%s' % strict_str])
init_config(build_options={'strict': options.strict})
self.assertEqual(build_option('strict'), strict_val)
diff --git a/test/framework/containers.py b/test/framework/containers.py
index 1d9abbf8e4..321df783d2 100644
--- a/test/framework/containers.py
+++ b/test/framework/containers.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2018-2024 Ghent University
+# Copyright 2018-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/docs.py b/test/framework/docs.py
index 6b5b68ad17..4b281df14a 100644
--- a/test/framework/docs.py
+++ b/test/framework/docs.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -25,7 +25,6 @@
"""
Unit tests for docs.py.
"""
-import inspect
import os
import re
import sys
@@ -38,106 +37,197 @@
from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains
from easybuild.tools.docs import md_title_and_table, rst_title_and_table
from easybuild.tools.options import EasyBuildOptions
-from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table
+from easybuild.tools.utilities import mk_md_table, mk_rst_table
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock
|-- bar
+|-- Bundle
+|-- CMakeMake
+|-- CmdCp
|-- ConfigureMake
| |-- MakeCp
+|-- EB_binutils
+|-- EB_bzip2
+|-- EB_CMake
|-- EB_EasyBuildMeta
|-- EB_FFTW
+|-- EB_FFTW_period_MPI
+|-- EB_flex
|-- EB_foo
| |-- EB_foofoo
+|-- EB_freetype
|-- EB_GCC
|-- EB_HPL
|-- EB_libtoy
+|-- EB_libxml2
+|-- EB_LLVM
+|-- EB_Mesa
|-- EB_OpenBLAS
|-- EB_OpenMPI
+|-- EB_OpenSSL_wrapper
+|-- EB_Perl
+|-- EB_Python
|-- EB_ScaLAPACK
|-- EB_toy_buggy
+|-- EB_XCrySDen
|-- ExtensionEasyBlock
| |-- DummyExtension
+| | |-- CustomDummyExtension
+| | | |-- ChildCustomDummyExtension
+| | |-- DeprecatedDummyExtension
+| | | |-- ChildDeprecatedDummyExtension
| |-- EB_toy
+| | |-- EB_toy_deprecated
| | |-- EB_toy_eula
| | |-- EB_toytoy
| |-- Toy_Extension
+|-- MesonNinja
|-- ModuleRC
+|-- PerlBundle
|-- PythonBundle
+|-- PythonPackage
+|-- Tarball
|-- Toolchain
Extension
|-- ExtensionEasyBlock
| |-- DummyExtension
+| | |-- CustomDummyExtension
+| | | |-- ChildCustomDummyExtension
+| | |-- DeprecatedDummyExtension
+| | | |-- ChildDeprecatedDummyExtension
| |-- EB_toy
+| | |-- EB_toy_deprecated
| | |-- EB_toy_eula
| | |-- EB_toytoy
-| |-- Toy_Extension"""
+| |-- Toy_Extension""" # noqa
LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock)
|-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
+|-- Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+|-- CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
+|-- CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
|-- ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
| |-- MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py)
+|-- EB_binutils (easybuild.easyblocks.binutils @ %(topdir)s/b/binutils.py)
+|-- EB_bzip2 (easybuild.easyblocks.bzip2 @ %(topdir)s/b/bzip2.py)
+|-- EB_CMake (easybuild.easyblocks.cmake @ %(topdir)s/c/cmake.py)
|-- EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py)
|-- EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py)
+|-- EB_FFTW_period_MPI (easybuild.easyblocks.fftwmpi @ %(topdir)s/f/fftwmpi.py)
+|-- EB_flex (easybuild.easyblocks.flex @ %(topdir)s/f/flex.py)
|-- EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py)
| |-- EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py)
+|-- EB_freetype (easybuild.easyblocks.freetype @ %(topdir)s/f/freetype.py)
|-- EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py)
|-- EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py)
|-- EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py)
+|-- EB_libxml2 (easybuild.easyblocks.libxml2 @ %(topdir)s/l/libxml2.py)
+|-- EB_LLVM (easybuild.easyblocks.llvm @ %(topdir)s/l/llvm.py)
+|-- EB_Mesa (easybuild.easyblocks.mesa @ %(topdir)s/m/mesa.py)
|-- EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py)
|-- EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py)
+|-- EB_OpenSSL_wrapper (easybuild.easyblocks.openssl_wrapper @ %(topdir)s/o/openssl_wrapper.py)
+|-- EB_Perl (easybuild.easyblocks.perl @ %(topdir)s/p/perl.py)
+|-- EB_Python (easybuild.easyblocks.python @ %(topdir)s/p/python.py)
|-- EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py)
|-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py)
+|-- EB_XCrySDen (easybuild.easyblocks.xcrysden @ %(topdir)s/x/xcrysden.py)
|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+| | |-- EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)
+|-- MesonNinja (easybuild.easyblocks.generic.mesonninja @ %(topdir)s/generic/mesonninja.py)
|-- ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py)
+|-- PerlBundle (easybuild.easyblocks.generic.perlbundle @ %(topdir)s/generic/perlbundle.py)
|-- PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py)
+|-- PythonPackage (easybuild.easyblocks.generic.pythonpackage @ %(topdir)s/generic/pythonpackage.py)
+|-- Tarball (easybuild.easyblocks.generic.tarball @ %(topdir)s/generic/tarball.py)
|-- Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py)
Extension (easybuild.framework.extension)
|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+| | |-- EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
-| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)"""
+| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa
LIST_EASYBLOCKS_SIMPLE_RST = """* **EasyBlock**
* bar
+ * Bundle
+ * CMakeMake
+ * CmdCp
* ConfigureMake
* MakeCp
+ * EB_binutils
+ * EB_bzip2
+ * EB_CMake
* EB_EasyBuildMeta
* EB_FFTW
+ * EB_FFTW_period_MPI
+ * EB_flex
* EB_foo
* EB_foofoo
+ * EB_freetype
* EB_GCC
* EB_HPL
* EB_libtoy
+ * EB_libxml2
+ * EB_LLVM
+ * EB_Mesa
* EB_OpenBLAS
* EB_OpenMPI
+ * EB_OpenSSL_wrapper
+ * EB_Perl
+ * EB_Python
* EB_ScaLAPACK
* EB_toy_buggy
+ * EB_XCrySDen
* ExtensionEasyBlock
* DummyExtension
+
+ * CustomDummyExtension
+
+ * ChildCustomDummyExtension
+
+ * DeprecatedDummyExtension
+
+ * ChildDeprecatedDummyExtension
+
+
* EB_toy
+ * EB_toy_deprecated
* EB_toy_eula
* EB_toytoy
* Toy_Extension
+ * MesonNinja
* ModuleRC
+ * PerlBundle
* PythonBundle
+ * PythonPackage
+ * Tarball
* Toolchain
* **Extension**
@@ -145,47 +235,89 @@
* ExtensionEasyBlock
* DummyExtension
+
+ * CustomDummyExtension
+
+ * ChildCustomDummyExtension
+
+ * DeprecatedDummyExtension
+
+ * ChildDeprecatedDummyExtension
+
+
* EB_toy
+ * EB_toy_deprecated
* EB_toy_eula
* EB_toytoy
* Toy_Extension
-"""
+""" # noqa
LIST_EASYBLOCKS_DETAILED_RST = """* **EasyBlock** (easybuild.framework.easyblock)
* bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
+ * Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+ * CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
+ * CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
* ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
* MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py)
+ * EB_binutils (easybuild.easyblocks.binutils @ %(topdir)s/b/binutils.py)
+ * EB_bzip2 (easybuild.easyblocks.bzip2 @ %(topdir)s/b/bzip2.py)
+ * EB_CMake (easybuild.easyblocks.cmake @ %(topdir)s/c/cmake.py)
* EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py)
* EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py)
+ * EB_FFTW_period_MPI (easybuild.easyblocks.fftwmpi @ %(topdir)s/f/fftwmpi.py)
+ * EB_flex (easybuild.easyblocks.flex @ %(topdir)s/f/flex.py)
* EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py)
* EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py)
+ * EB_freetype (easybuild.easyblocks.freetype @ %(topdir)s/f/freetype.py)
* EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py)
* EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py)
* EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py)
+ * EB_libxml2 (easybuild.easyblocks.libxml2 @ %(topdir)s/l/libxml2.py)
+ * EB_LLVM (easybuild.easyblocks.llvm @ %(topdir)s/l/llvm.py)
+ * EB_Mesa (easybuild.easyblocks.mesa @ %(topdir)s/m/mesa.py)
* EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py)
* EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py)
+ * EB_OpenSSL_wrapper (easybuild.easyblocks.openssl_wrapper @ %(topdir)s/o/openssl_wrapper.py)
+ * EB_Perl (easybuild.easyblocks.perl @ %(topdir)s/p/perl.py)
+ * EB_Python (easybuild.easyblocks.python @ %(topdir)s/p/python.py)
* EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py)
* EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py)
+ * EB_XCrySDen (easybuild.easyblocks.xcrysden @ %(topdir)s/x/xcrysden.py)
* ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
* DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+
+ * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+
+ * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+
+ * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+
+ * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
+
+
* EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+ * EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
* EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
* EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
* Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)
+ * MesonNinja (easybuild.easyblocks.generic.mesonninja @ %(topdir)s/generic/mesonninja.py)
* ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py)
+ * PerlBundle (easybuild.easyblocks.generic.perlbundle @ %(topdir)s/generic/perlbundle.py)
* PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py)
+ * PythonPackage (easybuild.easyblocks.generic.pythonpackage @ %(topdir)s/generic/pythonpackage.py)
+ * Tarball (easybuild.easyblocks.generic.tarball @ %(topdir)s/generic/tarball.py)
* Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py)
* **Extension** (easybuild.framework.extension)
@@ -193,82 +325,153 @@
* ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
* DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+
+ * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+
+ * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+
+ * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+
+ * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
+
+
* EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+ * EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
* EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
* EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
* Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)
-"""
+""" # noqa
LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock**
- bar
+ - Bundle
+ - CMakeMake
+ - CmdCp
- ConfigureMake
- MakeCp
+ - EB_binutils
+ - EB_bzip2
+ - EB_CMake
- EB_EasyBuildMeta
- EB_FFTW
+ - EB_FFTW_period_MPI
+ - EB_flex
- EB_foo
- EB_foofoo
+ - EB_freetype
- EB_GCC
- EB_HPL
- EB_libtoy
+ - EB_libxml2
+ - EB_LLVM
+ - EB_Mesa
- EB_OpenBLAS
- EB_OpenMPI
+ - EB_OpenSSL_wrapper
+ - EB_Perl
+ - EB_Python
- EB_ScaLAPACK
- EB_toy_buggy
+ - EB_XCrySDen
- ExtensionEasyBlock
- DummyExtension
+ - CustomDummyExtension
+ - ChildCustomDummyExtension
+ - DeprecatedDummyExtension
+ - ChildDeprecatedDummyExtension
- EB_toy
+ - EB_toy_deprecated
- EB_toy_eula
- EB_toytoy
- Toy_Extension
+ - MesonNinja
- ModuleRC
+ - PerlBundle
- PythonBundle
+ - PythonPackage
+ - Tarball
- Toolchain
- **Extension**
- ExtensionEasyBlock
- DummyExtension
+ - CustomDummyExtension
+ - ChildCustomDummyExtension
+ - DeprecatedDummyExtension
+ - ChildDeprecatedDummyExtension
- EB_toy
+ - EB_toy_deprecated
- EB_toy_eula
- EB_toytoy
- - Toy_Extension"""
+ - Toy_Extension""" # noqa
LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock)
- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
+ - Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+ - CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
+ - CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
- ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
- MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py)
+ - EB_binutils (easybuild.easyblocks.binutils @ %(topdir)s/b/binutils.py)
+ - EB_bzip2 (easybuild.easyblocks.bzip2 @ %(topdir)s/b/bzip2.py)
+ - EB_CMake (easybuild.easyblocks.cmake @ %(topdir)s/c/cmake.py)
- EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py)
- EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py)
+ - EB_FFTW_period_MPI (easybuild.easyblocks.fftwmpi @ %(topdir)s/f/fftwmpi.py)
+ - EB_flex (easybuild.easyblocks.flex @ %(topdir)s/f/flex.py)
- EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py)
- EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py)
+ - EB_freetype (easybuild.easyblocks.freetype @ %(topdir)s/f/freetype.py)
- EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py)
- EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py)
- EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py)
+ - EB_libxml2 (easybuild.easyblocks.libxml2 @ %(topdir)s/l/libxml2.py)
+ - EB_LLVM (easybuild.easyblocks.llvm @ %(topdir)s/l/llvm.py)
+ - EB_Mesa (easybuild.easyblocks.mesa @ %(topdir)s/m/mesa.py)
- EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py)
- EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py)
+ - EB_OpenSSL_wrapper (easybuild.easyblocks.openssl_wrapper @ %(topdir)s/o/openssl_wrapper.py)
+ - EB_Perl (easybuild.easyblocks.perl @ %(topdir)s/p/perl.py)
+ - EB_Python (easybuild.easyblocks.python @ %(topdir)s/p/python.py)
- EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py)
- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py)
+ - EB_XCrySDen (easybuild.easyblocks.xcrysden @ %(topdir)s/x/xcrysden.py)
- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+ - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+ - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+ - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+ - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+ - EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)
+ - MesonNinja (easybuild.easyblocks.generic.mesonninja @ %(topdir)s/generic/mesonninja.py)
- ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py)
+ - PerlBundle (easybuild.easyblocks.generic.perlbundle @ %(topdir)s/generic/perlbundle.py)
- PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py)
+ - PythonPackage (easybuild.easyblocks.generic.pythonpackage @ %(topdir)s/generic/pythonpackage.py)
+ - Tarball (easybuild.easyblocks.generic.tarball @ %(topdir)s/generic/tarball.py)
- Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py)
- **Extension** (easybuild.framework.extension)
- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+ - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+ - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+ - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+ - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
+ - EB_toy_deprecated (easybuild.easyblocks.toy_deprecated @ %(topdir)s/t/toy_deprecated.py)
- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
- - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)"""
+ - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa
LIST_SOFTWARE_SIMPLE_TXT = """
* GCC
-* gzip"""
+* gzip""" # noqa
GCC_DESCR = "The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, "
GCC_DESCR += "as well as libraries for these languages (libstdc++, libgcj,...)."
@@ -291,7 +494,7 @@
* gzip v1.4: GCC/4.6.3, system
* gzip v1.5: foss/2018a, intel/2018a
-""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_RST = """List of supported software
==========================
@@ -307,7 +510,7 @@
---
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_RST = """List of supported software
==========================
@@ -357,7 +560,7 @@
``1.4`` ``GCC/4.6.3``, ``system``
``1.5`` ``foss/2018a``, ``intel/2018a``
======= ===============================
-""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
@@ -369,7 +572,7 @@
## G
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_MD = """# List of supported software
@@ -403,7 +606,7 @@
version|toolchain
-------|-------------------------------
``1.4``|``GCC/4.6.3``, ``system``
-``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
@@ -415,7 +618,7 @@
## G
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_MD = """# List of supported software
@@ -449,7 +652,7 @@
version|toolchain
-------|-------------------------------
``1.4``|``GCC/4.6.3``, ``system``
-``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_JSON = """[
{
@@ -458,7 +661,7 @@
{
"name": "gzip"
}
-]"""
+]""" # noqa
LIST_SOFTWARE_DETAILED_JSON = """[
{
@@ -501,7 +704,7 @@
"version": "1.5",
"versionsuffix": ""
}
-]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
class DocsTest(EnhancedTestCase):
@@ -513,14 +716,33 @@ def test_get_easyblock_classes(self):
# result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic
eb_classes = get_easyblock_classes('easybuild.easyblocks.generic')
eb_names = [x.__name__ for x in eb_classes]
- expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC',
- 'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar']
+ expected = [
+ 'Bundle',
+ 'CMakeMake',
+ 'ChildCustomDummyExtension',
+ 'ChildDeprecatedDummyExtension',
+ 'CmdCp',
+ 'ConfigureMake',
+ 'CustomDummyExtension',
+ 'DeprecatedDummyExtension',
+ 'DummyExtension',
+ 'MakeCp',
+ 'MesonNinja',
+ 'ModuleRC',
+ 'PerlBundle',
+ 'PythonBundle',
+ 'PythonPackage',
+ 'Tarball',
+ 'Toolchain',
+ 'Toy_Extension',
+ 'bar',
+ ]
self.assertEqual(sorted(eb_names), expected)
def test_gen_easyblocks_overview(self):
""" Test gen_easyblocks_overview_* functions """
gen_easyblocks_pkg = 'easybuild.easyblocks.generic'
- modules = import_available_modules(gen_easyblocks_pkg)
+ names = [eb_class.__name__ for eb_class in get_easyblock_classes(gen_easyblocks_pkg)]
common_params = {
'ConfigureMake': ['configopts', 'buildopts', 'installopts'],
}
@@ -564,15 +786,9 @@ def test_gen_easyblocks_overview(self):
])
self.assertIn(check_configuremake, ebdoc)
- names = []
- for mod in modules:
- for name, _ in inspect.getmembers(mod, inspect.isclass):
- eb_class = getattr(mod, name)
- # skip imported classes that are not easyblocks
- if eb_class.__module__.startswith(gen_easyblocks_pkg):
- self.assertIn(name, ebdoc)
- names.append(name)
+ for name in names:
+ self.assertIn(name, ebdoc)
toc = [":ref:`" + n + "`" for n in sorted(set(names))]
pattern = " - ".join(toc)
@@ -610,17 +826,11 @@ def test_gen_easyblocks_overview(self):
])
self.assertIn(check_configuremake, ebdoc)
- names = []
- for mod in modules:
- for name, _ in inspect.getmembers(mod, inspect.isclass):
- eb_class = getattr(mod, name)
- # skip imported classes that are not easyblocks
- if eb_class.__module__.startswith(gen_easyblocks_pkg):
- self.assertIn(name, ebdoc)
- names.append(name)
+ for name in names:
+ self.assertIn(name, ebdoc)
- toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(set(names))]
+ toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(names)]
pattern = " - ".join(toc)
regex = re.compile(pattern)
self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 19218eca6a..de3b3e5198 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -28,6 +28,7 @@
@author: Jens Timmerman (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Maxime Boissonneault (Compute Canada)
+@author: Jan Andre Reuter (Juelich Supercomputing Centre)
"""
import os
import re
@@ -35,6 +36,7 @@
import sys
import tempfile
from inspect import cleandoc
+from test.framework.github import requires_github_access
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
@@ -48,12 +50,12 @@
from easybuild.tools import LooseVersion, config
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import get_module_syntax, update_build_option
+from easybuild.tools.environment import modify_env
from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file
-from easybuild.tools.filetools import verify_checksum, write_file
+from easybuild.tools.filetools import symlink, verify_checksum, write_file
from easybuild.tools.module_generator import module_generator
-from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches
+from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches
from easybuild.tools.version import get_git_revision, this_is_easybuild
-from easybuild.tools.py2vs3 import string_type
class EasyBlockTest(EnhancedTestCase):
@@ -195,18 +197,17 @@ def test_load_module(self):
self.mock_stdout(True)
eb.prepare_step(start_dir=False)
stderr = self.get_stderr()
- stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertFalse(stdout)
self.assertTrue(stderr.strip().startswith("WARNING: Long $TMPDIR path may cause problems with OpenMPI 2.x"))
# we expect $TMPDIR to be tweaked by the prepare step (OpenMPI 2.x doesn't like long $TMPDIR values)
tweaked_tmpdir = os.environ.get('TMPDIR')
self.assertNotEqual(tweaked_tmpdir, orig_tmpdir)
- eb.make_module_step()
- eb.load_module()
+ with self.mocked_stdout_stderr():
+ eb.make_module_step()
+ eb.load_module()
# $TMPDIR does *not* get reset to original value after loading of module
# (which involves resetting the environment before loading the module)
@@ -286,8 +287,11 @@ def test_make_module_extend_modpath(self):
txt = eb.make_module_extend_modpath()
if module_syntax == 'Tcl':
regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
- home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
- home += r'else { concat "HOME_NOT_DEFINED" } \]'
+ if self.modtool.supports_tcl_getenv:
+ home = r'\[getenv HOME "HOME_NOT_DEFINED"\]'
+ else:
+ home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
+ home += r'else { concat "HOME_NOT_DEFINED" } \]'
fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
regexs.extend([
# extension for user modules is guarded
@@ -312,7 +316,7 @@ def test_make_module_extend_modpath(self):
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
- # Repeat this but using an alternate envvars (instead of $HOME)
+ # Repeat this but using an alternative envvars (instead of $HOME)
list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS']
build_options = {
@@ -329,9 +333,12 @@ def test_make_module_extend_modpath(self):
for envvar in list_of_envvars:
if module_syntax == 'Tcl':
regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
- module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
- module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
- module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
+ if self.modtool.supports_tcl_getenv:
+ module_envvar = r'\[getenv %s "%s"]' % (envvar, envvar + '_NOT_DEFINED')
+ else:
+ module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
+ module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
+ module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
regexs.extend([
# extension for user modules is guarded
@@ -429,12 +436,13 @@ def test_make_module_req(self):
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
+ for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'):
+ path_components = (path, ) if isinstance(path, str) else path
+ os.mkdir(os.path.join(eb.installdir, *path_components))
+
write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar')
write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar')
- for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'):
- if isinstance(path, string_type):
- path = (path, )
- os.mkdir(os.path.join(eb.installdir, *path))
+ write_file(os.path.join(eb.installdir, 'share', 'man', 'pi'), 'Man page')
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
@@ -514,8 +522,18 @@ def test_make_module_req(self):
self.assertEqual(len(re.findall(r'^prepend_path\("%s", pathJoin\(root, "lib"\)\)$' % var,
guess, re.M)), 1)
- # check for behavior when a string value is used as dict value by make_module_req_guesses
- eb.make_module_req_guess = lambda: {'PATH': 'bin'}
+ # nuke default module load environment
+ default_mod_load_vars = [
+ 'ACLOCAL_PATH', 'CLASSPATH', 'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH', 'CPATH', 'GI_TYPELIB_PATH',
+ 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'MANPATH', 'PATH', 'PKG_CONFIG_PATH', 'XDG_DATA_DIRS',
+ ]
+ for env_var in default_mod_load_vars:
+ eb.module_load_environment.remove(env_var)
+
+ self.assertEqual(len(eb.module_load_environment.vars), 0)
+
+ # check for behavior when a string value is used as value of module_load_environment
+ eb.module_load_environment.PATH = 'bin'
with eb.module_generator.start_module_creation():
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
@@ -527,7 +545,7 @@ def test_make_module_req(self):
# check for correct behaviour if empty string is specified as one of the values
# prepend-path statements should be included for both the 'bin' subdir and the install root
- eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
+ eb.module_load_environment.PATH = ['bin', '']
with eb.module_generator.start_module_creation():
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
@@ -540,7 +558,7 @@ def test_make_module_req(self):
self.fail("Unknown module syntax: %s" % get_module_syntax())
# check for correct order of prepend statements when providing a list (and that no duplicates are allowed)
- eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']}
+ eb.module_load_environment.LD_LIBRARY_PATH = ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']
for path in ['pathA', 'pathB', 'pathC']:
os.mkdir(os.path.join(eb.installdir, 'lib', path))
write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test')
@@ -568,12 +586,14 @@ def test_make_module_req(self):
# If PATH or LD_LIBRARY_PATH contain only folders, do not add an entry
sub_lib_path = os.path.join('lib', 'path_folders')
sub_path_path = os.path.join('bin', 'path_folders')
- eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': sub_lib_path, 'PATH': sub_path_path}
+ eb.module_load_environment.LD_LIBRARY_PATH = sub_lib_path
+ eb.module_load_environment.PATH = sub_path_path
for path in (sub_lib_path, sub_path_path):
full_path = os.path.join(eb.installdir, path, 'subpath')
os.makedirs(full_path)
write_file(os.path.join(full_path, 'any.file'), 'test')
- txt = eb.make_module_req()
+ with eb.module_generator.start_module_creation():
+ txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search(r"prepend-path\s+LD_LIBRARY_PATH\s+\$%s\n" % sub_lib_path,
txt, re.M))
@@ -584,6 +604,231 @@ def test_make_module_req(self):
txt, re.M))
self.assertFalse(re.search(r'prepend_path\("PATH", pathJoin\(root, "%s"\)\)\n' % sub_path_path, txt, re.M))
+ # Module load environement may contain non-path variables
+ # TODO: remove whenever this is properly supported, in the meantime check warning
+ eb.module_load_environment.NONPATH = {'contents': 'non_path', 'var_type': "STRING"}
+ eb.module_load_environment.PATH = ['bin']
+ with self.mocked_stdout_stderr():
+ txt = eb.make_module_req()
+
+ self.assertEqual(list(eb.module_load_environment), ['PATH', 'LD_LIBRARY_PATH', 'NONPATH'])
+
+ if get_module_syntax() == 'Tcl':
+ self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
+ self.assertFalse(re.match(r"^\nprepend-path\s+NONPATH\s+\$root/non_path\n$", txt, re.M))
+ elif get_module_syntax() == 'Lua':
+ self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
+ self.assertFalse(re.match(r'^\nprepend_path\("NONPATH", pathJoin\(root, "non_path"\)\)\n$', txt, re.M))
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+
+ logtxt = read_file(eb.logfile)
+ self.assertTrue(re.search(r"WARNING Non-path variables found in module load env.*NONPATH", logtxt, re.M))
+
+ eb.module_load_environment.remove('NONPATH')
+
+ # make sure that entries that symlink to another directory are retained;
+ # the test case inspired by the directory structure for old imkl versions (like 2020.4)
+ remove_dir(eb.installdir)
+ # lib/ symlinked to libraries/
+ real_libdir = os.path.join(eb.installdir, 'libraries')
+ mkdir(real_libdir, parents=True)
+ symlink(real_libdir, os.path.join(eb.installdir, 'lib'))
+ # lib/intel64/ symlinked to lib/intel64_lin/
+ mkdir(os.path.join(eb.installdir, 'lib', 'intel64_lin'), parents=True)
+ symlink(os.path.join(eb.installdir, 'lib', 'intel64_lin'), os.path.join(eb.installdir, 'lib', 'intel64'))
+ # library file present in lib/intel64
+ write_file(os.path.join(eb.installdir, 'lib', 'intel64', 'libfoo.so'), 'libfoo.so')
+ # lib64/ symlinked to lib/
+ symlink(os.path.join(eb.installdir, 'lib'), os.path.join(eb.installdir, 'lib64'))
+
+ eb.module_load_environment.LD_LIBRARY_PATH = [os.path.join('lib', 'intel64')]
+ eb.module_load_environment.LIBRARY_PATH = eb.module_load_environment.LD_LIBRARY_PATH
+ with eb.module_generator.start_module_creation():
+ txt = eb.make_module_req()
+
+ if get_module_syntax() == 'Tcl':
+ self.assertTrue(re.search(r"^prepend-path\s+LD_LIBRARY_PATH\s+\$root/libraries/intel64_lin$", txt, re.M))
+ self.assertTrue(re.search(r"^prepend-path\s+LIBRARY_PATH\s+\$root/libraries/intel64_lin\n$", txt, re.M))
+ elif get_module_syntax() == 'Lua':
+ self.assertTrue(re.search(r'^prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "libraries/intel64_lin"\)\)$',
+ txt, re.M))
+ self.assertTrue(re.search(r'^prepend_path\("LIBRARY_PATH", pathJoin\(root, "libraries/intel64_lin"\)\)$',
+ txt, re.M))
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+
+ # test absolute paths
+ eb.module_load_environment.PATH = ['/bin']
+
+ with eb.module_generator.start_module_creation():
+ txt = eb.make_module_req()
+
+ if get_module_syntax() == 'Tcl':
+ self.assertTrue(re.search(r"^prepend-path\s+PATH\s+/bin$", txt, re.M))
+ elif get_module_syntax() == 'Lua':
+ self.assertTrue(re.search(r'^prepend_path\("PATH", "/bin"\)$', txt, re.M))
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+
+ # make sure that relative entries that symlink to directories outside of install dir trigger an error
+ symlink("/bin", os.path.join(eb.installdir, 'bin'))
+ eb.module_load_environment.PATH = ['bin']
+
+ with eb.module_generator.start_module_creation():
+ err_regex = "Expansion of search path glob.*pointing outside of install directory.*"
+ self.assertErrorRegex(EasyBuildError, err_regex, eb.make_module_req)
+
+ # Test modextrapaths: with absolute + empty paths, appending and custom delimiters
+ remove_dir(eb.installdir)
+ self.contents += '\n'.join([
+ "",
+ "modextrapaths = {",
+ " 'PATH': [''],",
+ " 'TEST_VAR': ['foo', 'baz', '/bin'],",
+ " 'TEST_VAR_CUSTOM': {'paths': ['foo', 'baz'], 'delimiter': ';', 'prepend': False},",
+ " 'LD_LIBRARY_PATH': 'foo',",
+ " MODULE_LOAD_ENV_HEADERS: ['include/foo', 'include/bar'],",
+ "}",
+ ])
+ self.writeEC()
+ ec = EasyConfig(self.eb_file)
+ eb = EasyBlock(ec)
+ eb.installdir = config.install_path()
+
+ # populate install dir
+ mkdir(os.path.join(eb.installdir, 'lib'), parents=True)
+ write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'install lib')
+ mkdir(os.path.join(eb.installdir, 'foo'), parents=True)
+ write_file(os.path.join(eb.installdir, 'foo', 'bar'), 'install file')
+ mkdir(os.path.join(eb.installdir, 'baz'), parents=True)
+ mkdir(os.path.join(eb.installdir, 'include', 'foo'), parents=True)
+ write_file(os.path.join(eb.installdir, 'include', 'foo', 'foo.h'), 'header file')
+ mkdir(os.path.join(eb.installdir, 'include', 'bar'), parents=True)
+
+ with eb.module_generator.start_module_creation():
+ txt = eb.make_module_req()
+
+ expected_patterns = [
+ r"^append[-_]path.*TEST_VAR_CUSTOM.*root.*foo.*",
+ r"^prepend[-_]path.*CPATH.*root.*include.*",
+ r"^prepend[-_]path.*CPATH.*root.*include/foo.*",
+ r"^prepend[-_]path.*LD_LIBRARY_PATH.*root.*lib",
+ r"^prepend[-_]path.*LD_LIBRARY_PATH.*root.*foo",
+ r"^prepend[-_]path.*TEST_VAR.*root.*foo",
+ r"^prepend[-_]path.*TEST_VAR.*/bin",
+ ]
+ if get_module_syntax() == 'Tcl':
+ expected_patterns.append(r"^prepend-path\s+PATH\s+\$root$")
+ elif get_module_syntax() == 'Lua':
+ expected_patterns.append(r'^prepend_path\("PATH", root\)$')
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+
+ for pattern in expected_patterns:
+ self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt))
+
+ non_expected_patterns = [
+ r"^append[-_]path.*TEST_VAR_APPEND.*root.*baz",
+ r"^prepend[-_]path.*CPATH.*root.*include/bar.*",
+ r"^prepend[-_]path.*TEST_VAR.*root.*baz",
+ ]
+ for pattern in non_expected_patterns:
+ self.assertFalse(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt))
+
+ # cleanup
+ eb.close_log()
+ os.remove(eb.logfile)
+
+ def test_module_search_path_headers(self):
+ """Test functionality of module-search-path-headers option"""
+ sp_headers_mode = {
+ "cpath": ["CPATH"],
+ "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+ }
+
+ self.contents = '\n'.join([
+ 'easyblock = "ConfigureMake"',
+ 'name = "pi"',
+ 'version = "3.14"',
+ 'homepage = "http://example.com"',
+ 'description = "test easyconfig"',
+ 'toolchain = SYSTEM',
+ ])
+ self.writeEC()
+
+ for build_opt, sp_headers in sp_headers_mode.items():
+ update_build_option('module_search_path_headers', build_opt)
+ eb = EasyBlock(EasyConfig(self.eb_file))
+ eb.installdir = config.install_path()
+ try:
+ os.makedirs(os.path.join(eb.installdir, 'include'))
+ write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file')
+ except FileExistsError:
+ pass
+
+ with eb.module_generator.start_module_creation():
+ guess = eb.make_module_req()
+
+ if not sp_headers:
+ # none option adds nothing to module file
+ if get_module_syntax() == 'Tcl':
+ tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$"
+ self.assertFalse(re.search(tcl_ref_pattern, guess, re.M))
+ elif get_module_syntax() == 'Lua':
+ lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$'
+ self.assertFalse(re.search(lua_ref_pattern, guess, re.M))
+ else:
+ for env_var in sp_headers:
+ if get_module_syntax() == 'Tcl':
+ tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$"
+ self.assertTrue(re.search(tcl_ref_pattern, guess, re.M))
+ elif get_module_syntax() == 'Lua':
+ lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$'
+ self.assertTrue(re.search(lua_ref_pattern, guess, re.M))
+
+ # test with easyconfig parameter
+ for ec_param, sp_headers in sp_headers_mode.items():
+ self.contents += f'\nmodule_search_path_headers = "{ec_param}"'
+ self.writeEC()
+ eb = EasyBlock(EasyConfig(self.eb_file))
+ eb.installdir = config.install_path()
+ try:
+ os.makedirs(os.path.join(eb.installdir, 'include'))
+ write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file')
+ except FileExistsError:
+ pass
+
+ for build_opt in sp_headers_mode:
+ update_build_option('module_search_path_headers', build_opt)
+ with eb.module_generator.start_module_creation():
+ guess = eb.make_module_req()
+ if not sp_headers:
+ # none option adds nothing to module file
+ if get_module_syntax() == 'Tcl':
+ tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$"
+ self.assertFalse(re.search(tcl_ref_pattern, guess, re.M))
+ elif get_module_syntax() == 'Lua':
+ lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$'
+ self.assertFalse(re.search(lua_ref_pattern, guess, re.M))
+ else:
+ for env_var in sp_headers:
+ if get_module_syntax() == 'Tcl':
+ tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$"
+ self.assertTrue(re.search(tcl_ref_pattern, guess, re.M))
+ elif get_module_syntax() == 'Lua':
+ lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$'
+ self.assertTrue(re.search(lua_ref_pattern, guess, re.M))
+
+ # test wrong easyconfig parameter
+ self.contents += '\nmodule_search_path_headers = "WRONG_OPT"'
+ self.writeEC()
+ ec = EasyConfig(self.eb_file)
+
+ error_pattern = "Unknown value selected for option module-search-path-headers"
+ with eb.module_generator.start_module_creation():
+ self.assertErrorRegex(EasyBuildError, error_pattern, EasyBlock, ec)
+
# cleanup
eb.close_log()
os.remove(eb.logfile)
@@ -641,49 +886,6 @@ def test_make_module_extra(self):
self.assertTrue(expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
- installver = '3.14-gompi-2018a'
-
- # also check how absolute paths specified in modexself.contents = '\n'.join([
- self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}"
- self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}"
- self.writeEC()
- ec = EasyConfig(self.eb_file)
- eb = EasyBlock(ec)
- eb.installdir = os.path.join(config.install_path(), 'pi', installver)
- eb.check_readiness_step()
-
- # absolute paths are not allowed by default
- error_pattern = "Absolute path .* passed to update_paths which only expects relative paths"
- self.assertErrorRegex(EasyBuildError, error_pattern, eb.make_module_step)
-
- # allow use of absolute paths, and verify contents of module
- self.contents += "\nallow_prepend_abs_path = True"
- self.contents += "\nallow_append_abs_path = True"
- self.writeEC()
- ec = EasyConfig(self.eb_file)
- eb = EasyBlock(ec)
- eb.installdir = os.path.join(config.install_path(), 'pi', installver)
- eb.check_readiness_step()
-
- modrootpath = eb.make_module_step()
-
- modpath = os.path.join(modrootpath, 'pi', installver)
- if get_module_syntax() == 'Lua':
- modpath += '.lua'
-
- self.assertExists(modpath)
- txt = read_file(modpath)
- patterns = [
- r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo",
- r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path",
- r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar",
- r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*foo",
- r"^append[-_]path.*TEST_PATH_VAR_APPEND.*/test/absolute/path",
- r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*bar",
- ]
- for pattern in patterns:
- self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt))
-
def test_make_module_deppaths(self):
"""Test for make_module_deppaths"""
init_config(build_options={'silent': True})
@@ -704,9 +906,10 @@ def test_make_module_deppaths(self):
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
- eb.check_readiness_step()
- eb.make_builddir()
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.check_readiness_step()
+ eb.make_builddir()
+ eb.prepare_step()
if get_module_syntax() == 'Tcl':
use_load = '\n'.join([
@@ -724,7 +927,8 @@ def test_make_module_deppaths(self):
self.fail("Unknown module syntax: %s" % get_module_syntax())
expected = use_load
- self.assertEqual(eb.make_module_deppaths().strip(), expected)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(eb.make_module_deppaths().strip(), expected)
def test_make_module_dep(self):
"""Test for make_module_dep"""
@@ -746,26 +950,32 @@ def test_make_module_dep(self):
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
- eb.check_readiness_step()
- eb.make_builddir()
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.check_readiness_step()
+ eb.make_builddir()
+ eb.prepare_step()
if get_module_syntax() == 'Tcl':
- tc_load = '\n'.join([
- "if { ![ is-loaded gompi/2018a ] } {",
- " module load gompi/2018a",
- "}",
- ])
- fftw_load = '\n'.join([
- "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
- " module load FFTW/3.3.7-gompi-2018a",
- "}",
- ])
- lapack_load = '\n'.join([
- "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
- " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
- "}",
- ])
+ if self.modtool.supports_safe_auto_load:
+ tc_load = "module load gompi/2018a"
+ fftw_load = "module load FFTW/3.3.7-gompi-2018a"
+ lapack_load = "module load OpenBLAS/0.2.20-GCC-6.4.0-2.28"
+ else:
+ tc_load = '\n'.join([
+ "if { ![ is-loaded gompi/2018a ] } {",
+ " module load gompi/2018a",
+ "}",
+ ])
+ fftw_load = '\n'.join([
+ "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+ " module load FFTW/3.3.7-gompi-2018a",
+ "}",
+ ])
+ lapack_load = '\n'.join([
+ "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
+ " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
+ "}",
+ ])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not ( isloaded("gompi/2018a") ) then',
@@ -786,7 +996,8 @@ def test_make_module_dep(self):
self.fail("Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
- self.assertEqual(eb.make_module_dep().strip(), expected)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(eb.make_module_dep().strip(), expected)
# provide swap info for FFTW to trigger an extra 'unload FFTW'
unload_info = {
@@ -794,12 +1005,18 @@ def test_make_module_dep(self):
}
if get_module_syntax() == 'Tcl':
- fftw_load = '\n'.join([
- "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
- " module unload FFTW",
- " module load FFTW/3.3.7-gompi-2018a",
- "}",
- ])
+ if self.modtool.supports_safe_auto_load:
+ fftw_load = '\n'.join([
+ "module unload FFTW",
+ "module load FFTW/3.3.7-gompi-2018a",
+ ])
+ else:
+ fftw_load = '\n'.join([
+ "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+ " module unload FFTW",
+ " module load FFTW/3.3.7-gompi-2018a",
+ "}",
+ ])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not ( isloaded("FFTW/3.3.7-gompi-2018a") ) then',
@@ -810,7 +1027,8 @@ def test_make_module_dep(self):
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
- self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
def test_make_module_dep_hmns(self):
"""Test for make_module_dep under HMNS"""
@@ -842,17 +1060,19 @@ def test_make_module_dep_hmns(self):
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
- eb.check_readiness_step()
- eb.make_builddir()
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.check_readiness_step()
+ eb.make_builddir()
+ eb.prepare_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
- mod_dep_txt = eb.make_module_dep()
+ with self.mocked_stdout_stderr():
+ mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/6.4.0-2.28', 'OpenMPI/2.1.2']:
- regex = re.compile('load.*%s' % mod)
+ regex = re.compile('(load|depends[-_]on).*%s' % mod)
self.assertFalse(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
- regex = re.compile('load.*FFTW/3.3.7')
+ regex = re.compile('(load|depends[-_]on).*FFTW/3.3.7')
self.assertTrue(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def test_make_module_dep_of_dep_hmns(self):
@@ -994,6 +1214,60 @@ def test_handle_iterate_opts(self):
self.assertEqual(eb.cfg.iterating, False)
self.assertEqual(eb.cfg['configopts'], ["--opt1 --anotheropt", "--opt2", "--opt3 --optbis"])
+ def test_post_processing_step(self):
+ """Test post_processing_step and deprecated post_install_step."""
+ init_config(build_options={'silent': True})
+
+ test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+
+ # these imports only work here, since EB_toy is a test easyblock
+ from easybuild.easyblocks.toy import EB_toy
+ from easybuild.easyblocks.toy_deprecated import EB_toy_deprecated
+
+ cwd = os.getcwd()
+ toy_ec = EasyConfig(toy_ec_fn)
+ eb = EB_toy_deprecated(toy_ec)
+ eb.silent = True
+ depr_msg = r"EasyBlock.post_install_step\(\) is deprecated, use EasyBlock.post_processing_step\(\) instead"
+ expected_error = r"DEPRECATED \(since v6.0\).*" + depr_msg
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_error, eb.run_all_steps, True)
+
+ change_dir(cwd)
+ toy_ec = EasyConfig(toy_ec_fn)
+ eb = EB_toy(toy_ec)
+ eb.silent = True
+ with self.mocked_stdout_stderr() as (_, stderr):
+ eb.run_all_steps(True)
+ # no deprecation warning
+ stderr = stderr.getvalue()
+ self.assertFalse(stderr)
+
+ libtoy_post_a = os.path.join(eb.installdir, 'lib', 'libtoy_post.a')
+ self.assertExists(libtoy_post_a)
+
+ # check again with toy easyblock that still uses post_install_step,
+ # to verify that the expected file is being created when deprecated functionality is allow
+ remove_file(libtoy_post_a)
+ modify_env(os.environ, self.orig_environ, verbose=False)
+ change_dir(cwd)
+
+ self.allow_deprecated_behaviour()
+ toy_ec = EasyConfig(toy_ec_fn)
+ eb = EB_toy_deprecated(toy_ec)
+ eb.silent = True
+ with self.mocked_stdout_stderr() as (stdout, stderr):
+ eb.run_all_steps(True)
+
+ regex = re.compile(depr_msg, re.M)
+ stdout = stdout.getvalue()
+ self.assertTrue("This step is deprecated.\n" in stdout)
+ stderr = stderr.getvalue()
+ self.assertTrue(regex.search(stderr), f"Pattern {regex.pattern} found in: {stderr}")
+
+ self.assertExists(libtoy_post_a)
+
def test_extensions_step(self):
"""Test the extensions_step"""
init_config(build_options={'silent': True})
@@ -1031,6 +1305,64 @@ def test_extensions_step(self):
eb.close_log()
os.remove(eb.logfile)
+ def test_extensions_step_deprecations(self):
+ """Test extension install with deprecated substeps."""
+ install_substeps = ["pre_install_extension", "install_extension", "post_install_extension"]
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = '\n'.join([
+ 'easyblock = "ConfigureMake"',
+ 'name = "pi"',
+ 'version = "3.14"',
+ 'homepage = "http://example.com"',
+ 'description = "test easyconfig"',
+ 'toolchain = SYSTEM',
+ 'exts_defaultclass = "DummyExtension"',
+ 'exts_list = ["ext1"]',
+ 'exts_list = [',
+ ' "dummy_ext",',
+ ' ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),',
+ ' ("deprec_ext", "0.0", {"easyblock": "DeprecatedDummyExtension"}),',
+ ' ("childcustom_ext", "0.0", {"easyblock": "ChildCustomDummyExtension"}),',
+ ' ("childdeprec_ext", "0.0", {"easyblock": "ChildDeprecatedDummyExtension"}),',
+ ']',
+ ])
+ write_file(test_ec, test_ec_txt)
+ ec = process_easyconfig(test_ec)[0]
+ eb = get_easyblock_instance(ec)
+ eb.prepare_for_extensions()
+ eb.init_ext_instances()
+
+ # Default DummyExtension without deprecated or custom install substeps
+ ext = eb.ext_instances[0]
+ self.assertEqual(ext.__class__.__name__, "DummyExtension")
+ for substep in install_substeps:
+ self.assertEqual(ext.install_extension_substep(substep), None)
+ # CustomDummyExtension
+ ext = eb.ext_instances[1]
+ self.assertEqual(ext.__class__.__name__, "CustomDummyExtension")
+ for substep in install_substeps:
+ expected_return = f"Extension installed with custom {substep}()"
+ self.assertEqual(ext.install_extension_substep(substep), expected_return)
+ # DeprecatedDummyExtension
+ ext = eb.ext_instances[2]
+ self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension")
+ for substep in install_substeps:
+ expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*"
+ self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep)
+ # ChildCustomDummyExtension
+ ext = eb.ext_instances[3]
+ self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension")
+ for substep in install_substeps:
+ expected_return = f"Extension installed with custom {substep}()"
+ self.assertEqual(ext.install_extension_substep(substep), expected_return)
+ # ChildDeprecatedDummyExtension
+ ext = eb.ext_instances[4]
+ self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension")
+ for substep in install_substeps:
+ expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*"
+ self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep)
+
def test_init_extensions(self):
"""Test creating extension instances."""
@@ -1086,7 +1418,7 @@ def test_extension_source_tmpl(self):
eb = EasyBlock(EasyConfig(self.eb_file))
error_pattern = r"source_tmpl value must be a string! "
- error_pattern += r"\(found value of type 'list'\): \['bar-0\.0\.tar\.gz'\]"
+ error_pattern += r"\(found value of type 'list'\): \['%\(name\)s-%\(version\)s.tar.gz'\]"
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
self.contents = self.contents.replace("'source_tmpl': [SOURCE_TAR_GZ]", "'source_tmpl': SOURCE_TAR_GZ")
@@ -1175,10 +1507,11 @@ def test_make_module_step(self):
modextravars['TEST_PUSHENV'] = {'value': '123', 'pushenv': True}
modextrapaths = {
- 'PATH': ('xbin', 'pibin'),
+ 'PATH': ('', 'xbin', 'pibin'),
'CPATH': 'pi/include',
+ 'TCLLIBPATH': {'paths': 'pi', 'delimiter': ' '},
+ 'APPEND_PATH': {'paths': 'pi', 'prepend': False},
}
- modextrapaths_append = {'APPEND_PATH': 'append_path'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
@@ -1192,7 +1525,6 @@ def test_make_module_step(self):
"hiddendependencies = [('test', '1.2.3'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
- "modextrapaths_append = %s" % str(modextrapaths_append),
])
# test if module is generated correctly
@@ -1200,15 +1532,20 @@ def test_make_module_step(self):
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
- eb.check_readiness_step()
- eb.make_builddir()
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.check_readiness_step()
+ eb.make_builddir()
+ eb.prepare_step()
- # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored
- os.makedirs(os.path.join(eb.installdir, 'bin'))
- write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello')
+ # populate install dir
+ for bin_dir in ('bin', 'xbin', 'pibin'):
+ mkdir(os.path.join(eb.installdir, bin_dir), parents=True)
+ write_file(os.path.join(eb.installdir, bin_dir, 'dummy.exe'), 'hello')
+ mkdir(os.path.join(eb.installdir, 'pi', 'include'), parents=True)
+ write_file(os.path.join(eb.installdir, 'pi', 'include', 'dummy.h'), 'hello')
- modpath = os.path.join(eb.make_module_step(), name, version)
+ with self.mocked_stdout_stderr():
+ modpath = os.path.join(eb.make_module_step(), name, version)
if get_module_syntax() == 'Lua':
modpath += '.lua'
self.assertExists(modpath)
@@ -1249,55 +1586,66 @@ def test_make_module_step(self):
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, vals) in modextrapaths.items():
- if isinstance(vals, string_type):
- vals = [vals]
- for val in vals:
+ placement = 'prepend'
+ delim_tcl = ''
+ delim_lua = ''
+ if isinstance(vals, dict):
+ paths = vals['paths']
+ if isinstance(paths, str):
+ paths = [paths]
+ if 'delimiter' in vals:
+ delim_tcl = fr'+-d\s+"{vals["delimiter"]}"\s'
+ delim_lua = fr', "{vals["delimiter"]}"'
+ if 'prepend' in vals:
+ placement = 'prepend' if vals['prepend'] else 'append'
+ elif isinstance(vals, str):
+ paths = [vals]
+ else:
+ paths = vals
+
+ for val in paths:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
+ if val == '':
+ full_val = r'\$root'
+ else:
+ full_val = fr'\$root/{val}'
+ regex = re.compile(fr'^{placement}-path\s{delim_tcl}+{key}\s+{full_val}$', re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
+ if val == '':
+ full_val = 'root'
+ else:
+ full_val = fr'pathJoin\(root, "{val}"\)'
+ regex = re.compile(fr'^{placement}_path\("{key}", {full_val}{delim_lua}\)$', re.M)
else:
- self.fail("Unknown module syntax: %s" % get_module_syntax())
- self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+ self.fail(f"Unknown module syntax: {get_module_syntax()}")
+ self.assertTrue(regex.search(txt), f"Pattern {regex.pattern} found in {txt}")
# Check for duplicates
num_prepends = len(regex.findall(txt))
- self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
-
- for (key, vals) in modextrapaths_append.items():
- if isinstance(vals, string_type):
- vals = [vals]
- for val in vals:
- if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M)
- elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
- else:
- self.fail("Unknown module syntax: %s" % get_module_syntax())
- self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+ self.assertEqual(num_prepends, 1, f"Expected exactly 1 {regex.pattern} command in {txt}")
for (name, ver) in [('GCC', '6.4.0-2.28')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s\s*$' % os.path.join(name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s"\)$' % os.path.join(name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('test', '1.2.3')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s/.%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s/.%s"\)$' % (name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s/.?%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s/.?%s"\)$' % (name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
@@ -1324,7 +1672,8 @@ def test_make_module_step(self):
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
- self.assertErrorRegex(EasyBuildError, error_pattern, eb.make_module_step)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, eb.make_module_step)
def test_gen_dirs(self):
"""Test methods that generate/set build/install directory names."""
@@ -1390,13 +1739,15 @@ def test_make_builddir(self):
eb.gen_builddir()
# by default, make_builddir will re-create the build directory (i.e. remove existing & re-create)
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
builddir = eb.builddir
testfile = os.path.join(builddir, 'test123', 'foobar.txt')
write_file(testfile, 'test123')
self.assertExists(testfile)
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
self.assertEqual(builddir, eb.builddir)
# file is gone because directory was removed and re-created
self.assertNotExists(testfile)
@@ -1406,7 +1757,8 @@ def test_make_builddir(self):
# make sure that build directory does *not* get re-created when we're building in installation directory
# and we're iterating over a list of (pre)config/build/installopts
eb.build_in_installdir = True
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
# also need to create install directory since build dir == install dir
eb.make_installdir()
builddir = eb.builddir
@@ -1418,7 +1770,8 @@ def test_make_builddir(self):
# with iteration count > 0, build directory is not re-created because of build-in-installdir
eb.iter_idx = 1
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
eb.make_installdir()
self.assertEqual(builddir, eb.builddir)
self.assertExists(testfile)
@@ -1427,7 +1780,8 @@ def test_make_builddir(self):
# resetting iteration index to 0 results in re-creating build directory
eb.iter_idx = 0
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
eb.make_installdir()
self.assertEqual(builddir, eb.builddir)
self.assertNotExists(testfile)
@@ -1457,14 +1811,14 @@ def test_fetch_sources(self):
toy_source = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz')
- eb.fetch_sources()
+ with self.mocked_stdout_stderr():
+ eb.fetch_sources()
self.assertEqual(len(eb.src), 1)
self.assertTrue(os.path.samefile(eb.src[0]['path'], toy_source))
self.assertEqual(eb.src[0]['name'], 'toy-0.0.tar.gz')
self.assertEqual(eb.src[0]['cmd'], None)
- self.assertEqual(len(eb.src[0]['checksum']), 7)
- self.assertEqual(eb.src[0]['checksum'][0], 'be662daa971a640e40be5c804d9d7d10')
- self.assertEqual(eb.src[0]['checksum'][1], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
+ self.assertEqual(len(eb.src[0]['checksum']), 2)
+ self.assertEqual(eb.src[0]['checksum'][0], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
# reconfigure EasyBuild so we can check 'downloaded' sources
os.environ['EASYBUILD_SOURCEPATH'] = self.test_prefix
@@ -1494,7 +1848,8 @@ def test_fetch_sources(self):
'extract_cmd': "tar xfz %s",
},
]
- eb.fetch_sources(sources, checksums=[])
+ with self.mocked_stdout_stderr():
+ eb.fetch_sources(sources, checksums=[])
toy_source_dir = os.path.join(self.test_prefix, 't', 'toy')
expected_sources = ['toy-0.0-extra.txt', 'toy-0.0_gzip.patch.gz', 'toy-0.0-renamed.tar.gz']
@@ -1527,15 +1882,54 @@ def test_fetch_sources(self):
error_pattern = "Found one or more unexpected keys in 'sources' specification: {'nosuchkey': 'foobar'}"
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_sources, sources, checksums=[])
+ @requires_github_access()
+ def test_fetch_sources_git(self):
+ """Test fetch_sources method from git repo."""
+
+ testdir = os.path.abspath(os.path.dirname(__file__))
+ ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0]
+ eb = get_easyblock_instance(ec)
+ eb.src = []
+ sources = [
+ {
+ 'filename': 'testrepository.tar.xz',
+ 'git_config': {
+ 'repo_name': 'testrepository',
+ 'url': 'https://github.com/easybuilders',
+ 'tag': 'branch_tag_for_test',
+ }
+ }
+ ]
+ checksums = ["00000000"]
+
+ if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ eb.fetch_sources(sources, checksums=checksums)
+
+ if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+ self.disallow_deprecated_behaviour()
+
+ self.assertEqual(len(eb.src), 1)
+ self.assertEqual(eb.src[0]['name'], "testrepository.tar.xz")
+ self.assertExists(eb.src[0]['path'])
+ self.assertEqual(eb.src[0]['cmd'], None)
+
+ reference_checksum = "00000000"
+ if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+ # checksums of tarballs made by EB cannot be reliably checked prior to Python 3.9
+ # due to changes introduced in python/cpython#90021
+ reference_checksum = None
+
+ self.assertEqual(eb.src[0]['checksum'], reference_checksum)
+
+ # cleanup
+ remove_file(eb.src[0]['path'])
+
def test_download_instructions(self):
"""Test use of download_instructions easyconfig parameter."""
- # skip test when using Python 2, since it somehow fails then,
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333
- if sys.version_info[0] == 2:
- print("Skipping test_download_instructions because Python 2.x is being used")
- return
-
orig_test_ec = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'software_with_missing_sources'",
@@ -1556,7 +1950,8 @@ def test_download_instructions(self):
common_error_pattern = "^Couldn't find file software_with_missing_sources-0.0.tar.gz anywhere"
error_pattern = common_error_pattern + ", and downloading it didn't work either"
- self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
download_instructions = "download_instructions = 'Manual download from example.com required'"
sources = "sources = [SOURCE_TAR_GZ]"
@@ -1569,11 +1964,9 @@ def test_download_instructions(self):
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
- self.mock_stdout(False)
- self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required")
- self.assertEqual(stdout, '')
+ self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr)
+ self.assertIn("Make the files available in the active source path", stderr)
# create dummy source file
write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '')
@@ -1585,11 +1978,10 @@ def test_download_instructions(self):
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required")
- self.assertEqual(stdout, '')
+ self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr)
+ self.assertIn("Make the files available in the active source path", stderr)
# wipe top-level download instructions, try again
self.contents = self.contents.replace(download_instructions, '')
@@ -1601,10 +1993,8 @@ def test_download_instructions(self):
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertEqual(stdout, '')
# inject download instructions for extension
download_instructions = ' ' * 8 + "'download_instructions': "
@@ -1618,11 +2008,10 @@ def test_download_instructions(self):
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com")
- self.assertEqual(stdout, '')
+ self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr)
+ self.assertIn("Make the files available in the active source path", stderr)
# download instructions should also be printed if 'source_tmpl' is used to specify extension sources
self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,")
@@ -1633,11 +2022,10 @@ def test_download_instructions(self):
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com")
- self.assertEqual(stdout, '')
+ self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr)
+ self.assertIn("Make the files available in the active source path", stderr)
# create dummy source file for extension
write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '')
@@ -1647,11 +2035,9 @@ def test_download_instructions(self):
self.mock_stdout(True)
eb.fetch_step()
stderr = self.get_stderr().strip()
- stdout = self.get_stdout().strip()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertEqual(stderr, '')
- self.assertEqual(stdout, '')
def test_fetch_patches(self):
"""Test fetch_patches method."""
@@ -1720,41 +2106,50 @@ def test_obtain_file(self):
# 'downloading' a file to (first) sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)])
shutil.copy2(toy_tarball_path, tmpdir_subdir)
- res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
# test no_download option
urls = ['file://%s' % tmpdir_subdir]
- error_pattern = "Couldn't find file toy-0.0.tar.gz anywhere, and downloading it is disabled"
- self.assertErrorRegex(EasyBuildError, error_pattern, eb.obtain_file,
- toy_tarball, urls=urls, alt_location='alt_toy', no_download=True)
+ error_pattern = "Couldn't find file 'toy-0.0.tar.gz' anywhere, and downloading it is disabled"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, eb.obtain_file,
+ toy_tarball, urls=urls, alt_location='alt_toy', no_download=True)
# 'downloading' a file to (first) alternative sourcepath works
- res = eb.obtain_file(toy_tarball, urls=urls, alt_location='alt_toy')
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=urls, alt_location='alt_toy')
self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball))
# make sure that directory in which easyconfig file is located is *ignored* when alt_location is used
dummy_toy_tar_gz = os.path.join(os.path.dirname(test_ec), 'toy-0.0.tar.gz')
write_file(dummy_toy_tar_gz, '')
- res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, dummy_toy_tar_gz)
- res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball))
remove_file(dummy_toy_tar_gz)
# finding a file in sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)])
- res = eb.obtain_file(toy_tarball)
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
- # finding a file in the alternate location works
- res = eb.obtain_file(toy_tarball, alt_location='alt_toy')
+ # finding a file in the alternative location works
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, alt_location='alt_toy')
self.assertEqual(res, alt_toy_tarball_path)
# sourcepath has preference over downloading
- res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, toy_tarball_path)
- res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
self.assertEqual(res, alt_toy_tarball_path)
init_config(args=["--sourcepath=%s:%s" % (tmpdir, sandbox_sources)])
@@ -1769,10 +2164,8 @@ def test_obtain_file(self):
self.mock_stdout(True)
res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], force_download=True)
stderr = self.get_stderr()
- stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
- self.assertEqual(stdout, '')
msg = "WARNING: Found file toy-0.0.tar.gz at %s, but re-downloading it anyway..." % toy_tarball_path
self.assertEqual(stderr.strip(), msg)
@@ -1783,13 +2176,15 @@ def test_obtain_file(self):
# obtain_file yields error for non-existing files
fn = 'thisisclearlyanonexistingfile'
error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn
- self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir])
# also test triggering error when downloading from a URL that includes URL-encoded characters
# cfr. https://github.com/easybuilders/easybuild-framework/pull/4005
url = 'file://%s' % os.path.dirname(tmpdir_subdir)
url += '%2F' + os.path.basename(tmpdir_subdir)
- self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=[url])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=[url])
# file specifications via URL also work, are downloaded to (first) sourcepath
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)])
@@ -1801,7 +2196,8 @@ def test_obtain_file(self):
fn = os.path.basename(file_url)
res = None
try:
- res = eb.obtain_file(file_url)
+ with self.mocked_stdout_stderr():
+ res = eb.obtain_file(file_url)
except EasyBuildError as err:
# if this fails, it should be because there's no online access
download_fail_regex = re.compile('socket error')
@@ -1843,7 +2239,8 @@ def test_fallback_source_url(self):
ec = process_easyconfig(udunits_ec)[0]
eb = EasyBlock(ec['ec'])
- eb.fetch_step()
+ with self.mocked_stdout_stderr():
+ eb.fetch_step()
expected_path = os.path.join(self.test_prefix, 'u', 'UDUNITS', 'udunits-2.2.26.tar.gz')
self.assertTrue(os.path.samefile(eb.src[0]['path'], expected_path))
@@ -1917,7 +2314,8 @@ def test_obtain_file_extension(self):
toy_ec = process_easyconfig(toy_ec_file)[0]
toy_eb = EasyBlock(toy_ec['ec'])
- toy_eb.fetch_step()
+ with self.mocked_stdout_stderr():
+ toy_eb.fetch_step()
test_ext = toy_eb.exts[-1]
test_ext_src_fn = os.path.basename(test_ext['src'])
@@ -1990,8 +2388,9 @@ def test_exclude_path_to_top_of_module_tree(self):
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
- eb.toolchain.prepare()
- modpath = eb.make_module_step()
+ with self.mocked_stdout_stderr():
+ eb.toolchain.prepare()
+ modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
@@ -2032,16 +2431,18 @@ def test_patch_step(self):
# test applying patches without sources
ec['ec']['sources'] = []
eb = EasyBlock(ec['ec'])
- eb.fetch_step()
- eb.extract_step()
- self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
+ with self.mocked_stdout_stderr():
+ eb.fetch_step()
+ eb.extract_step()
+ self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
# test actual patching of unpacked sources
ec['ec']['sources'] = orig_sources
eb = EasyBlock(ec['ec'])
- eb.fetch_step()
- eb.extract_step()
- eb.patch_step()
+ with self.mocked_stdout_stderr():
+ eb.fetch_step()
+ eb.extract_step()
+ eb.patch_step()
# verify that patches were applied
toydir = os.path.join(eb.builddir, 'toy-0.0')
self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source'])
@@ -2051,9 +2452,10 @@ def test_patch_step(self):
# check again with backup of patched files enabled
update_build_option('backup_patched_files', True)
eb = EasyBlock(ec['ec'])
- eb.fetch_step()
- eb.extract_step()
- eb.patch_step()
+ with self.mocked_stdout_stderr():
+ eb.fetch_step()
+ eb.extract_step()
+ eb.patch_step()
# verify that patches were applied
toydir = os.path.join(eb.builddir, 'toy-0.0')
self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig'])
@@ -2086,8 +2488,9 @@ def test_extensions_sanity_check(self):
eb.silent = True
error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n"
error_pattern += r"failing sanity check for 'toy' extension: "
- error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found'
- self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True)
+ error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found'
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True)
# purposely put sanity check command in place that breaks the build,
# to check whether sanity check is only run once;
@@ -2096,7 +2499,8 @@ def test_extensions_sanity_check(self):
toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')])
eb = EB_toy(toy_ec)
eb.silent = True
- eb.run_all_steps(True)
+ with self.mocked_stdout_stderr():
+ eb.run_all_steps(True)
def test_parallel(self):
"""Test defining of parallelism."""
@@ -2106,56 +2510,204 @@ def test_parallel(self):
handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
- write_file(toy_ec1, toytxt + "\nparallel = 123")
+ write_file(toy_ec1, toytxt + "\nparallel = 13")
handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
- write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67")
+ write_file(toy_ec2, toytxt + "\nparallel = 12\nmaxparallel = 6")
handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec3, toytxt + "\nparallel = False")
+ handle, toy_ec4 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
+ os.close(handle)
+ write_file(toy_ec4, toytxt + "\nmaxparallel = 6")
+
+ handle, toy_ec5 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
+ os.close(handle)
+ write_file(toy_ec5, toytxt + "\nmaxparallel = False")
+
# default: parallelism is derived from # available cores + ulimit
- test_eb = EasyBlock(EasyConfig(toy_ec))
- test_eb.check_readiness_step()
- self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0)
-
- # only 'parallel' easyconfig parameter specified (no 'parallel' build option)
- test_eb = EasyBlock(EasyConfig(toy_ec1))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 123)
-
- # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
- test_eb = EasyBlock(EasyConfig(toy_ec2))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 67)
-
- # make sure 'parallel = False' is not overriden (no 'parallel' build option)
- test_eb = EasyBlock(EasyConfig(toy_ec3))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], False)
-
- # only 'parallel' build option specified
- init_config(build_options={'parallel': '97', 'validate': False})
- test_eb = EasyBlock(EasyConfig(toy_ec))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 97)
-
- # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
- test_eb = EasyBlock(EasyConfig(toy_ec1))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 97)
-
- # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
- test_eb = EasyBlock(EasyConfig(toy_ec2))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 67)
-
- # make sure 'parallel = False' is not overriden (with 'parallel' build option)
- test_eb = EasyBlock(EasyConfig(toy_ec3))
- test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 0)
+ # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here
+ auto_parallel = 16 - 4 # Using + 3 below which must still be less
+ st.det_parallelism._default_parallelism = auto_parallel
+
+ # 'parallel' build option NOT specified
+ test_cases = {
+ '': auto_parallel,
+ 'parallel = False': 1,
+ 'parallel = 1': 1,
+ 'parallel = 6': 6,
+ f'parallel = {auto_parallel + 3}': auto_parallel + 3, # Setting parallel disables auto-detection
+ 'maxparallel = False': 1,
+ 'maxparallel = 1': 1,
+ 'maxparallel = 6': 6,
+ f'maxparallel = {auto_parallel + 3}': auto_parallel,
+ 'parallel = 8\nmaxparallel = 6': 6,
+ 'parallel = 8\nmaxparallel = 9': 8,
+ 'parallel = False\nmaxparallel = 6': 1,
+ 'parallel = 8\nmaxparallel = False': 1,
+ }
+
+ for txt, expected in test_cases.items():
+ with self.subTest(ec_params=txt):
+ self.contents = toytxt + '\n' + txt
+ self.writeEC()
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg.parallel, expected)
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ self.assertEqual(test_eb.cfg['parallel'], expected)
+
+ # 'parallel' build option specified
+ buildopt_parallel = 11
+ # When build option is given the auto-parallelism is ignored. Verify by setting it very low
+ st.det_parallelism._default_parallelism = 2
+ init_config(build_options={
+ 'parallel': str(buildopt_parallel),
+ 'validate': False,
+ })
+
+ test_cases = {
+ '': buildopt_parallel,
+ 'parallel = False': 1,
+ 'parallel = 1': 1,
+ 'parallel = 6': 6,
+ f'parallel = {buildopt_parallel + 2}': buildopt_parallel,
+ 'maxparallel = False': 1,
+ 'maxparallel = 1': 1,
+ 'maxparallel = 6': 6,
+ f'maxparallel = {buildopt_parallel + 2}': buildopt_parallel,
+ 'parallel = 8\nmaxparallel = 6': 6,
+ 'parallel = 8\nmaxparallel = 9': 8,
+ 'parallel = False\nmaxparallel = 6': 1,
+ 'parallel = 8\nmaxparallel = False': 1,
+ }
+
+ for txt, expected in test_cases.items():
+ with self.subTest(ec_params=txt):
+ self.contents = toytxt + '\n' + txt
+ self.writeEC()
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg.parallel, expected)
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ self.assertEqual(test_eb.cfg['parallel'], expected)
+
+ # re-check when --max-parallel is used instead
+ buildopt_max_parallel = 8
+ st.det_parallelism._default_parallelism = 16
+ init_config(build_options={
+ 'max_parallel': buildopt_max_parallel,
+ 'validate': False,
+ })
+
+ test_cases = {
+ '': buildopt_max_parallel,
+ 'parallel = False': 1,
+ 'parallel = 1': 1,
+ 'parallel = 6': 6,
+ # --max-parallel value limits max. parallelism, so only 8 cores will be used when 'parallel = 10' is used
+ f'parallel = {buildopt_max_parallel + 2}': buildopt_max_parallel,
+ 'maxparallel = False': 1,
+ 'maxparallel = 1': 1,
+ 'maxparallel = 6': 6,
+ # minimum of 'maxparallel' easyconfig parameter and --max-parallel configuration option is used
+ f'maxparallel = {buildopt_max_parallel + 2}': buildopt_max_parallel,
+ 'parallel = 8\nmaxparallel = 6': 6,
+ 'parallel = 8\nmaxparallel = 9': 8,
+ 'parallel = False\nmaxparallel = 6': 1,
+ 'parallel = 8\nmaxparallel = False': 1,
+ }
+
+ for txt, expected in test_cases.items():
+ with self.subTest(ec_params=txt):
+ self.contents = toytxt + '\n' + txt
+ self.writeEC()
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg.parallel, expected)
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ self.assertEqual(test_eb.cfg['parallel'], expected)
+
+ # re-check when both --max-parallel and --parallel are used (--max-parallel wins)
+ init_config(build_options={
+ 'max_parallel': buildopt_max_parallel,
+ 'parallel': buildopt_parallel,
+ 'validate': False,
+ })
+
+ for txt, expected in test_cases.items():
+ with self.subTest(ec_params=txt):
+ self.contents = toytxt + '\n' + txt
+ self.writeEC()
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg.parallel, expected)
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ self.assertEqual(test_eb.cfg['parallel'], expected)
+
+ # Template updated correctly
+ self.contents = toytxt + '\nmaxparallel=2'
+ self.writeEC()
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+
+ test_eb.cfg['buildopts'] = '-j %(parallel)s'
+ self.assertEqual(test_eb.cfg['buildopts'], '-j 2')
+ # Might be done in an easyblock step
+ test_eb.cfg.parallel = 42
+ self.assertEqual(test_eb.cfg['buildopts'], '-j 42')
+ # Unaffected by build settings
+ test_eb.cfg.parallel = 421337
+ self.assertEqual(test_eb.cfg['buildopts'], '-j 421337')
+ # False is equal to 1
+ test_eb.cfg.parallel = False
+ self.assertEqual(test_eb.cfg['buildopts'], '-j 1')
+
+ # Legacy behavior. To be removed after deprecation of the parallel EC parameter
+ self.contents = toytxt + '\nmaxparallel=99'
+ self.writeEC()
+ with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ parallel = buildopt_max_parallel - 2
+ test_eb.cfg['parallel'] = parallel # Old Easyblocks might change that before the ready step
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg.parallel, parallel)
+ self.assertEqual(test_eb.cfg['parallel'], parallel)
+ # Afterwards it also gets reflected directly ignoring maxparallel
+ parallel = buildopt_max_parallel * 3
+ test_eb.cfg['parallel'] = parallel
+ self.assertEqual(test_eb.cfg.parallel, parallel)
+ self.assertEqual(test_eb.cfg['parallel'], parallel)
+
+ # Reset mocked value
+ del st.det_parallelism._default_parallelism
+
+ def test_keepsymlinks(self):
+ """Test keepsymlinks parameter (default: True)."""
+ topdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ toytxt = read_file(toy_ec)
+
+ test_cases = {
+ '': True,
+ 'keepsymlinks = False': False,
+ 'keepsymlinks = True': True,
+ }
+
+ for txt, expected in test_cases.items():
+ with self.subTest(ec_params=txt):
+ self.contents = toytxt + '\n' + txt
+ self.writeEC()
+ test_eb = EasyBlock(EasyConfig(self.eb_file))
+ test_eb.post_init()
+ self.assertEqual(test_eb.cfg['keepsymlinks'], expected)
def test_guess_start_dir(self):
"""Test guessing the start dir."""
@@ -2172,7 +2724,8 @@ def check_start_dir(expected_start_dir):
eb = EasyBlock(ec['ec'])
eb.silent = True
eb.cfg['stop'] = 'patch'
- eb.run_all_steps(False)
+ with self.mocked_stdout_stderr():
+ eb.run_all_steps(False)
eb.guess_start_dir()
abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir)
self.assertTrue(os.path.samefile(eb.cfg['start_dir'], abs_expected_start_dir))
@@ -2213,7 +2766,7 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=Non
eb.extensions_step(fetch=True, install=False)
# extract sources of the extension
ext = eb.ext_instances[-1]
- ext.run(unpack_src=unpack_src)
+ ext.install_extension(unpack_src=unpack_src)
if expected_start_dir is None:
self.assertIsNone(ext.start_dir)
@@ -2342,14 +2895,16 @@ def test_prepare_step_load_tc_deps_modules(self):
os.environ['EBROOTHWLOC'] = self.test_prefix
# loaded of modules for toolchain + dependencies can be disabled via load_tc_deps_modules=False
- eb.prepare_step(load_tc_deps_modules=False)
+ with self.mocked_stdout_stderr():
+ eb.prepare_step(load_tc_deps_modules=False)
self.assertEqual(self.modtool.list(), [])
del os.environ['EBROOTGCC']
del os.environ['EBROOTHWLOC']
# modules for toolchain + dependencies are still loaded by default
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.prepare_step()
loaded_modules = self.modtool.list()
self.assertEqual(len(loaded_modules), 2)
self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28')
@@ -2388,7 +2943,8 @@ def test_prepare_step_hmns(self):
eb = EasyBlock(test_ec['ec'])
mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'system-system'), parents=True)
- eb.prepare_step()
+ with self.mocked_stdout_stderr():
+ eb.prepare_step()
loaded_modules = self.modtool.list()
self.assertEqual(len(loaded_modules), 1)
@@ -2404,7 +2960,8 @@ def test_prepare_step_cuda_cache(self):
ec = process_easyconfig(toy_ec)[0]
eb = EasyBlock(ec['ec'])
eb.silent = True
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
eb.prepare_step(start_dir=False)
logtxt = read_file(eb.logfile)
@@ -2419,10 +2976,12 @@ def test_prepare_step_cuda_cache(self):
ec = process_easyconfig(test_ec)[0]
eb = EasyBlock(ec['ec'])
eb.silent = True
- eb.make_builddir()
+ with self.mocked_stdout_stderr():
+ eb.make_builddir()
write_file(eb.logfile, '')
- eb.prepare_step(start_dir=False)
+ with self.mocked_stdout_stderr():
+ eb.prepare_step(start_dir=False)
logtxt = read_file(eb.logfile)
self.assertNotIn('Disabling CUDA PTX cache', logtxt)
self.assertIn('Enabling CUDA PTX cache', logtxt)
@@ -2430,7 +2989,8 @@ def test_prepare_step_cuda_cache(self):
init_config(build_options={'cuda_cache_maxsize': 0}) # Disable
write_file(eb.logfile, '')
- eb.prepare_step(start_dir=False)
+ with self.mocked_stdout_stderr():
+ eb.prepare_step(start_dir=False)
logtxt = read_file(eb.logfile)
self.assertIn('Disabling CUDA PTX cache', logtxt)
self.assertNotIn('Enabling CUDA PTX cache', logtxt)
@@ -2440,7 +3000,8 @@ def test_prepare_step_cuda_cache(self):
cuda_cache_dir = os.path.join(self.test_prefix, 'custom-cuda-cache')
init_config(build_options={'cuda_cache_maxsize': 1234, 'cuda_cache_dir': cuda_cache_dir})
write_file(eb.logfile, '')
- eb.prepare_step(start_dir=False)
+ with self.mocked_stdout_stderr():
+ eb.prepare_step(start_dir=False)
logtxt = read_file(eb.logfile)
self.assertNotIn('Disabling CUDA PTX cache', logtxt)
self.assertIn('Enabling CUDA PTX cache', logtxt)
@@ -2485,8 +3046,8 @@ def test_checksum_step(self):
copy_file(toy_ec, self.test_prefix)
toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
ectxt = read_file(toy_ec)
- # replace MD5 checksum for toy-0.0.tar.gz
- ectxt = ectxt.replace('be662daa971a640e40be5c804d9d7d10', '00112233445566778899aabbccddeeff')
+ # replace SHA256 checksum for toy-0.0.tar.gz
+ ectxt = ectxt.replace('44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', '76543210' * 8)
# replace SHA256 checksums for source of bar extension
ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8)
write_file(toy_ec, ectxt)
@@ -2501,13 +3062,6 @@ def test_checksum_step(self):
error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed"
self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info)
- # also check with deprecated fetch_extension_sources method
- self.allow_deprecated_behaviour()
- self.mock_stderr(True)
- self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources)
- self.mock_stderr(False)
- self.disallow_deprecated_behaviour()
-
# create test easyconfig from which checksums have been stripped
test_ec = os.path.join(self.test_prefix, 'test.eb')
ectxt = read_file(toy_ec)
@@ -2519,8 +3073,8 @@ def test_checksum_step(self):
# make sure that test easyconfig file indeed doesn't contain any checksums (either top-level or for extensions)
self.assertEqual(ec_json['ec']['checksums'], [])
- for ext in ec_json['ec']['exts_list']:
- if isinstance(ext, string_type):
+ for ext in ec_json['ec'].get_ref('exts_list'):
+ if isinstance(ext, str):
continue
elif isinstance(ext, tuple):
self.assertEqual(ext[2].get('checksums', []), [])
@@ -2555,17 +3109,10 @@ def test_checksum_step(self):
error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed"
self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step)
- # also check verification of checksums for extensions, which is part of fetch_extension_sources
+ # also check verification of checksums for extensions, which is part of collect_exts_file_info
error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed"
self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info)
- # also check with deprecated fetch_extension_sources method
- self.allow_deprecated_behaviour()
- self.mock_stderr(True)
- self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources)
- self.mock_stderr(False)
- self.disallow_deprecated_behaviour()
-
# if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error)
build_options = {
'ignore_checksums': True,
@@ -2590,20 +3137,8 @@ def test_checksum_step(self):
self.mock_stderr(False)
self.mock_stdout(False)
self.assertEqual(stdout, '')
- self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
-
- # also check with deprecated fetch_extension_sources method
- self.allow_deprecated_behaviour()
- self.mock_stderr(True)
- self.mock_stdout(True)
- eb.fetch_extension_sources()
- stderr = self.get_stderr()
- stdout = self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
- self.assertEqual(stdout, '')
- self.assertTrue(stderr.strip().endswith("WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz"))
- self.disallow_deprecated_behaviour()
+ self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz\n\n\n"
+ "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz")
def test_check_checksums(self):
"""Test for check_checksums_for and check_checksums methods."""
@@ -2659,10 +3194,10 @@ def run_checks():
# no checksum issues
self.assertEqual(eb.check_checksums(), [])
- # tuple of two alternate SHA256 checksums: OK
+ # tuple of two alternative SHA256 checksums: OK
eb.cfg['checksums'] = [
(
- # two alternate checksums for toy-0.0.tar.gz
+ # two alternative checksums for toy-0.0.tar.gz
'a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd',
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
),
@@ -2713,6 +3248,20 @@ def run_checks():
eb.json_checksums = None
self.assertEqual(eb.check_checksums(), [])
+ # more checks for check_checksums_for method, which also takes regular dict as input
+ self.assertEqual(eb.check_checksums_for({}), [])
+ expected = "Checksums missing for one or more sources/patches in test.eb: "
+ expected += "found 1 sources + 0 patches vs 0 checksums"
+ self.assertEqual(eb.check_checksums_for({'sources': ['test.tar.gz']}), [expected])
+
+ # example from QuantumESPRESSO easyconfig, template used in extract_cmd should not cause trouble
+ eb.cfg['sources'] = [{
+ 'filename': 'q-e-qe-%(version)s.tar.gz',
+ 'extract_cmd': 'mkdir -p %(builddir)s/qe-%(version)s && tar xzvf %s --strip-components=1 -C $_',
+ 'source_urls': ['https://gitlab.com/QEF/q-e/-/archive/qe-%(version)s'],
+ }]
+ res = eb.check_checksums_for(eb.cfg)
+
def test_this_is_easybuild(self):
"""Test 'this_is_easybuild' function (and get_git_revision function used by it)."""
# make sure both return a non-Unicode string
@@ -2771,7 +3320,8 @@ def test_stale_module_caches(self):
# installing both one.eb and two.eb in one go should work
# this verifies whether the "module show" cache is cleared in between builds,
# since one/1.0 is required for ec2, and the underlying one/1.0.2 is installed via ec1 in the same session
- self.eb_main([ec1, ec2], raise_error=True, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([ec1, ec2], raise_error=True, do_build=True, verbose=True)
def test_avail_easyblocks(self):
"""Test avail_easyblocks function."""
@@ -2895,6 +3445,175 @@ def run_sanity_check_step(sanity_check_paths, enhance_sanity_check):
run_sanity_check_step({}, False)
run_sanity_check_step({}, True)
+ def test_create_easyblock_without_logfile(self):
+ """
+ Test creating an EasyBlock without a logfile.
+ This represents scenarios found in Bundle and QuantumESPRESSO, where an EasyBlock is
+ created within another EasyBlock.
+ """
+ self.contents = '\n'.join([
+ 'easyblock = "ConfigureMake"',
+ 'name = "pi"',
+ 'version = "3.14"',
+ 'homepage = "http://example.com"',
+ 'description = "test easyconfig"',
+ 'toolchain = SYSTEM',
+ ])
+ self.writeEC()
+ # Ensure that the default case works as expected
+ eb = EasyBlock(EasyConfig(self.eb_file))
+ self.assertNotEqual(eb.log, None)
+ self.assertNotEqual(eb.logfile, None)
+ # Get reference to the actual log instance and ensure that it works
+ # This is NOT eb.log, which represents a separate logger with a separate name.
+ file_log = fancylogger.getLogger(name=None)
+ self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False)
+
+ # Now, create another EasyBlock by passing logfile from first EasyBlock.
+ eb_external_logfile = EasyBlock(EasyConfig(self.eb_file), logfile=eb.logfile)
+ self.assertNotEqual(eb_external_logfile.log, None)
+ self.assertTrue(eb_external_logfile.external_logfile)
+ self.assertEqual(eb_external_logfile.logfile, eb.logfile)
+ # Try to log something in it.
+ eb_external_logfile.log.info("Test message")
+
+ # Try to close EasyBlock with external logfile. This should not affect the logger.
+ eb_external_logfile.close_log()
+ self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False)
+ # Then close the log from creating EasyBlock. This should work as expected.
+ eb.close_log()
+ self.assertEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False)
+
+ os.remove(eb.logfile)
+
+ def test_expand_module_search_path(self):
+ """Testcase for expand_module_search_path"""
+ top_dir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(top_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ eb = EasyBlock(EasyConfig(toy_ec))
+ eb.installdir = config.install_path()
+ test_emsp = eb.expand_module_search_path # shortcut
+
+ # create test directories and files
+ os.makedirs(eb.installdir)
+ test_directories = (
+ 'empty_dir',
+ 'dir_empty_subdir',
+ ('dir_empty_subdir', 'empty_subdir'),
+ 'dir_with_file',
+ 'dir_full_subdirs',
+ ('dir_full_subdirs', 'subdir1'),
+ ('dir_full_subdirs', 'subdir2'),
+ )
+ for path in test_directories:
+ path_components = (path, ) if isinstance(path, str) else path
+ os.mkdir(os.path.join(eb.installdir, *path_components))
+
+ write_file(os.path.join(eb.installdir, 'dir_with_file', 'file.txt'), 'test file')
+ write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1')
+ write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2')
+ write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1')
+
+ self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), [])
+ self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+ self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH), ["empty_dir"])
+ self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+ self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH), ["dir_empty_subdir"])
+ self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+ self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH), ["dir_with_file"])
+ self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_FILES), ["dir_with_file"])
+ self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_TOP_FILES), ["dir_with_file"])
+ self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH), ["dir_full_subdirs"])
+ self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_FILES), ["dir_full_subdirs"])
+ self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+
+ # test globs
+ ref_expanded_paths = ["dir_empty_subdir/empty_subdir"]
+ self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH), ref_expanded_paths)
+ self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+ ref_expanded_paths = ["dir_full_subdirs/subdir1", "dir_full_subdirs/subdir2"]
+ self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH)), ref_expanded_paths)
+ self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_FILES)), ref_expanded_paths)
+ self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_TOP_FILES)), ref_expanded_paths)
+ ref_expanded_paths = ["dir_full_subdirs/subdir2/file21.txt"]
+ self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH), ref_expanded_paths)
+ self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_FILES), ref_expanded_paths)
+ self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_TOP_FILES), ref_expanded_paths)
+ self.assertEqual(test_emsp("nonexistent/*", True), [])
+ self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH), [])
+ self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+
+ # test just one lib directory
+ os.mkdir(os.path.join(eb.installdir, "lib"))
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), [])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+ write_file(os.path.join(eb.installdir, "lib", "libtest.so"), "not actually a lib")
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
+
+ # test both lib and lib64 directories
+ os.mkdir(os.path.join(eb.installdir, "lib64"))
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
+ write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib")
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"])
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"])
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"])
+
+ # test lib64 symlinked to lib
+ remove_dir(os.path.join(eb.installdir, "lib64"))
+ os.symlink("lib", os.path.join(eb.installdir, "lib64"))
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib", "lib"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib", "lib"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib", "lib"])
+
+ # test lib symlinked to lib64
+ remove_dir(os.path.join(eb.installdir, "lib"))
+ remove_file(os.path.join(eb.installdir, "lib64"))
+ os.mkdir(os.path.join(eb.installdir, "lib64"))
+ write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib")
+ os.symlink("lib64", os.path.join(eb.installdir, "lib"))
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib64"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib64"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64", "lib64"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64", "lib64"])
+ self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64", "lib64"])
+
+ # test both lib and lib64 symlinked to some other folder
+ remove_dir(os.path.join(eb.installdir, "lib64"))
+ remove_file(os.path.join(eb.installdir, "lib"))
+ os.mkdir(os.path.join(eb.installdir, "some_dir"))
+ write_file(os.path.join(eb.installdir, "some_dir", "libtest.so"), "not actually a lib")
+ os.symlink("some_dir", os.path.join(eb.installdir, "lib"))
+ os.symlink("some_dir", os.path.join(eb.installdir, "lib64"))
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["some_dir"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["some_dir"])
+ self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["some_dir"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["some_dir"])
+ self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"])
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["some_dir", "some_dir"])
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["some_dir", "some_dir"])
+ self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["some_dir", "some_dir"])
+
def suite():
""" return all the tests in this file """
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index c43c84be0e..21a6066557 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -51,7 +51,7 @@
from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER
from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig, create_paths, copy_easyconfigs
from easybuild.framework.easyconfig.easyconfig import det_subtoolchain_version, fix_deprecated_easyconfigs
-from easybuild.framework.easyconfig.easyconfig import is_generic_easyblock, get_easyblock_class, get_module_path
+from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path
from easybuild.framework.easyconfig.easyconfig import letter_dir_for, process_easyconfig, resolve_template
from easybuild.framework.easyconfig.easyconfig import triage_easyconfig_params, verify_easyconfig_filename
from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3
@@ -73,7 +73,6 @@
from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.options import parse_external_modules_metadata
-from easybuild.tools.py2vs3 import reload
from easybuild.tools.robot import det_robot_path, resolve_dependencies
from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64
from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version
@@ -86,11 +85,7 @@
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
-
+ pass
EXPECTED_DOTTXT_TOY_DEPS = """digraph graphname {
toy;
@@ -120,6 +115,11 @@ def setUp(self):
github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT)
self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None
+ self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS
+ self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES
+ self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS
+ self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES
+
def prep(self):
"""Prepare for test."""
# (re)cleanup last test file
@@ -133,10 +133,17 @@ def prep(self):
def tearDown(self):
""" make sure to remove the temporary file """
st.get_cpu_architecture = self.orig_get_cpu_architecture
+
super(EasyConfigTest, self).tearDown()
if os.path.exists(self.eb_file):
os.remove(self.eb_file)
+ # restore orignal values of DEPRECATED_EASYCONFIG_TEMPLATES & co in easyconfig.templates
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES
+
def test_empty(self):
""" empty files should not parse! """
self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "")
@@ -487,8 +494,11 @@ def test_exts_list(self):
"checksums": [
# SHA256 checksum for source (gzip-1.4.eb)
"6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",
- # SHA256 checksum for 'patch' (toy-0.0.eb)
- "2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c",
+ # SHA256 checksum for 'patch' (toy-0.0.eb);
+ # using dict value with key that has a template value,
+ # to make sure that works as expected...
+ {"toy-0.%(version_minor)s.eb":
+ "177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad"},
],
}),
# Can use templates in name and version
@@ -512,7 +522,8 @@ def test_exts_list(self):
self.assertEqual(exts_sources[1]['version'], '2.0')
self.assertEqual(exts_sources[1]['options'], {
'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45',
- '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'],
+ {'toy-0.%(version_minor)s.eb':
+ '177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad'}],
'patches': [('toy-0.0.eb', '.')],
'source_tmpl': 'gzip-1.4.eb',
'source_urls': [('http://example.com', 'suffix')],
@@ -522,9 +533,11 @@ def test_exts_list(self):
self.assertEqual(exts_sources[3]['name'], 'ext-pi')
self.assertEqual(exts_sources[3]['version'], '3.0')
- modfile = os.path.join(eb.make_module_step(), 'PI', '3.14' + eb.module_generator.MODULE_FILE_EXTENSION)
+ with self.mocked_stdout_stderr():
+ modfile = os.path.join(eb.make_module_step(), 'PI', '3.14' + eb.module_generator.MODULE_FILE_EXTENSION)
modtxt = read_file(modfile)
- regex = re.compile('EBEXTSLISTPI.*ext1-1.0,ext2-2.0')
+ # verify that templates used for extensions are resolved as they should
+ regex = re.compile('EBEXTSLISTPI.*"ext1-1.0,ext2-2.0,ext-PI-3.14,ext-pi-3.0')
self.assertTrue(regex.search(modtxt), "Pattern '%s' found in: %s" % (regex.pattern, modtxt))
def test_extensions_templates(self):
@@ -568,7 +581,8 @@ def test_extensions_templates(self):
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
- eb.fetch_step()
+ with self.mocked_stdout_stderr():
+ eb.fetch_step()
# inject OS dependency that can not be fullfilled,
# to check whether OS deps are validated again for each extension (they shouldn't be);
@@ -577,7 +591,8 @@ def test_extensions_templates(self):
eb.cfg.rawtxt += "\nosdependencies = ['this_os_dep_does_not_exist']"
# run extensions step to install 'toy' extension
- eb.extensions_step()
+ with self.mocked_stdout_stderr():
+ eb.extensions_step()
# check whether template values were resolved correctly in Extension instances that were created/used
toy_ext = eb.ext_instances[0]
@@ -660,8 +675,8 @@ def test_tweaking(self):
'version = "3.14"',
'toolchain = {"name": "GCC", "version": "4.6.3"}',
'patches = %s',
- 'parallel = 1',
- 'keepsymlinks = True',
+ 'maxparallel = 1',
+ 'keepsymlinks = False',
]) % str(patches)
self.prep()
@@ -679,11 +694,11 @@ def test_tweaking(self):
'versionsuffix': versuff,
'toolchain_version': tcver,
'patches': new_patches,
- 'keepsymlinks': 'True', # Don't change this
+ 'keepsymlinks': 'False', # Don't change this
# It should be possible to overwrite values with True/False/None as they often have special meaning
'runtest': 'False',
'hidden': 'True',
- 'parallel': 'None', # Good example: parallel=None means "Auto detect"
+ 'maxparallel': 'None', # Good example: maxparallel=None means "unlimitted"
# Adding new options (added only by easyblock) should also be possible
# and in case the string "True/False/None" is really wanted it is possible to quote it first
'test_none': '"False"',
@@ -700,7 +715,7 @@ def test_tweaking(self):
self.assertEqual(eb['patches'], new_patches)
self.assertIs(eb['runtest'], False)
self.assertIs(eb['hidden'], True)
- self.assertIsNone(eb['parallel'])
+ self.assertIsNone(eb['maxparallel'])
self.assertEqual(eb['test_none'], 'False')
self.assertEqual(eb['test_bool'], 'True')
self.assertEqual(eb['test_123'], 'None')
@@ -737,6 +752,32 @@ def test_tweaking(self):
# cleanup
os.remove(tweaked_fn)
+ def test_parse_easyconfig(self):
+ """Test parse_easyconfig function"""
+ self.contents = textwrap.dedent("""
+ easyblock = "ConfigureMake"
+ name = "PI"
+ version = "3.14"
+ homepage = "http://example.com"
+ description = "test easyconfig"
+ toolchain = SYSTEM
+ """)
+ self.prep()
+ ecs, gen_ecs = parse_easyconfigs([(self.eb_file, False)])
+ self.assertEqual(len(ecs), 1)
+ self.assertEqual(ecs[0]['spec'], self.eb_file)
+ self.assertIsInstance(ecs[0]['ec'], EasyConfig)
+ self.assertFalse(gen_ecs)
+ # Passing the same EC multiple times is ignored
+ ecs, gen_ecs = parse_easyconfigs([(self.eb_file, False), (self.eb_file, False)])
+ self.assertEqual(len(ecs), 1)
+ # Similar for symlinks
+ linked_ec = os.path.join(self.test_prefix, 'linked.eb')
+ os.symlink(self.eb_file, linked_ec)
+ ecs, gen_ecs = parse_easyconfigs([(self.eb_file, False), (linked_ec, False)])
+ self.assertEqual(len(ecs), 1)
+ self.assertEqual(ecs[0]['spec'], self.eb_file)
+
def test_alt_easyconfig_paths(self):
"""Test alt_easyconfig_paths function that collects list of additional paths for easyconfig files."""
@@ -790,7 +831,8 @@ def test_tweak_multiple_tcs(self):
untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb')
easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)])
tweak_specs = {'moduleclass': 'debugger'}
- easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths)
+ easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths,
+ return_map=True)
# Check that all expected tweaked easyconfigs exists
tweaked_openmpi_1 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_1))
tweaked_openmpi_2 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_2))
@@ -802,6 +844,7 @@ def test_tweak_multiple_tcs(self):
"Tweaked value not found in " + tweaked_openmpi_content_1)
self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_2,
"Tweaked value not found in " + tweaked_openmpi_content_2)
+ self.assertEqual(tweak_map, {tweaked_openmpi_1: untweaked_openmpi_1, tweaked_openmpi_2: untweaked_openmpi_2})
def test_installversion(self):
"""Test generation of install version."""
@@ -1189,7 +1232,6 @@ def test_templating_constants(self):
'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s',
]),
'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}',
- 'modextrapaths_append = {"PATH_APPEND": "appended_path"}',
'license_file = HOME + "/licenses/PI/license.txt"',
"github_account = 'easybuilders'",
]) % inp
@@ -1198,9 +1240,11 @@ def test_templating_constants(self):
ec.validate()
# temporarily disable templating, just so we can check later whether it's *still* disabled
+ self.assertTrue(ec.templating_enabled)
with ec.disable_templating():
ec.generate_template_values()
- self.assertFalse(ec.enable_templating)
+ self.assertFalse(ec.templating_enabled)
+ self.assertTrue(ec.templating_enabled)
self.assertEqual(ec['description'], "test easyconfig PI")
self.assertEqual(ec['sources'][0], 'PI-3.04.tar.gz')
@@ -1230,7 +1274,6 @@ def test_templating_constants(self):
self.assertEqual(ec['modloadmsg'], expected)
self.assertEqual(ec['modunloadmsg'], expected)
self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'})
- self.assertEqual(ec['modextrapaths_append'], {'PATH_APPEND': 'appended_path'})
self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt'))
# test the escaping insanity here (ie all the crap we allow in easyconfigs)
@@ -1260,6 +1303,56 @@ def test_templating_constants(self):
ec = EasyConfig(test_ec)
self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy'])
+ def test_template_constant_import(self):
+ """Test importing template constants works"""
+ from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT
+ from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
+ self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0])
+ self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0])
+ self.assertEqual(SHLIB_EXT, get_shared_lib_ext())
+
+ def test_ec_method_resolve_template(self):
+ """Test the `resolve_template` method of easyconfig instances."""
+ # don't use any escaping insanity here, since it is templated itself
+ self.contents = textwrap.dedent("""
+ easyblock = "ConfigureMake"
+ name = "PI"
+ version = "3.14"
+ homepage = "http://example.com"
+ description = "test easyconfig %(name)s version %(version_major)s"
+ toolchain = SYSTEM
+ installopts = "PREFIX=%(installdir)s"
+ """)
+ self.prep()
+ ec = EasyConfig(self.eb_file, validate=False)
+
+ # We can resolve anything with values from the EC
+ self.assertEqual(ec.resolve_template('%(namelower)s %(version_major)s begins with %(nameletterlower)s'),
+ 'pi 3 begins with p')
+
+ # `resolve_template` does basically the same resolving any value on acccess
+ description = ec.get('description', resolve=False)
+ self.assertIn('%', description, 'Description needs a template for the next test')
+ self.assertEqual(ec.resolve_template(description), ec['description'])
+
+ val = "PREFIX=%(installdir)s"
+
+ # by default unresolved template value triggers an error being raised
+ error_pattern = "Failed to resolve all templates"
+ self.assertErrorRegex(EasyBuildError, error_pattern, ec.resolve_template, val)
+ self.assertErrorRegex(EasyBuildError, error_pattern, ec.get, 'installopts')
+
+ # this can be (temporarily) disabled
+ with ec.allow_unresolved_templates():
+ self.assertFalse(ec.expect_resolved_template_values)
+ self.assertEqual(ec.resolve_template(val), val)
+ self.assertEqual(ec['installopts'], val)
+
+ # Enforced again
+ self.assertTrue(ec.expect_resolved_template_values)
+ self.assertErrorRegex(EasyBuildError, error_pattern, ec.resolve_template, val)
+ self.assertErrorRegex(EasyBuildError, error_pattern, ec.get, 'installopts')
+
def test_templating_cuda_toolchain(self):
"""Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain."""
@@ -1367,7 +1460,7 @@ def test_templating_doc(self):
# expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between)
temps = [
easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG,
- easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3,
+ list(easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS.keys()) * 3,
easyconfig.templates.TEMPLATE_NAMES_CONFIG,
easyconfig.templates.TEMPLATE_NAMES_LOWER,
easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,
@@ -1423,6 +1516,30 @@ def test_start_dir_template(self):
self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt)
self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt)
+ def test_rpath_template(self):
+ """Test the %(rpath)s template"""
+ test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "configopts = '--with-rpath=%(rpath_enabled)s'"
+ write_file(test_ec, test_ec_txt)
+
+ ec = EasyConfig(test_ec)
+ expected = '--with-rpath=true' if get_os_name() == 'Linux' else '--with-rpath=false'
+ self.assertEqual(ec['configopts'], expected)
+
+ # force True
+ update_build_option('rpath', True)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--with-rpath=true")
+
+ # force False
+ update_build_option('rpath', False)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--with-rpath=false")
+
def test_sysroot_template(self):
"""Test the %(sysroot)s template"""
@@ -1480,6 +1597,47 @@ def test_software_commit_template(self):
self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit)
self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit)
+ def test_template_deprecation_and_alternative(self):
+ """Test deprecation of (and alternative) templates"""
+
+ self.prep()
+
+ template_test_deprecations = {
+ 'builddir': ('depr_build_dir', '1000000000'),
+ 'cudaver': ('depr_cuda_ver', '1000000000'),
+ 'start_dir': ('depr_start_dir', '1000000000'),
+ }
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = template_test_deprecations
+
+ template_test_alternatives = {
+ 'installdir': 'alt_install_dir',
+ 'version_major_minor': 'alt_ver_maj_min',
+ }
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = template_test_alternatives
+
+ tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'"
+ " && echo %(alt_install_dir)s %(version_major_minor)s")
+ tmpl_dict = {
+ 'depr_build_dir': '/example/build_dir',
+ 'depr_cuda_ver': '12.1.1',
+ 'installdir': '/example/installdir',
+ 'start_dir': '/example/build_dir/start_dir',
+ 'alt_ver_maj_min': '1.2',
+ 'namelower': 'foo',
+ }
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ res = resolve_template(tmpl_str, tmpl_dict)
+ stderr = stderr.getvalue()
+
+ for tmpl in [*template_test_deprecations.keys(), *template_test_alternatives.keys()]:
+ self.assertNotIn("%(" + tmpl + ")s", res)
+
+ for old, (new, ver) in template_test_deprecations.items():
+ depr_str = (f"WARNING: Deprecated functionality, will no longer work in EasyBuild v{ver}: "
+ f"Easyconfig template '{old}' is deprecated, use '{new}' instead")
+ self.assertIn(depr_str, stderr)
+
def test_constant_doc(self):
"""test constant documentation"""
doc = avail_easyconfig_constants()
@@ -1587,9 +1745,10 @@ def test_buildininstalldir(self):
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
- eb.post_init()
- eb.make_builddir()
- eb.make_installdir()
+ with self.mocked_stdout_stderr():
+ eb.post_init()
+ eb.make_builddir()
+ eb.make_installdir()
self.assertEqual(eb.builddir, eb.installdir)
self.assertTrue(os.path.isdir(eb.builddir))
@@ -1671,17 +1830,6 @@ def test_get_easyblock_class(self):
self.assertErrorRegex(EasyBuildError, "neither name nor easyblock were specified", get_easyblock_class, None)
self.assertEqual(get_easyblock_class(None, error_on_missing_easyblock=False), None)
- # also test deprecated default_fallback named argument
- self.assertErrorRegex(EasyBuildError, "DEPRECATED", get_easyblock_class, None, name='gzip',
- default_fallback=False)
-
- orig_value = easybuild.tools.build_log.CURRENT_VERSION
- easybuild.tools.build_log.CURRENT_VERSION = '3.9'
- self.mock_stderr(True)
- self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None)
- self.mock_stderr(False)
- easybuild.tools.build_log.CURRENT_VERSION = orig_value
-
def test_letter_dir(self):
"""Test letter_dir_for function."""
test_cases = {
@@ -1809,27 +1957,70 @@ def foo(key):
self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
+ def test_alternative_easyconfig_parameters(self):
+ """Test handling of alternative easyconfig parameters."""
+
+ test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds')
+ test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+
+ # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = {}
+ self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec)
+
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = {
+ 'env_mod_class': 'moduleclass',
+ 'post_install_cmds': 'postinstallcmds',
+ }
+ ec = EasyConfig(test_ec)
+
+ expected = 'tools'
+ self.assertEqual(ec['moduleclass'], expected)
+ self.assertEqual(ec['env_mod_class'], expected)
+
+ expected = ['echo TOY > %(installdir)s/README']
+ with ec.disable_templating():
+ self.assertEqual(ec['postinstallcmds'], expected)
+ self.assertEqual(ec['post_install_cmds'], expected)
+
+ # test setting of easyconfig parameter with original & alternative name
+ ec['moduleclass'] = 'test1'
+ self.assertEqual(ec['moduleclass'], 'test1')
+ self.assertEqual(ec['env_mod_class'], 'test1')
+ ec.update('moduleclass', 'test2')
+ self.assertEqual(ec['moduleclass'], 'test1 test2 ')
+ self.assertEqual(ec['env_mod_class'], 'test1 test2 ')
+
+ ec['env_mod_class'] = 'test3'
+ self.assertEqual(ec['moduleclass'], 'test3')
+ self.assertEqual(ec['env_mod_class'], 'test3')
+ ec.update('env_mod_class', 'test4')
+ self.assertEqual(ec['moduleclass'], 'test3 test4 ')
+ self.assertEqual(ec['env_mod_class'], 'test3 test4 ')
+
def test_deprecated_easyconfig_parameters(self):
- """Test handling of replaced easyconfig parameters."""
- os.environ.pop('EASYBUILD_DEPRECATED')
- easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version
+ """Test handling of deprecated easyconfig parameters."""
+ self.allow_deprecated_behaviour()
init_config()
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'))
- orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS)
- easyconfig.parser.DEPRECATED_PARAMETERS.update({
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = {
'foobar': ('barfoo', '0.0'), # deprecated since forever
- 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while
- })
-
- # copy classes before reloading, so we can restore them (other isinstance checks fail)
- orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig)
- orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS)
- reload(easyconfig.parser)
+ # won't be actually deprecated for a while;
+ # note that we should map foobarbarfoo to a valid easyconfig parameter here,
+ # or we'll hit errors when parsing an easyconfig file that uses it
+ 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'),
+ }
- for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items():
+ for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS.items():
if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION:
# deprecation error
error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey)
@@ -1842,18 +2033,42 @@ def foo(key):
self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
else:
# only deprecation warning, but key is replaced when getting/setting
- ec[key] = 'test123'
- self.assertEqual(ec[newkey], 'test123')
- self.assertEqual(ec[key], 'test123')
- ec[newkey] = '123test'
- self.assertEqual(ec[newkey], '123test')
- self.assertEqual(ec[key], '123test')
-
- easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters
- reload(easyconfig.parser)
- reload(easyconfig.easyconfig)
- easyconfig.easyconfig.EasyConfig = orig_EasyConfig
- easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS
+ with self.mocked_stdout_stderr():
+ ec[key] = 'test123'
+ self.assertEqual(ec[newkey], 'test123')
+ self.assertEqual(ec[key], 'test123')
+ ec[newkey] = '123test'
+ self.assertEqual(ec[newkey], '123test')
+ self.assertEqual(ec[key], '123test')
+
+ variables = {
+ 'name': 'example',
+ 'version': '1.2.3',
+ 'foobar': 'foobar',
+ 'local_var': 'test',
+ }
+ ec = {
+ 'name': None,
+ 'version': None,
+ 'homepage': None,
+ 'toolchain': None,
+ }
+ ec_params, unknown_keys = triage_easyconfig_params(variables, ec)
+ # deprecated easyconfig parameter 'foobar' is retained as easyconfig parameter;
+ # only local_var is not retained, since that's a local variable
+ self.assertEqual(unknown_keys, [])
+ expected = {'name': 'example', 'version': '1.2.3', 'foobar': 'foobar'}
+ self.assertEqual(ec_params, expected)
+
+ # try parsing an easyconfig file that defines a deprecated easyconfig parameter
+ toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, read_file(toy_ec))
+ write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True)
+
+ with self.mocked_stdout_stderr():
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo')
def test_unknown_easyconfig_parameter(self):
"""Check behaviour when unknown easyconfig parameters are used."""
@@ -2115,7 +2330,8 @@ def test_external_dependencies(self):
os.environ['PI_PREFIX'] = '/test/prefix/PI'
os.environ['TEST_INC'] = '/test/prefix/test/include'
ec.toolchain.dry_run = True
- ec.toolchain.prepare(deps=ec.dependencies(), silent=True)
+ with self.mocked_stdout_stderr():
+ ec.toolchain.prepare(deps=ec.dependencies(), silent=True)
self.assertEqual(os.environ.get('EBROOTBAR'), '/foo/bar')
self.assertEqual(os.environ.get('EBROOTFOO'), '/foo/bar')
@@ -2178,8 +2394,8 @@ def test_external_dependencies_templates(self):
'pyshortver': '3.6',
'pyver': '3.6.5',
}
- for key in expected_template_values:
- self.assertEqual(ec.template_values[key], expected_template_values[key])
+ for key, expected in expected_template_values.items():
+ self.assertEqual(ec.template_values[key], expected)
self.assertEqual(ec['versionsuffix'], '-Python-3.6.5-Perl-5.30')
@@ -2255,8 +2471,8 @@ def test_quote_str(self):
'foo\\bar': '"foo\\bar"',
}
- for t in teststrings:
- self.assertEqual(quote_str(t), teststrings[t])
+ for t, expected in teststrings.items():
+ self.assertEqual(quote_str(t), expected)
# test escape_newline
self.assertEqual(quote_str("foo\nbar", escape_newline=False), '"foo\nbar"')
@@ -2354,11 +2570,11 @@ def test_dump(self):
test_ec = os.path.join(self.test_prefix, 'test.eb')
ec = EasyConfig(os.path.join(test_ecs_dir, ecfile))
- ec.enable_templating = False
- ecdict = ec.asdict()
- ec.dump(test_ec)
- # dict representation of EasyConfig instance should not change after dump
- self.assertEqual(ecdict, ec.asdict())
+ with ec.disable_templating():
+ ecdict = ec.asdict()
+ ec.dump(test_ec)
+ # dict representation of EasyConfig instance should not change after dump
+ self.assertEqual(ecdict, ec.asdict())
ectxt = read_file(test_ec)
patterns = [
@@ -2373,7 +2589,6 @@ def test_dump(self):
# parse result again
dumped_ec = EasyConfig(test_ec)
- dumped_ec.enable_templating = False
# check that selected parameters still have the same value
params = [
@@ -2382,9 +2597,10 @@ def test_dump(self):
'dependencies', # checking this is important w.r.t. filtered hidden dependencies being restored in dump
'exts_list', # exts_lists (in Python easyconfig) use another layer of templating so shouldn't change
]
- for param in params:
- if param in ec:
- self.assertEqual(ec[param], dumped_ec[param])
+ with ec.disable_templating(), dumped_ec.disable_templating():
+ for param in params:
+ if param in ec:
+ self.assertEqual(ec[param], dumped_ec[param])
ec_txt = textwrap.dedent("""
easyblock = 'EB_toy'
@@ -2538,8 +2754,8 @@ def test_dump_autopep8(self):
def test_dump_extra(self):
"""Test EasyConfig's dump() method for files containing extra values"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_extra (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_extra pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -2581,8 +2797,8 @@ def test_dump_extra(self):
def test_dump_template(self):
""" Test EasyConfig's dump() method for files containing templates"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_template (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_template pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -2635,7 +2851,7 @@ def test_dump_template(self):
ec.dump(testec)
ectxt = read_file(testec)
- self.assertTrue(ec.enable_templating) # templating should still be enabled after calling dump()
+ self.assertTrue(ec.templating_enabled) # templating should still be enabled after calling dump()
patterns = [
r"easyblock = 'EB_foo'",
@@ -2670,8 +2886,8 @@ def test_dump_template(self):
def test_dump_comments(self):
""" Test dump() method for files containing comments """
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_comments (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_comments pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -3365,6 +3581,8 @@ def test_template_constant_dict(self):
arch_regex = re.compile('^[a-z0-9_]+$')
+ rpath = 'true' if get_os_name() == 'Linux' else 'false'
+
expected = {
'bitbucket_account': 'gzip',
'github_account': 'gzip',
@@ -3373,7 +3591,7 @@ def test_template_constant_dict(self):
'namelower': 'gzip',
'nameletter': 'g',
'nameletterlower': 'g',
- 'parallel': None,
+ 'rpath_enabled': rpath,
'software_commit': '',
'sysroot': '',
'toolchain_name': 'foss',
@@ -3400,17 +3618,17 @@ def test_template_constant_dict(self):
except AttributeError:
pass # Ignore if not present
orig_get_avail_core_count = st.get_avail_core_count
- st.get_avail_core_count = lambda: 42
+ st.get_avail_core_count = lambda: 12
# also check template values after running check_readiness_step (which runs set_parallel)
eb = EasyBlock(ec)
- eb.check_readiness_step()
+ with self.mocked_stdout_stderr():
+ eb.check_readiness_step()
st.get_avail_core_count = orig_get_avail_core_count
res = template_constant_dict(ec)
res.pop('arch')
- expected['parallel'] = 42
self.assertEqual(res, expected)
toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb')
@@ -3452,11 +3670,11 @@ def test_template_constant_dict(self):
'toolchain_name': 'system',
'toolchain_version': 'system',
'nameletterlower': 't',
- 'parallel': None,
'pymajver': '3',
'pyminver': '7',
'pyshortver': '3.7',
'pyver': '3.7.2',
+ 'rpath_enabled': rpath,
'software_commit': '',
'sysroot': '',
'version': '0.01',
@@ -3487,7 +3705,7 @@ def test_template_constant_dict(self):
ec = EasyConfigParser(filename=test_ec).get_config_dict()
expected['module_name'] = None
- for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'):
+ for key in ('bitbucket_account', 'github_account', 'versionprefix'):
del expected[key]
dep_names = [x[0] for x in ec['dependencies']]
@@ -3523,6 +3741,7 @@ def test_template_constant_dict(self):
'namelower': 'foo',
'nameletter': 'f',
'nameletterlower': 'f',
+ 'rpath_enabled': rpath,
'software_commit': '',
'sysroot': '',
'version': '1.2.3',
@@ -3599,6 +3818,7 @@ def test_hidden_toolchain(self):
args = [
ec_file,
'--dry-run',
+ '--robot',
]
outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt))
@@ -3716,7 +3936,7 @@ def test_resolve_template(self):
# On unknown values the value is returned unchanged
for value in ('%(invalid)s', '%(name)s %(invalid)s', '%%%(invalid)s', '% %(invalid)s', '%s %(invalid)s'):
- self.assertEqual(resolve_template(value, tmpl_dict), value)
+ self.assertEqual(resolve_template(value, tmpl_dict, expect_resolved=False), value)
def test_det_subtoolchain_version(self):
"""Test det_subtoolchain_version function"""
@@ -3754,19 +3974,6 @@ def test_det_subtoolchain_version(self):
for subtoolchain_name in subtoolchains[current_tc['name']]]
self.assertEqual(versions, [None, ''])
- # --add-dummy-to-minimal-toolchains is still supported, but deprecated
- self.allow_deprecated_behaviour()
- init_config(build_options={'add_system_to_minimal_toolchains': False, 'add_dummy_to_minimal_toolchains': True})
- self.mock_stderr(True)
- versions = [det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands)
- for subtoolchain_name in subtoolchains[current_tc['name']]]
- stderr = self.get_stderr()
- self.mock_stderr(False)
- self.assertEqual(versions, [None, ''])
- depr_msg = "WARNING: Deprecated functionality, will no longer work in v5.0: "
- depr_msg += "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
- self.assertIn(depr_msg, stderr)
-
# and GCCcore if existing too
init_config(build_options={'add_system_to_minimal_toolchains': True})
current_tc = {'name': 'GCC', 'version': '4.9.3-2.25'}
@@ -3929,22 +4136,6 @@ def test_get_paths_for(self):
if env_eb_script_path:
os.environ['EB_SCRIPT_PATH'] = env_eb_script_path
- def test_is_generic_easyblock(self):
- """Test for is_generic_easyblock function."""
-
- # is_generic_easyblock in easyconfig.py is deprecated, moved to filetools.py
- self.allow_deprecated_behaviour()
-
- self.mock_stderr(True)
-
- for name in ['Binary', 'ConfigureMake', 'CMakeMake', 'PythonPackage', 'JAR']:
- self.assertTrue(is_generic_easyblock(name))
-
- for name in ['EB_bzip2', 'EB_DL_underscore_POLY_underscore_Classic', 'EB_GCC', 'EB_WRF_minus_Fire']:
- self.assertFalse(is_generic_easyblock(name))
-
- self.mock_stderr(False)
-
def test_get_module_path(self):
"""Test get_module_path function."""
self.assertEqual(get_module_path('EB_bzip2', generic=False), 'easybuild.easyblocks.bzip2')
@@ -4308,9 +4499,6 @@ def test_fix_deprecated_easyconfigs(self):
"""Test fix_deprecated_easyconfigs function."""
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
- gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.4.eb')
-
- gzip_ec_txt = read_file(gzip_ec)
toy_ec_txt = read_file(toy_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
@@ -4327,115 +4515,42 @@ def test_fix_deprecated_easyconfigs(self):
regex = re.compile(r'^(toolchain\s*=.*)$', re.M)
test_ectxt = regex.sub(r'\1\n\nsome_list = [x + "1" for x in ["one", "two", "three"]]', test_ectxt)
- # test fixing the use of 'dummy' toolchain to SYSTEM
- tc_regex = re.compile('^toolchain = .*', re.M)
- tc_strs = [
- "{'name': 'dummy', 'version': 'dummy'}",
- "{'name': 'dummy', 'version': ''}",
- "{'name': 'dummy', 'version': '1.2.3'}",
- "{'version': '', 'name': 'dummy'}",
- "{'version': 'dummy', 'name': 'dummy'}",
- ]
-
unknown_params_error_pattern = "Use of 2 unknown easyconfig parameters detected in test.eb: foo, some_list"
- for tc_str in tc_strs:
- # first check if names of local variables get fixed if 'dummy' toolchain is not used
- init_config(build_options={'local_var_naming_check': 'error', 'silent': True})
-
- write_file(test_ec, test_ectxt)
- self.assertErrorRegex(EasyBuildError, unknown_params_error_pattern, EasyConfig, test_ec)
-
- self.mock_stderr(True)
- self.mock_stdout(True)
- fix_deprecated_easyconfigs([test_ec])
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
- self.assertFalse(stderr)
- self.assertIn("test.eb... FIXED!", stdout)
-
- # parsing now works
- ec = EasyConfig(test_ec)
-
- # cleanup
- remove_file(glob.glob(os.path.join(test_ec + '.orig*'))[0])
-
- # now inject use of 'dummy' toolchain
- write_file(test_ec, tc_regex.sub("toolchain = %s" % tc_str, test_ectxt))
-
- test_ec_txt = read_file(test_ec)
- regex = re.compile("^toolchain = {.*'name': 'dummy'.*$", re.M)
- self.assertTrue(regex.search(test_ec_txt), "Pattern '%s' found in: %s" % (regex.pattern, test_ec_txt))
-
- # mimic default behaviour where only warnings are being printed;
- # use of dummy toolchain or local variables not following recommended naming scheme is not fatal by default
- init_config(build_options={'local_var_naming_check': 'warn', 'silent': False})
- self.mock_stderr(True)
- self.mock_stdout(True)
- ec = EasyConfig(test_ec)
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
-
- self.assertFalse(stdout)
+ # check if names of local variables get fixed
+ init_config(build_options={'local_var_naming_check': 'error', 'silent': True})
- warnings = [
- "WARNING: Use of 2 unknown easyconfig parameters detected in test.eb: foo, some_list",
- "Use of 'dummy' toolchain is deprecated, use 'system' toolchain instead",
- ]
- for warning in warnings:
- self.assertIn(warning, stderr)
-
- init_config(build_options={'local_var_naming_check': 'error', 'silent': True})
-
- # easyconfig doesn't parse because of local variables with name other than 'local_*'
- self.assertErrorRegex(EasyBuildError, unknown_params_error_pattern, EasyConfig, test_ec)
-
- self.mock_stderr(True)
- self.mock_stdout(True)
- fix_deprecated_easyconfigs([toy_ec, test_ec, gzip_ec])
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
-
- ectxt = read_file(test_ec)
- self.assertFalse(regex.search(ectxt), "Pattern '%s' *not* found in: %s" % (regex.pattern, ectxt))
- regex = re.compile("^toolchain = SYSTEM$", re.M)
- self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt))
-
- self.assertEqual(gzip_ec_txt, read_file(gzip_ec))
- self.assertEqual(toy_ec_txt, read_file(toy_ec))
- self.assertTrue(test_ec_txt != read_file(test_ec))
-
- # original easyconfig is backed up automatically
- test_ecs = sorted([f for f in os.listdir(self.test_prefix) if f.startswith('test.eb')])
- self.assertEqual(len(test_ecs), 2)
- backup_test_ec = os.path.join(self.test_prefix, test_ecs[1])
- self.assertEqual(test_ec_txt, read_file(backup_test_ec))
+ write_file(test_ec, test_ectxt)
+ self.assertErrorRegex(EasyBuildError, unknown_params_error_pattern, EasyConfig, test_ec)
- remove_file(backup_test_ec)
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ fix_deprecated_easyconfigs([test_ec])
+ stderr, stdout = self.get_stderr(), self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertFalse(stderr)
+ self.assertIn("test.eb... FIXED!", stdout)
- # parsing works now, toolchain is replaced with system toolchain
- ec = EasyConfig(test_ec)
- self.assertEqual(ec['toolchain'], {'name': 'system', 'version': 'system'})
- self.assertEqual(ec['configopts'], "--foobar --barfoo --barfoobaz")
+ # parsing now works
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--foobar --barfoo --barfoobaz")
+ self.assertFalse(stderr)
+ stdout = stdout.split('\n')
+ self.assertEqual(len(stdout), 6)
+ patterns = [
+ r"^\* \[1/1\] fixing .*/test.eb\.\.\. FIXED!$",
+ r"^\s*\(changes made in place, original copied to .*/test.eb.orig_[0-9_]+\)$",
+ r'^$',
+ r"^All done! Fixed 1 easyconfigs \(out of 1 found\).$",
+ r'^$',
+ r'^$',
+ ]
+ for idx, pattern in enumerate(patterns):
+ self.assertTrue(re.match(pattern, stdout[idx]), "Pattern '%s' matches '%s'" % (pattern, stdout[idx]))
- self.assertFalse(stderr)
- stdout = stdout.split('\n')
- self.assertEqual(len(stdout), 8)
- patterns = [
- r"^\* \[1/3\] fixing .*/t/toy/toy-0.0.eb\.\.\. \(no changes made\)$",
- r"^\* \[2/3\] fixing .*/test.eb\.\.\. FIXED!$",
- r"^\s*\(changes made in place, original copied to .*/test.eb.orig_[0-9_]+\)$",
- r"^\* \[3/3\] fixing .*/g/gzip/gzip-1.4.eb\.\.\. \(no changes made\)$",
- r'^$',
- r"^All done! Fixed 1 easyconfigs \(out of 3 found\).$",
- r'^$',
- r'^$',
- ]
- for idx, pattern in enumerate(patterns):
- self.assertTrue(re.match(pattern, stdout[idx]), "Pattern '%s' matches '%s'" % (pattern, stdout[idx]))
+ # cleanup
+ remove_file(glob.glob(os.path.join(test_ec + '.orig*'))[0])
def test_parse_list_comprehension_scope(self):
"""Test parsing of an easyconfig file that uses a local variable in list comprehension."""
@@ -4495,15 +4610,19 @@ def test_triage_easyconfig_params(self):
self.assertEqual(sorted(unknown_keys), ['bleh', 'foobar'])
# check behaviour when easyconfig parameters that use a name indicating a local variable were defined
- ec.update({
+ local_vars = {
'x': None,
'local_foo': None,
'_foo': None,
'_': None,
- })
+ }
+ ec.update(local_vars)
error = "Found 4 easyconfig parameters that are considered local variables: _, _foo, local_foo, x"
self.assertErrorRegex(EasyBuildError, error, triage_easyconfig_params, variables, ec)
+ for key in local_vars:
+ del ec[key]
+
def test_local_vars_detection(self):
"""Test detection of using unknown easyconfig parameters that are likely local variables."""
@@ -4645,31 +4764,39 @@ def test_cuda_compute_capabilities(self):
description = 'test'
toolchain = SYSTEM
cuda_compute_capabilities = ['5.1', '7.0', '7.1']
- installopts = '%(cuda_compute_capabilities)s'
- preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"'
- prebuildopts = '%(cuda_cc_semicolon_sep)s'
- configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"'
+ configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
+ prebuildopts = '%(cuda_cc_semicolon_sep)s'
+ buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" '
+ 'semi="%(cuda_int_semicolon_sep)s"')
+ preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"'
+ installopts = '%(cuda_compute_capabilities)s'
""")
self.prep()
ec = EasyConfig(self.eb_file)
- self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
- self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"')
- self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
+ self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" '
'space="sm_51 sm_70 sm_71"')
- self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
+ self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
+ self.assertEqual(ec['buildopts'], 'comma="51,70,71" '
+ 'space="51 70 71" '
+ 'semi="51;70;71"')
+ self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"')
+ self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
# build options overwrite it
init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']})
ec = EasyConfig(self.eb_file)
- self.assertEqual(ec['installopts'], '4.2,6.3')
- self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"')
- self.assertEqual(ec['prebuildopts'], '4.2;6.3')
+ self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" '
'space="sm_42 sm_63"')
- self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
+ self.assertEqual(ec['buildopts'], 'comma="42,63" '
+ 'space="42 63" '
+ 'semi="42;63"')
+ self.assertEqual(ec['prebuildopts'], '4.2;6.3')
+ self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"')
+ self.assertEqual(ec['installopts'], '4.2,6.3')
def test_det_copy_ec_specs(self):
"""Test det_copy_ec_specs function."""
@@ -4697,60 +4824,51 @@ def test_det_copy_ec_specs(self):
return
# use fixed PR (speeds up the test due to caching in fetch_files_from_pr;
- # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007
- from_pr = 8007
- arrow_ec_fn = 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb'
- bat_ec_fn = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'
- bat_patch_fn = 'bat-0.3.3-fix-pyspark.patch'
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22345
+ from_pr = 22345
+ ec_fn = 'QuantumESPRESSO-7.4-foss-2024a.eb'
+ patch_fn = 'QuantumESPRESSO-7.4-parallel-symmetrization.patch'
pr_files = [
- arrow_ec_fn,
- bat_ec_fn,
- bat_patch_fn,
+ ec_fn,
+ patch_fn,
]
# if no paths are specified, default is to copy all files touched by PR to current working directory
paths, target_path = det_copy_ec_specs([], from_pr)
- self.assertEqual(len(paths), 3)
+ self.assertEqual(len(paths), 2)
filenames = sorted([os.path.basename(x) for x in paths])
self.assertEqual(filenames, sorted(pr_files))
self.assertTrue(os.path.samefile(target_path, cwd))
# last argument is used as target directory,
# unless it corresponds to a file touched by PR
- args = [bat_ec_fn, 'target_dir']
+ args = [ec_fn, 'target_dir']
paths, target_path = det_copy_ec_specs(args, from_pr)
self.assertEqual(len(paths), 1)
- self.assertEqual(os.path.basename(paths[0]), bat_ec_fn)
+ self.assertEqual(os.path.basename(paths[0]), ec_fn)
self.assertEqual(target_path, 'target_dir')
- args = [bat_ec_fn]
+ args = [ec_fn]
paths, target_path = det_copy_ec_specs(args, from_pr)
self.assertEqual(len(paths), 1)
- self.assertEqual(os.path.basename(paths[0]), bat_ec_fn)
+ self.assertEqual(os.path.basename(paths[0]), ec_fn)
self.assertTrue(os.path.samefile(target_path, cwd))
- args = [arrow_ec_fn, bat_ec_fn]
+ args = [ec_fn, patch_fn]
paths, target_path = det_copy_ec_specs(args, from_pr)
self.assertEqual(len(paths), 2)
- self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn)
- self.assertEqual(os.path.basename(paths[1]), bat_ec_fn)
- self.assertTrue(os.path.samefile(target_path, cwd))
-
- args = [bat_ec_fn, bat_patch_fn]
- paths, target_path = det_copy_ec_specs(args, from_pr)
- self.assertEqual(len(paths), 2)
- self.assertEqual(os.path.basename(paths[0]), bat_ec_fn)
- self.assertEqual(os.path.basename(paths[1]), bat_patch_fn)
+ self.assertEqual(os.path.basename(paths[0]), ec_fn)
+ self.assertEqual(os.path.basename(paths[1]), patch_fn)
self.assertTrue(os.path.samefile(target_path, cwd))
# also test with combination of local files and files from PR
- args = [arrow_ec_fn, 'test.eb', 'test.patch', bat_patch_fn]
+ args = [ec_fn, 'test.eb', 'test.patch', patch_fn]
paths, target_path = det_copy_ec_specs(args, from_pr)
self.assertEqual(len(paths), 4)
- self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn)
+ self.assertEqual(os.path.basename(paths[0]), ec_fn)
self.assertEqual(paths[1], 'test.eb')
self.assertEqual(paths[2], 'test.patch')
- self.assertEqual(os.path.basename(paths[3]), bat_patch_fn)
+ self.assertEqual(os.path.basename(paths[3]), patch_fn)
self.assertTrue(os.path.samefile(target_path, cwd))
def test_recursive_module_unload(self):
@@ -4759,6 +4877,10 @@ def test_recursive_module_unload(self):
toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb')
test_ec = os.path.join(self.test_prefix, 'test.eb')
test_ec_txt = read_file(toy_ec)
+
+ # this test only makes sense if depends_on is not used
+ self.allow_deprecated_behaviour()
+ test_ec_txt += '\nmodule_depends_on = False'
write_file(test_ec, test_ec_txt)
test_module = os.path.join(self.test_installpath, 'modules', 'all', 'foss', '2018a')
@@ -4769,7 +4891,10 @@ def test_recursive_module_unload(self):
recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n'
recursive_unload_pat += r'\s*load\("%(mod)s"\)'
else:
- guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
+ if self.modtool.supports_safe_auto_load:
+ guarded_load_pat = r'\nmodule load %(mod)s'
+ else:
+ guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n'
recursive_unload_pat += r'\s*module load %(mod)s'
@@ -4784,8 +4909,9 @@ def test_recursive_module_unload(self):
self.assertFalse(ec['recursive_module_unload'])
eb = EasyBlock(ec)
eb.builddir = self.test_prefix
- eb.prepare_step()
- eb.make_module_step()
+ with self.mocked_stdout_stderr():
+ eb.prepare_step()
+ eb.make_module_step()
modtxt = read_file(test_module)
fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
@@ -4797,43 +4923,62 @@ def test_recursive_module_unload(self):
# recursive_module_unload easyconfig parameter is honored
test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb')
test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = True'
+ # this test only makes sense if depends_on is not used
+ test_ec_bis_txt += '\nmodule_depends_on = False'
write_file(test_ec_bis, test_ec_bis_txt)
ec_bis = EasyConfig(test_ec_bis)
self.assertTrue(ec_bis['recursive_module_unload'])
eb_bis = EasyBlock(ec_bis)
eb_bis.builddir = self.test_prefix
- eb_bis.prepare_step()
- eb_bis.make_module_step()
+ with self.mocked_stdout_stderr():
+ eb_bis.prepare_step()
+ eb_bis.make_module_step()
modtxt = read_file(test_module)
- fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
- self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
- fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
- self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+ if self.modtool.supports_safe_auto_load:
+ fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+ else:
+ fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
# recursive_mod_unload build option is honored
update_build_option('recursive_mod_unload', True)
eb = EasyBlock(ec)
eb.builddir = self.test_prefix
- eb.prepare_step()
- eb.make_module_step()
+ with self.mocked_stdout_stderr():
+ eb.prepare_step()
+ eb.make_module_step()
modtxt = read_file(test_module)
- fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
- self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
- fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
- self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+ if self.modtool.supports_safe_auto_load:
+ fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+ else:
+ fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
# disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled
self.assertTrue(build_option('recursive_mod_unload'))
test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb')
test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = False'
+ # this test only makes sense if depends_on is not used
+ test_ec_bis_txt += '\nmodule_depends_on = False'
write_file(test_ec_bis, test_ec_bis_txt)
ec_bis = EasyConfig(test_ec_bis)
self.assertEqual(ec_bis['recursive_module_unload'], False)
eb_bis = EasyBlock(ec_bis)
eb_bis.builddir = self.test_prefix
- eb_bis.prepare_step()
- eb_bis.make_module_step()
+ with self.mocked_stdout_stderr():
+ eb_bis.prepare_step()
+ eb_bis.make_module_step()
modtxt = read_file(test_module)
fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
@@ -4928,6 +5073,9 @@ def test_get_cuda_cc_template_value(self):
'cuda_compute_capabilities': '6.5,7.0',
'cuda_cc_space_sep': '6.5 7.0',
'cuda_cc_semicolon_sep': '6.5;7.0',
+ 'cuda_int_comma_sep': '65,70',
+ 'cuda_int_space_sep': '65 70',
+ 'cuda_int_semicolon_sep': '65;70',
'cuda_sm_comma_sep': 'sm_65,sm_70',
'cuda_sm_space_sep': 'sm_65 sm_70',
}
@@ -4937,8 +5085,8 @@ def test_get_cuda_cc_template_value(self):
update_build_option('cuda_compute_capabilities', ['6.5', '7.0'])
ec = EasyConfig(self.eb_file)
- for key in cuda_template_values:
- self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key])
+ for key, expected in cuda_template_values.items():
+ self.assertEqual(ec.get_cuda_cc_template_value(key), expected)
update_build_option('cuda_compute_capabilities', None)
ec = EasyConfig(self.eb_file)
@@ -4950,8 +5098,8 @@ def test_get_cuda_cc_template_value(self):
self.prep()
ec = EasyConfig(self.eb_file)
- for key in cuda_template_values:
- self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key])
+ for key, expected in cuda_template_values.items():
+ self.assertEqual(ec.get_cuda_cc_template_value(key), expected)
def test_count_files(self):
"""Tests for EasyConfig.count_files method."""
@@ -4982,7 +5130,7 @@ def test_count_files(self):
toy_exts_ec = EasyConfig(toy_exts)
self.assertEqual(len(toy_exts_ec['sources']), 1)
self.assertEqual(len(toy_exts_ec['patches']), 1)
- self.assertEqual(len(toy_exts_ec['exts_list']), 4)
+ self.assertEqual(len(toy_exts_ec.get_ref('exts_list')), 4)
self.assertEqual(toy_exts_ec.count_files(), 7)
test_ec = os.path.join(self.test_prefix, 'test.eb')
@@ -4993,7 +5141,7 @@ def test_count_files(self):
' ("test-ext-one", "0.0", {',
' "sources": ["test-ext-one-0.0-part1.tgz", "test-ext-one-0.0-part2.zip"],',
# if both 'sources' and 'source_tmpl' are specified, 'source_tmpl' is ignored,
- # see EasyBlock.fetch_extension_sources, so it should be too when counting files
+ # see EasyBlock.collect_exts_file_info, so it should be too when counting files
' "source_tmpl": "test-ext-one-%(version)s.tar.gz",',
' }),',
' ("test-ext-two", "0.0", {',
@@ -5061,6 +5209,43 @@ def test_easyconfigs_caches(self):
regex = re.compile(r"libtoy/0\.0 is already installed", re.M)
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+ def test_templates(self):
+ """
+ Test use of template values like %(version)s
+ """
+ test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += '\ndescription = "name: %(name)s, version: %(version)s"'
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+ ec = EasyConfig(test_ec)
+
+ # get_ref provides access to non-templated raw value
+ self.assertEqual(ec.get_ref('description'), "name: %(name)s, version: %(version)s")
+ self.assertEqual(ec['description'], "name: toy, version: 0.0")
+
+ # error when using wrong template value or using template value that can not be resolved yet too early
+ test_ec_txt += '\ndescription = "name: %(name)s, version: %(version)s, pyshortver: %(pyshortver)s"'
+ write_file(test_ec, test_ec_txt)
+ ec = EasyConfig(test_ec)
+
+ self.assertEqual(ec.get_ref('description'), "name: %(name)s, version: %(version)s, pyshortver: %(pyshortver)s")
+ error_pattern = r"Failed to resolve all templates in.* %\(pyshortver\)s.* using template dictionary:"
+ self.assertErrorRegex(EasyBuildError, error_pattern, ec.__getitem__, 'description')
+
+ # EasyBuild can be configured to allow unresolved templates
+ update_build_option('allow_unresolved_templates', True)
+ self.assertEqual(ec.get_ref('description'), "name: %(name)s, version: %(version)s, pyshortver: %(pyshortver)s")
+ with self.mocked_stdout_stderr() as (stdout, stderr):
+ self.assertEqual(ec['description'], "name: %(name)s, version: %(version)s, pyshortver: %(pyshortver)s")
+
+ self.assertFalse(stdout.getvalue())
+ regex = re.compile(r"WARNING: Failed to resolve all templates.* %\(pyshortver\)s", re.M)
+ self.assertRegex(stderr.getvalue(), regex)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py
index 7b84a2c867..537b4c1977 100644
--- a/test/framework/easyconfigformat.py
+++ b/test/framework/easyconfigformat.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py
index fa943df938..4fbe6676a0 100644
--- a/test/framework/easyconfigparser.py
+++ b/test/framework/easyconfigparser.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,7 +39,6 @@
from easybuild.framework.easyconfig.parser import EasyConfigParser
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file
-from easybuild.tools.py2vs3 import string_type
TESTDIRBASE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
@@ -198,11 +197,11 @@ def test_easyconfig_constants(self):
# make sure both keys and values are of appropriate types
for constant_name in constants:
- self.assertIsInstance(constant_name, string_type, "Constant name %s is a string" % constant_name)
+ self.assertIsInstance(constant_name, str, "Constant name %s is a string" % constant_name)
val = constants[constant_name]
fail_msg = "The constant %s should have an acceptable type, found %s (%s)" % (constant_name,
type(val), str(val))
- self.assertIsInstance(val, (string_type, dict, tuple), fail_msg)
+ self.assertIsInstance(val, (str, dict, tuple), fail_msg)
# check a couple of randomly picked constant values
self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz')
diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb
index b2195b5ce0..a4fba84756 100644
--- a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb
+++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb
@@ -29,8 +29,7 @@ dependencies = [
# it's nice to have an up to date openssl for security reasons
]
-# zlib is only included here for the sake of testing parsing of .yeb easyconfigs!
-osdependencies = ['zlib', ('openssl-devel', 'libssl-dev', 'libopenssl-devel')]
+osdependencies = [('openssl-devel', 'libssl-dev', 'libopenssl-devel')]
# order is important!
# package versions updated May 28th 2015
diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb
new file mode 100644
index 0000000000..b9f408e3ef
--- /dev/null
+++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb
@@ -0,0 +1,23 @@
+easyblock = 'ConfigureMake'
+
+name = 'Python'
+version = '2.7.15'
+
+homepage = 'http://python.org/'
+description = """Python is a programming language that lets you work more quickly and integrate your systems
+more effectively."""
+
+toolchain = SYSTEM
+
+source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/']
+sources = [SOURCE_TGZ]
+
+# This just serves to have a Python as a dependency to test e.g. the Python version templates
+# So all dependencies and extensions are removed
+dependencies = []
+
+osdependencies = []
+
+exts_list = []
+
+moduleclass = 'lang'
diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb
new file mode 100644
index 0000000000..a61cabbc32
--- /dev/null
+++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb
@@ -0,0 +1,23 @@
+easyblock = 'ConfigureMake'
+
+name = 'Python'
+version = '3.7.2'
+
+homepage = 'http://python.org/'
+description = """Python is a programming language that lets you work more quickly and integrate your systems
+more effectively."""
+
+toolchain = SYSTEM
+
+source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/']
+sources = [SOURCE_TGZ]
+
+# This just serves to have a Python as a dependency to test e.g. the Python version templates
+# So all dependencies and extensions are removed
+dependencies = []
+
+osdependencies = []
+
+exts_list = []
+
+moduleclass = 'lang'
diff --git a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb
index c272f8dbbd..c6df4d42c6 100644
--- a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb
+++ b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb
@@ -31,7 +31,7 @@ dependencies = [
# ('Tcl', '8.6.4'),
]
-parallel = 1
+maxparallel = 1
sanity_check_paths = {
'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],
diff --git a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-foss-2018a.eb
index 593ca3018a..937553eaf3 100644
--- a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-foss-2018a.eb
+++ b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-foss-2018a.eb
@@ -31,7 +31,7 @@ dependencies = [
# ('Tcl', '8.6.4'),
]
-parallel = 1
+maxparallel = 1
sanity_check_paths = {
'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],
diff --git a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-2018a.eb b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-2018a.eb
index 5463f54352..b0be557296 100644
--- a/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-2018a.eb
+++ b/test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-2018a.eb
@@ -31,7 +31,7 @@ dependencies = [
# ('Tcl', '8.6.4'),
]
-parallel = 1
+maxparallel = 1
sanity_check_paths = {
'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],
diff --git a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20-broken.eb b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20-broken.eb
index 6b3ce0d492..e85f325169 100644
--- a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20-broken.eb
+++ b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20-broken.eb
@@ -23,6 +23,6 @@ builddependencies = [('CMake','2.8.10')]
dependencies = [(local_blaslib, local_blasver, '')]
# parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
moduleclass = 'numlib'
diff --git a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb
index bb6a33480d..4a2c2bcaf2 100644
--- a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb
+++ b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb
@@ -22,6 +22,6 @@ versionsuffix = "-%s-%s" % (local_blaslib, local_blasver)
dependencies = [(local_blaslib, local_blasver, '')]
# parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
moduleclass = 'numlib'
diff --git a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2018a-OpenBLAS-0.2.20.eb b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2018a-OpenBLAS-0.2.20.eb
index d8c8855873..b3db5dbbf0 100644
--- a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2018a-OpenBLAS-0.2.20.eb
+++ b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2018a-OpenBLAS-0.2.20.eb
@@ -22,6 +22,6 @@ versionsuffix = "-%s-%s" % (local_blaslib, local_blasver)
dependencies = [(local_blaslib, local_blasver)]
# parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
moduleclass = 'numlib'
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
index 4ae349d0a0..1e3f34a777 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
@@ -9,13 +9,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
index ced8241a45..8d5b7b7f99 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
@@ -10,13 +10,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
{SOURCE_TAR_GZ: '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
'bar.tgz': '33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f'},
]]
@@ -26,10 +23,14 @@ exts_default_options = {
'source_urls': ['http://example.com/%(name)s'],
}
+local_bar_buildopts = " && gcc bar.c -o anotherbar && "
+# used to check whether $TOY_LIBS_PATH is defined even when 'lib' subdirectory doesn't exist yet
+local_bar_buildopts += 'echo "TOY_EXAMPLES=$TOY_EXAMPLES" > %(installdir)s/toy_libs_path.txt'
+
exts_list = [
'ulimit', # extension that is part of "standard library"
('bar', '0.0', {
- 'buildopts': " && gcc bar.c -o anotherbar",
+ 'buildopts': local_bar_buildopts,
'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'], # checksum for
# custom extension filter to verify use of stdin value being passed to filter command
'exts_filter': ("cat | grep '^bar$'", '%(name)s'),
@@ -41,7 +42,7 @@ exts_list = [
'unknowneasyconfigparameterthatshouldbeignored': 'foo',
# set boolean value (different from default value) to trigger (now fixed) bug with --inject-checksums
# cfr. https://github.com/easybuilders/easybuild-framework/pull/3034
- 'keepsymlinks': True,
+ 'keepsymlinks': False,
}),
('barbar', '1.2', {
'start_dir': 'src',
@@ -58,6 +59,8 @@ sanity_check_paths = {
'dirs': [],
}
+modextrapaths = {'TOY_EXAMPLES': 'examples'}
+
postinstallcmds = ["echo TOY > %(installdir)s/README"]
moduleclass = 'tools'
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
index 925432d02a..c8d0504764 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
@@ -9,13 +9,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
index 68ece2259f..e02e21f7ae 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
@@ -11,8 +11,8 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
checksums = [
- ('adler32', '0x998410035'),
- 'a99f2a72cee1689a2f7e3ace0356efb1',
+ '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
+ '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',
]
moduleclass = 'tools'
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
index 90cc7429d3..286cdf7f6c 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
@@ -9,13 +9,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
index c2a88616b1..0e087cec7c 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
@@ -8,13 +8,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/v2.0/toy-with-sections.eb b/test/framework/easyconfigs/v2.0/toy-with-sections.eb
index 34b9af0dcd..a1a508bcf0 100644
--- a/test/framework/easyconfigs/v2.0/toy-with-sections.eb
+++ b/test/framework/easyconfigs/v2.0/toy-with-sections.eb
@@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License
sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0
checksums = [
- 'be662daa971a640e40be5c804d9d7d10', # MD5
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256
]
diff --git a/test/framework/easyconfigs/v2.0/toy.eb b/test/framework/easyconfigs/v2.0/toy.eb
index a1cdfaf6d8..3c5a000f1b 100644
--- a/test/framework/easyconfigs/v2.0/toy.eb
+++ b/test/framework/easyconfigs/v2.0/toy.eb
@@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License
sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0
checksums = [
- 'be662daa971a640e40be5c804d9d7d10', # MD5
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256
]
diff --git a/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb b/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb
deleted file mode 100644
index 7244848a78..0000000000
--- a/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb
+++ /dev/null
@@ -1,18 +0,0 @@
-easyblock: Toolchain
-
-name: CrayCCE
-version: 5.1.29
-
-homepage: (none)
-description: Toolchain using Cray compiler wrapper, using PrgEnv-cray module.
-
-toolchain: system, system
-
-dependencies:
- # also loads cray-libsci
- - name: PrgEnv-cray/5.1.29
- external_module: true
- - name: fftw/3.3.4.0
- external_module: true
-
-moduleclass: toolchain
diff --git a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb b/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb
deleted file mode 100644
index 527f7bf8a5..0000000000
--- a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb
+++ /dev/null
@@ -1,131 +0,0 @@
-_internal_variables_:
- - &numpyversion 1.9.2
- - &scipyversion 0.15.1
-
-easyblock: ConfigureMake
-
-name: Python
-version: 2.7.10
-
-homepage: http://python.org/
-description: |
- Python is a programming language that lets you work more quickly and integrate your systems
- more effectively.
-
-toolchain: {name: intel, version: 2018a}
-toolchainopts: {pic: True, opt: True, optarch: True}
-
-source_urls: ['http://www.python.org/ftp/python/%(version)s/']
-sources: [*SOURCE_TGZ]
-
-# python needs bzip2 to build the bz2 package
-# commented out for testing to avoid having to add them all - dependencies are tested in other files
-dependencies: [
-# [bzip2, 1.0.6],
-# [zlib, 1.2.8],
-# [libreadline, '6.3'],
-# [ncurses, '5.9'],
-# [SQLite, 3.8.10.2],
-# [Tk, 8.6.4, -no-X11],
-# [OpenSSL, 1.0.1m], # OS dependency should be preferred if the os version is more recent then this version, its
-# nice to have an up to date openssl for security reasons
-]
-
-# zlib is only included here for the sake of testing parsing of .yeb easyconfigs!
-osdependencies: [zlib, [openssl-devel, libssl-dev, libopenssl-devel]]
-
-# order is important!
-# package versions updated May 28th 2015
-exts_list: [
- [setuptools, '16.0', {
- source_urls: ["https://pypi.python.org/packages/source/s/setuptools/"],
- }],
- [pip, 7.0.1, {
- source_urls: ["https://pypi.python.org/packages/source/p/pip/"],
- }],
- [nose, 1.3.6, {
- source_urls: ["https://pypi.python.org/packages/source/n/nose/"],
- }],
- [numpy, *numpyversion, {
- source_urls: [
- [!join ["http://sourceforge.net/projects/numpy/files/NumPy/", *numpyversion], download]
- ],
- patches: [
- numpy-1.8.0-mkl.patch, # % numpyversion,
- ],
- }],
- [scipy, *scipyversion, {
- source_urls: [
- [!join ["http://sourceforge.net/projects/scipy/files/scipy/", *scipyversion], download]],
- }],
- [blist, 1.3.6, {
- source_urls: ["https://pypi.python.org/packages/source/b/blist/"],
- }],
- [mpi4py, 1.3.1, {
- source_urls: ["http://bitbucket.org/mpi4py/mpi4py/downloads/"],
- }],
- [paycheck, 1.0.2, {
- source_urls: ["https://pypi.python.org/packages/source/p/paycheck/"],
- }],
- [argparse, 1.3.0, {
- source_urls: ["https://pypi.python.org/packages/source/a/argparse/"],
- }],
- [pbr, 1.0.1, {
- source_urls: ["https://pypi.python.org/packages/source/p/pbr/"],
- }],
- [lockfile, 0.10.2, {
- source_urls: ["https://pypi.python.org/packages/source/l/lockfile/"],
- }],
- [Cython, '0.22', {
- source_urls: ["http://www.cython.org/release/"],
- }],
- [six, 1.9.0, {
- source_urls: ["https://pypi.python.org/packages/source/s/six/"],
- }],
- [dateutil, 2.4.2, {
- source_tmpl: python-%(name)s-%(version)s.tar.gz,
- source_urls: ["https://pypi.python.org/packages/source/p/python-dateutil/"],
- }],
- [deap, 1.0.2, {
- source_tmpl: "%(name)s-%(version)s.post2.tar.gz",
- source_urls: ["https://pypi.python.org/packages/source/d/deap/"],
- }],
- [decorator, 3.4.2, {
- source_urls: ["https://pypi.python.org/packages/source/d/decorator/"],
- }],
- [arff, 2.0.2, {
- source_tmpl: liac-%(name)s-%(version)s.zip,
- source_urls: ["https://pypi.python.org/packages/source/l/liac-arff/"],
- }],
- [pycrypto, 2.6.1, {
- modulename: Crypto,
- source_urls: ["http://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/"],
- }],
- [ecdsa, '0.13', {
- source_urls: ["https://pypi.python.org/packages/source/e/ecdsa/"],
- }],
- [paramiko, 1.15.2, {
- source_urls: ["https://pypi.python.org/packages/source/p/paramiko/"],
- }],
- [pyparsing, 2.0.3, {
- source_urls: ["https://pypi.python.org/packages/source/p/pyparsing/"],
- }],
- [netifaces, 0.10.4, {
- source_urls: ["https://pypi.python.org/packages/source/n/netifaces"],
- }],
- [netaddr, 0.7.14, {
- source_urls: ["https://pypi.python.org/packages/source/n/netaddr"],
- }],
- [mock, 1.0.1, {
- source_urls: ["https://pypi.python.org/packages/source/m/mock"],
- }],
- [pytz, '2015.4', {
- source_urls: ["https://pypi.python.org/packages/source/p/pytz"],
- }],
- [pandas, 0.16.1, {
- source_urls: ["https://pypi.python.org/packages/source/p/pandas"],
- }],
-]
-
-moduleclass: lang
-
diff --git a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb b/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb
deleted file mode 100644
index a9d0afea39..0000000000
--- a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb
+++ /dev/null
@@ -1,32 +0,0 @@
-_internal_variables_:
- - &versionstr 3081002
-
-easyblock: ConfigureMake
-
-name: SQLite
-version: 3.8.10.2
-
-homepage: http://www.sqlite.org/
-# quotes on description to escape special character :
-description: "SQLite: SQL Database Engine in a C Library"
-
-toolchain: foss, 2018a
-
-# eg. http://www.sqlite.org/2014/sqlite-autoconf-3080600.tar.gz
-source_urls: ["http://www.sqlite.org/2015/"]
-sources: [!join [sqlite-autoconf-, *versionstr, .tar.gz]]
-
-# commented out for testing to avoid having to add them all - dependencies are tested in other files
-dependencies: [
-# [libreadline, '6.3'],
-# [Tcl, 8.6.4],
-]
-
-parallel: 1
-
-sanity_check_paths: {
- files: [bin/sqlite3, include/sqlite3ext.h, include/sqlite3.h, lib/libsqlite3.a, lib/libsqlite3.so],
- dirs: [lib/pkgconfig],
-}
-
-moduleclass: devel
diff --git a/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb b/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb
deleted file mode 100644
index f9c0f7f074..0000000000
--- a/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb
+++ /dev/null
@@ -1,26 +0,0 @@
-%YAML 1.2
----
-# not really (there's an EB_bzip2 easyblock), but fine for use in unit tests
-easyblock: ConfigureMake
-
-name: bzip2
-version: 1.0.6
-
-homepage: 'http://www.bzip.org/'
-# preserve newlines using '|' to ensure match with description from .eb easyconfig
-description: |
- bzip2 is a freely available, patent free, high-quality data compressor. It typically
- compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical
- compressors), whilst being around twice as fast at compression and six times faster at decompression.
-
-# bad toolchain with four parameters
-toolchain: GCC, 4.9, False, 2
-toolchainopts: {pic: True}
-
-sources:
- # SOURCE_TAR_GZ is a known constant, hence the *
- - *SOURCE_TAR_GZ
-source_urls:
- - http://www.bzip.org/%(version)s
-
-moduleclass: tools
diff --git a/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb b/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb
deleted file mode 100644
index a2fd43d756..0000000000
--- a/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb
+++ /dev/null
@@ -1,28 +0,0 @@
-%YAML 1.2
----
-# not really (there's an EB_bzip2 easyblock), but fine for use in unit tests
-easyblock: ConfigureMake
-
-name: bzip2
-version: 1.0.6
-
-homepage: 'http://www.bzip.org/'
-# preserve newlines using '|' to ensure match with description from .eb easyconfig
-description: |
- bzip2 is a freely available, patent free, high-quality data compressor. It typically
- compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical
- compressors), whilst being around twice as fast at compression and six times faster at decompression.
-
-toolchain: [GCC, 4.9.2]
-toolchainopts: {pic: True}
-
-sources:
- # SOURCE_TAR_GZ is a known constant, hence the *
- - *SOURCE_TAR_GZ
-source_urls:
- - http://www.bzip.org/%(version)s
-
-builddependencies:
- - gzip: 1.6
-
-moduleclass: tools
diff --git a/test/framework/easyconfigs/yeb/foss-2018a.yeb b/test/framework/easyconfigs/yeb/foss-2018a.yeb
deleted file mode 100644
index 99de832ac9..0000000000
--- a/test/framework/easyconfigs/yeb/foss-2018a.yeb
+++ /dev/null
@@ -1,42 +0,0 @@
-_internal_variables_:
- - &version 2018a
-
- - &comp_name GCC
- - &comp_version 6.4.0-2.28
- - &comp [*comp_name, *comp_version]
-
- - &blaslib OpenBLAS
- - &blasver 0.2.20
- - &blas !join [*blaslib, -, *blasver]
-
- - &comp_mpi_tc [gompi, *version]
-
-
-easyblock: Toolchain
-
-name: foss
-version: *version
-
-homepage: (none)
-description: |
- GNU Compiler Collection (GCC) based compiler toolchain, including
- OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK.
-
-toolchain: {name: system, version: system}
-
-# compiler toolchain dependencies
-# we need GCC and OpenMPI as explicit dependencies instead of gompi toolchain
-# because of toolchain preperation functions
-dependencies:
- - *comp_name: *comp_version
- - OpenMPI: 2.1.2
- toolchain: *comp
- - *blaslib: *blasver
- toolchain: *comp
- - FFTW: 3.3.7
- toolchain: *comp_mpi_tc
- - ScaLAPACK: 2.0.2
- versionsuffix: !join [-, *blas]
- toolchain: *comp_mpi_tc
-
-moduleclass: toolchain
diff --git a/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb b/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb
deleted file mode 100644
index da64d1518d..0000000000
--- a/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb
+++ /dev/null
@@ -1,25 +0,0 @@
-%YAML 1.2
----
-easyblock: ConfigureMake
-
-name: gzip
-version: 1.6
-
-homepage: 'http://www.gnu.org/software/gzip/'
-description:
- gzip (GNU zip) is a popular data compression program
- as a replacement for compress
-
-toolchain: {name: GCC, version: 4.9.2}
-
-# http://ftp.gnu.org/gnu/gzip/gzip-1.6.tar.gz
-source_urls: [*GNU_SOURCE]
-sources: ['%(name)s-%(version)s.tar.gz']
-
-# make sure the gzip, gunzip and compress binaries are available after installation
-sanity_check_paths: {
- files: [bin/gunzip, bin/gzip, bin/uncompress],
- dirs: [],
-}
-
-moduleclass: tools
diff --git a/test/framework/easyconfigs/yeb/intel-2018a.yeb b/test/framework/easyconfigs/yeb/intel-2018a.yeb
deleted file mode 100644
index c1ec0daf29..0000000000
--- a/test/framework/easyconfigs/yeb/intel-2018a.yeb
+++ /dev/null
@@ -1,24 +0,0 @@
-_internal_variables_:
- - &compver 2018.1.163
-
-easyblock: Toolchain
-
-name: intel
-version: 2018a
-
-homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/
-description:
- Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers,
- Intel MPI & Intel MKL.
-
-toolchain: {name: system, version: system}
-
-# fake intel toolchain easyconfig, no dependencies (good enough for testing)
-dependencies: [
- # [icc, *compver],
- # [ifort, *compver],
- # [impi, 2018.1.163],
- # [imkl, 2018.1.163]
-]
-
-moduleclass: toolchain
diff --git a/test/framework/easyconfigs/yeb/toy-0.0.yeb b/test/framework/easyconfigs/yeb/toy-0.0.yeb
deleted file mode 100644
index 2f5265ea6d..0000000000
--- a/test/framework/easyconfigs/yeb/toy-0.0.yeb
+++ /dev/null
@@ -1,38 +0,0 @@
-%YAML 1.2
----
-
-name: toy
-version: 0.0
-
-homepage: 'https://easybuilders.github.io/easybuild'
-description: "Toy C program, 100% toy."
-
-toolchain: system, system
-
-sources:
- - *SOURCE_TAR_GZ
-
-checksums: [[
- 'be662daa971a640e40be5c804d9d7d10', # default [MD5]
- '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ['adler32', '0x998410035'],
- ['crc32', '0x1553842328'],
- ['md5', 'be662daa971a640e40be5c804d9d7d10'],
- ['sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'],
- ['size', 273],
-]]
-
-patches: [
- 'toy-0.0_fix-silly-typo-in-printf-statement.patch',
- ['toy-extra.txt', 'toy-0.0'],
-]
-
-sanity_check_paths: {
- files: [['bin/yot', 'bin/toy']],
- dirs: ['bin'],
-}
-
-postinstallcmds: ["echo TOY > %(installdir)s/README"]
-
-moduleclass: tools
-# trailing comment, leave this here, it may trigger bugs with extract_comments()
diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py
index 4da54b450a..300cf6aff2 100644
--- a/test/framework/easyconfigversion.py
+++ b/test/framework/easyconfigversion.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/easystack.py b/test/framework/easystack.py
index a51059d280..900ed43afd 100644
--- a/test/framework/easystack.py
+++ b/test/framework/easystack.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -155,7 +155,10 @@ def test_easystack_restore_env_after_each_build(self):
'--easystack',
test_es_path
]
+ self.mock_stdout(True)
+ stdout = self.eb_main(args, do_build=True, raise_error=True)
stdout = self.eb_main(args, do_build=True, raise_error=True, reset_env=False, redo_init_config=False)
+ self.mock_stdout(False)
regex = re.compile(r"WARNING Loaded modules detected: \[.*gompi/2018.*\]\n")
self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout))
diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py
index 169fe988a2..1b858c0471 100644
--- a/test/framework/ebconfigobj.py
+++ b/test/framework/ebconfigobj.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -116,6 +116,23 @@ def test_squash_simple(self):
res = cov.squash(version, tc['name'], tc['version'])
self.assertEqual(res, {}) # very simple
+ # Ensure that a version of '0' with trailing '.0's is matched against '0.0' but not anything higher
+ # This is for testing the DEFAULT_UNDEFINED_VERSION detection
+ for num_zeroes in range(1, 6):
+ tc = tc_first
+ zero_version = '.'.join(['0'] * num_zeroes)
+ txt = [
+ '[SUPPORTED]',
+ 'versions = ' + zero_version,
+ 'toolchains = ' + tc_tmpl % tc,
+ '[DEFAULT]',
+ 'y=a',
+ ]
+ co = ConfigObj(txt)
+ cov = EBConfigObj(co)
+ self.assertEqual(cov.squash('0.0', tc['name'], tc['version']), {'y': 'a'})
+ self.assertEqual(cov.squash('0.1', tc['name'], tc['version']), {})
+
def test_squash_invalid(self):
"""Try to squash invalid files. Should trigger error"""
tc_first = {'version': '10', 'name': self.tc_first}
@@ -123,8 +140,8 @@ def test_squash_invalid(self):
tc_tmpl = '%(name)s == %(version)s'
- default_version = '1.0'
- all_wrong_versions = [default_version, '>= 0.0', '< 1.0']
+ default_version = '1.1'
+ all_wrong_versions = [default_version, '>= 0.0', '< 1.1']
# all txt should have default version and first toolchain unmodified
diff --git a/test/framework/environment.py b/test/framework/environment.py
index f184c6b9c7..75919f6a13 100644
--- a/test/framework/environment.py
+++ b/test/framework/environment.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index f6dd6fd7ac..fe37a07820 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -41,18 +41,19 @@
import stat
import sys
import tempfile
+import textwrap
import time
+import types
+from io import StringIO
from test.framework.github import requires_github_access
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
-from easybuild.tools import run
+from urllib import request
import easybuild.tools.filetools as ft
-import easybuild.tools.py2vs3 as py2vs3
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
+from easybuild.tools.config import IGNORE, ERROR, WARN, build_option, update_build_option
from easybuild.tools.multidiff import multidiff
-from easybuild.tools.py2vs3 import StringIO, std_urllib
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import LINUX, get_os_type
@@ -292,10 +293,6 @@ def test_checksums(self):
ft.write_file(fp, "easybuild\n")
known_checksums = {
- 'adler32': '0x379257805',
- 'crc32': '0x1457143216',
- 'md5': '7167b64b1ca062b9674ffef46f9325db',
- 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190',
'sha256': '1c49562c4b404f3120a3fa0926c8d09c99ef80e470f7de03ffdfa14047960ea5',
'sha512': '7610f6ce5e91e56e350d25c917490e4815f7986469fafa41056698aec256733e'
'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4',
@@ -322,15 +319,13 @@ def test_checksums(self):
ft._log.setLevel(old_log_level)
- # default checksum type is MD5
- self.assertEqual(ft.compute_checksum(fp), known_checksums['md5'])
+ # default checksum type is SHA256
+ self.assertEqual(ft.compute_checksum(fp), known_checksums['sha256'])
- # both MD5 and SHA256 checksums can be verified without specifying type
- self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+ # SHA256 checksums can be verified without specifying type
self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256']))
- # providing non-matching MD5 and SHA256 checksums results in failed verification
- self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c'))
+ # providing non-matching SHA256 checksums results in failed verification
self.assertFalse(ft.verify_checksum(fp, '7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db'))
# checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed
@@ -344,25 +339,14 @@ def test_checksums(self):
for checksum_type, checksum in broken_checksums.items():
self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum)
self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum)))
- # md5 is default
- self.assertFalse(ft.compute_checksum(fp) == broken_checksums['md5'])
- self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5']))
+ # sha256 is default
+ self.assertFalse(ft.compute_checksum(fp) == broken_checksums['sha256'])
self.assertFalse(ft.verify_checksum(fp, broken_checksums['sha256']))
# test specify alternative checksums
alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', known_checksums['sha256'])
self.assertTrue(ft.verify_checksum(fp, alt_checksums))
- alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712',
- '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'])
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
- alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712')
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
- alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712')
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
alt_checksums = (known_checksums['sha256'],)
self.assertTrue(ft.verify_checksum(fp, alt_checksums))
@@ -372,6 +356,8 @@ def test_checksums(self):
# Check dictionary
alt_checksums = (known_checksums['sha256'],)
self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']}))
+ # None is accepted
+ self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): None}))
faulty_dict = {'wrong-name': known_checksums['sha256']}
self.assertErrorRegex(EasyBuildError,
"Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*",
@@ -384,16 +370,86 @@ def test_checksums(self):
init_config(build_options=build_options)
self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None)
- self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256']))
# Test dictionary-type checksums
- for checksum in [known_checksums[x] for x in ('md5', 'sha256')]:
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum,
+ fp, {os.path.basename(fp): None})
+ for checksum in [known_checksums[x] for x in ['sha256']]:
dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
self.assertTrue(ft.verify_checksum(fp, dict_checksum))
del dict_checksum[os.path.basename(fp)]
self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
+ def test_deprecated_checksums(self):
+ """Test checksum functionality."""
+
+ fp = os.path.join(self.test_prefix, 'test.txt')
+ ft.write_file(fp, "easybuild\n")
+
+ known_checksums = {
+ 'adler32': '0x379257805',
+ 'crc32': '0x1457143216',
+ 'md5': '7167b64b1ca062b9674ffef46f9325db',
+ 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190',
+ }
+
+ self.allow_deprecated_behaviour()
+ self.mock_stderr(True) # just to capture deprecation warning
+
+ # make sure checksums computation/verification is correct
+ for checksum_type, checksum in known_checksums.items():
+ self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum)
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
+
+ # MD5 checksums can be verified without specifying type
+ self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+
+ # providing non-matching MD5 checksums results in failed verification
+ self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c'))
+
+ # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed
+ # checksum of length other than 32/64 yields an error
+ error_pattern = r"Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)"
+ for checksum in ['tooshort', 'inbetween32and64charactersisnotgoodeither', known_checksums['md5'] + 'foo']:
+ self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum)
+
+ # make sure faulty checksums are reported
+ broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()}
+ for checksum_type, checksum in broken_checksums.items():
+ self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum)
+ self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum)))
+ self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5']))
+
+ # test specify alternative checksums
+ alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712',
+ '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'])
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712')
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712')
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ # check whether missing checksums are enforced
+ build_options = {
+ 'enforce_checksums': True,
+ }
+ init_config(build_options=build_options)
+
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None)
+ self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+
+ # Test dictionary-type checksums
+ for checksum in [known_checksums[x] for x in ['md5']]:
+ dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
+ self.assertTrue(ft.verify_checksum(fp, dict_checksum))
+ del dict_checksum[os.path.basename(fp)]
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
+
+ self.mock_stderr(False)
+
def test_common_path_prefix(self):
"""Test get common path prefix for a list of paths."""
self.assertEqual(ft.det_common_path_prefix(['/foo/bar/foo', '/foo/bar/baz', '/foo/bar/bar']), '/foo/bar')
@@ -420,6 +476,43 @@ def test_normalize_path(self):
self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz')
self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz')
+ def test_is_parent_path(self):
+ """Test is_parent_path"""
+ self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/test0'))
+ self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/test0/test1'))
+ self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar'))
+ self.assertFalse(ft.is_parent_path('/foo/bar/test0', '/foo/bar'))
+ self.assertFalse(ft.is_parent_path('/foo/bar', '/foo/test'))
+
+ # Check that trailing slashes are ignored
+ self.assertTrue(ft.is_parent_path('/foo/bar/', '/foo/bar'))
+ self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/'))
+ self.assertTrue(ft.is_parent_path('/foo/bar/', '/foo/bar/'))
+
+ # Check that is also accepts relative paths
+ self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar/test0'))
+ self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar/test0/test1'))
+ self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar'))
+ self.assertFalse(ft.is_parent_path('foo/bar/test0', 'foo/bar'))
+ self.assertFalse(ft.is_parent_path('foo/bar', 'foo/test'))
+
+ # Check that relative paths are accounted
+ self.assertTrue(ft.is_parent_path('foo/../baz', 'bar/../baz'))
+
+ # Check that symbolic links are accounted
+ ft.mkdir(os.path.join(self.test_prefix, 'base'))
+ ft.mkdir(os.path.join(self.test_prefix, 'base', 'concrete'))
+ ft.symlink(
+ os.path.join(self.test_prefix, 'base', 'concrete'),
+ os.path.join(self.test_prefix, 'base', 'link')
+ )
+ self.assertTrue(
+ ft.is_parent_path(
+ os.path.join(self.test_prefix, 'base', 'link'),
+ os.path.join(self.test_prefix, 'base', 'concrete', 'file')
+ )
+ )
+
def test_det_file_size(self):
"""Test det_file_size function."""
@@ -435,7 +528,7 @@ def test_det_file_size(self):
# also try with actual HTTP header
try:
- fh = std_urllib.urlopen(test_url)
+ fh = request.urlopen(test_url)
self.assertEqual(ft.det_file_size(fh.info()), expected_size)
fh.close()
@@ -447,7 +540,7 @@ def test_det_file_size(self):
res.close()
except ImportError:
pass
- except std_urllib.URLError:
+ except request.URLError:
print("Skipping online test for det_file_size (working offline)")
def test_download_file(self):
@@ -458,29 +551,34 @@ def test_download_file(self):
test_dir = os.path.abspath(os.path.dirname(__file__))
toy_source_dir = os.path.join(test_dir, 'sandbox', 'sources', 'toy')
source_url = 'file://%s/%s' % (toy_source_dir, fn)
- res = ft.download_file(fn, source_url, target_location)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, source_url, target_location)
self.assertEqual(res, target_location, "'download' of local file works")
downloads = glob.glob(target_location + '*')
self.assertEqual(len(downloads), 1)
# non-existing files result in None return value
- self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
# install broken proxy handler for opening local files
# this should make urlopen use this broken proxy for downloading from a file:// URL
- proxy_handler = std_urllib.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
- std_urllib.install_opener(std_urllib.build_opener(proxy_handler))
+ proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
+ request.install_opener(request.build_opener(proxy_handler))
# downloading over a broken proxy results in None return value (failed download)
# this tests whether proxies are taken into account by download_file
- self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.download_file(fn, source_url, target_location), None,
+ "download over broken proxy fails")
# modify existing download so we can verify re-download
ft.write_file(target_location, '')
# restore a working file handler, and retest download of local file
- std_urllib.install_opener(std_urllib.build_opener(std_urllib.FileHandler()))
- res = ft.download_file(fn, source_url, target_location)
+ request.install_opener(request.build_opener(request.FileHandler()))
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, source_url, target_location)
self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")
# existing file was re-downloaded, so a backup should have been created of the existing file
@@ -496,12 +594,21 @@ def test_download_file(self):
target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt')
url = 'https://raw.githubusercontent.com/easybuilders/easybuild-framework/master/README.rst'
try:
- std_urllib.urlopen(url)
- res = ft.download_file(fn, url, target_location)
+ request.urlopen(url)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, url, target_location)
self.assertEqual(res, target_location, "download with specified timeout works")
- except std_urllib.URLError:
+ except request.URLError:
print("Skipping timeout test in test_download_file (working offline)")
+ # check whether disabling trace output works
+ target_location = os.path.join(self.test_prefix, 'test.txt')
+ with self.mocked_stdout_stderr():
+ ft.download_file(fn, source_url, target_location, forced=True, trace=False)
+ stdout = self.get_stdout()
+ self.assertEqual(stdout, '')
+ ft.remove_file(target_location)
+
# also test behaviour of download_file under --dry-run
build_options = {
'extended_dry_run': True,
@@ -520,9 +627,10 @@ def test_download_file(self):
self.assertEqual(path, target_location)
self.assertNotExists(target_location)
- self.assertTrue(re.match("^file written: .*/foo$", txt))
+ self.assertTrue(re.match("file written: .*/foo", txt))
- ft.download_file(fn, source_url, target_location, forced=True)
+ with self.mocked_stdout_stderr():
+ ft.download_file(fn, source_url, target_location, forced=True)
self.assertExists(target_location)
self.assertTrue(os.path.samefile(path, target_location))
@@ -542,7 +650,8 @@ def fake_urllib_open(*args, **kwargs):
# if requests is available, file is downloaded
if ft.HAVE_REQUESTS:
- res = ft.download_file(fn, url, target)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, url, target)
self.assertTrue(res and os.path.exists(res))
self.assertIn("https://easybuild.io", ft.read_file(res))
@@ -558,7 +667,8 @@ def fake_urllib_open(*args, **kwargs):
# if requests is available, file is downloaded
if ft.HAVE_REQUESTS:
- res = ft.download_file(fn, url, target)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, url, target)
self.assertTrue(res and os.path.exists(res))
self.assertIn("https://easybuild.io", ft.read_file(res))
@@ -592,18 +702,22 @@ def fake_urllib_open(url, *args, **kwargs):
target_path = os.path.join(self.test_prefix, fn)
# first try without allowing insecure downloads (default)
- res = ft.download_file(fn, url, target_path)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, url, target_path)
self.assertEqual(res, None)
update_build_option('insecure_download', True)
+ self.mock_stdout(True)
self.mock_stderr(True)
res = ft.download_file(fn, url, target_path)
stderr = self.get_stderr()
+ self.mock_stdout(False)
self.mock_stderr(False)
self.assertIn("WARNING: Not checking server certificates while downloading toy-0.0.eb", stderr)
self.assertExists(res)
- self.assertTrue(ft.read_file(res).startswith("name = 'toy'"))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(ft.read_file(res).startswith("name = 'toy'"))
# also test insecure download via requests fallback
if ft.HAVE_REQUESTS:
@@ -629,14 +743,17 @@ def fake_requests_get(url, *args, **kwargs):
ft.requests.get = fake_requests_get
update_build_option('insecure_download', False)
- res = ft.download_file(fn, url, target_path)
+ with self.mocked_stdout_stderr():
+ res = ft.download_file(fn, url, target_path)
self.assertEqual(res, None)
update_build_option('insecure_download', True)
self.mock_stderr(True)
+ self.mock_stdout(True)
res = ft.download_file(fn, url, target_path)
stderr = self.get_stderr()
self.mock_stderr(False)
+ self.mock_stdout(False)
self.assertIn("WARNING: Not checking server certificates while downloading README.rst", stderr)
self.assertExists(res)
@@ -918,7 +1035,7 @@ def test_is_binary(self):
self.assertTrue(ft.is_binary(b'\00'))
self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere"))
- self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb')))
+ self.assertTrue(ft.is_binary(ft.read_file('/bin/bash', mode='rb')))
def test_det_patched_files(self):
"""Test det_patched_files function."""
@@ -1157,9 +1274,9 @@ def test_multidiff(self):
self.assertTrue(lines[8].startswith(expected))
# no postinstallcmds in toy-0.0-deps.eb
- expected = "29 %s+ postinstallcmds = " % green
+ expected = "26 %s+ postinstallcmds = " % green
self.assertTrue(any(line.startswith(expected) for line in lines))
- expected = "30 %s+%s (1/2) toy-0.0" % (green, endcol)
+ expected = "27 %s+%s (1/2) toy-0.0" % (green, endcol)
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
self.assertEqual(lines[-1], "=====")
@@ -1178,9 +1295,9 @@ def test_multidiff(self):
self.assertTrue(lines[8].startswith(expected))
# no postinstallcmds in toy-0.0-deps.eb
- expected = "29 + postinstallcmds = "
+ expected = "26 + postinstallcmds = "
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
- expected = "30 + (1/2) toy-0.0-"
+ expected = "27 + (1/2) toy-0.0-"
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
self.assertEqual(lines[-1], "=====")
@@ -1436,27 +1553,38 @@ def test_apply_regex_substitutions(self):
# passing empty list of substitions is a no-op
ft.write_file(testfile, testtxt)
- ft.apply_regex_substitutions(testfile, [], on_missing_match=run.IGNORE)
+ ft.apply_regex_substitutions(testfile, [], on_missing_match=IGNORE)
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, testtxt)
# Check handling of on_missing_match
ft.write_file(testfile, testtxt)
regex_subs_no_match = [('Not there', 'Not used')]
- error_pat = 'Nothing found to replace in %s' % testfile
+ error_pat = "Nothing found to replace 'Not there' in %s" % testfile
# Error
self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match,
- on_missing_match=run.ERROR)
+ on_missing_match=ERROR)
+ # First matches, but 2nd not
+ regex_subs_part_match = [regex_subs[0], ('Not there', 'Not used')]
+ self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_part_match,
+ on_missing_match=ERROR, match_all=True)
+ # First matched so OK with match_all
+ ft.apply_regex_substitutions(testfile, regex_subs_part_match,
+ on_missing_match=ERROR, match_all=False)
# Warn
with self.log_to_testlogfile():
- ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN)
+ ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=WARN)
+ logtxt = ft.read_file(self.logfile)
+ self.assertIn('WARNING ' + error_pat, logtxt)
+ with self.log_to_testlogfile():
+ ft.apply_regex_substitutions(testfile, regex_subs_part_match, on_missing_match=WARN, match_all=True)
logtxt = ft.read_file(self.logfile)
self.assertIn('WARNING ' + error_pat, logtxt)
# Ignore
with self.log_to_testlogfile():
- ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE)
+ ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=IGNORE)
logtxt = ft.read_file(self.logfile)
self.assertIn('INFO ' + error_pat, logtxt)
@@ -1465,6 +1593,24 @@ def test_apply_regex_substitutions(self):
path = os.path.join(self.test_prefix, 'nosuchfile.txt')
self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, path, regex_subs)
+ # Replace multi-line strings
+ testtxt = "This si wrong\nBut mkae right\nLeave this!"
+ expected_testtxt = 'This is wrong.\nBut make right\nLeave this!'
+ ft.write_file(testfile, testtxt)
+ repl = ('This si( .*)\n(.*)mkae right$', 'This is wrong.\nBut make right')
+ ft.apply_regex_substitutions(testfile, [repl], backup=False, on_missing_match=ERROR, single_line=False)
+ new_testtxt = ft.read_file(testfile)
+ self.assertEqual(new_testtxt, expected_testtxt)
+ # Supports capture groups
+ ft.write_file(testfile, testtxt)
+ repls = [
+ ('This si( .*)\n(.*)mkae right$', r'This is\1.\n\2make right'),
+ ('Lea(ve)', r'Do \g<0>\1'), # Reference to full match
+ ]
+ ft.apply_regex_substitutions(testfile, repls, backup=False, on_missing_match=ERROR, single_line=False)
+ new_testtxt = ft.read_file(testfile)
+ self.assertEqual(new_testtxt, expected_testtxt.replace('Leave', 'Do Leaveve'))
+
# make sure apply_regex_substitutions can patch files that include UTF-8 characters
testtxt = b"foo \xe2\x80\x93 bar" # This is an UTF-8 "-"
ft.write_file(testfile, testtxt)
@@ -1485,34 +1631,32 @@ def test_apply_regex_substitutions(self):
# also test apply_regex_substitutions with a *list* of paths
# cfr. https://github.com/easybuilders/easybuild-framework/issues/3493
+ # and a compiled regex
test_dir = os.path.join(self.test_prefix, 'test_dir')
test_file1 = os.path.join(test_dir, 'one.txt')
test_file2 = os.path.join(test_dir, 'two.txt')
ft.write_file(test_file1, "Donald is an elephant")
ft.write_file(test_file2, "2 + 2 = 5")
regexs = [
- ('Donald', 'Dumbo'),
+ (re.compile('donald', re.I), 'Dumbo'), # Only matches if this is used as-is
('= 5', '= 4'),
]
ft.apply_regex_substitutions([test_file1, test_file2], regexs)
# also check dry run mode
init_config(build_options={'extended_dry_run': True})
- self.mock_stderr(True)
- self.mock_stdout(True)
- ft.apply_regex_substitutions([test_file1, test_file2], regexs)
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ ft.apply_regex_substitutions([test_file1, test_file2], regexs)
+ stderr, stdout = self.get_stderr(), self.get_stdout()
self.assertFalse(stderr)
- regex = re.compile('\n'.join([
+ regex = '\n'.join([
r"applying regex substitutions to file\(s\): .*/test_dir/one.txt, .*/test_dir/two.txt",
- r" \* regex pattern 'Donald', replacement string 'Dumbo'",
+ r" \* regex pattern 'donald', replacement string 'Dumbo'",
r" \* regex pattern '= 5', replacement string '= 4'",
'',
- ]))
- self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+ ])
+ self.assertTrue(re.search(regex, stdout), "Pattern '%s' should be found in: %s" % (regex, stdout))
def test_find_flexlm_license(self):
"""Test find_flexlm_license function."""
@@ -1612,7 +1756,8 @@ def test_is_alt_pypi_url(self):
def test_pypi_source_urls(self):
"""Test pypi_source_urls() function."""
- res = ft.pypi_source_urls('easybuild')
+ with self.mocked_stdout_stderr():
+ res = ft.pypi_source_urls('easybuild')
eb340_url = 'https://pypi.python.org/packages/'
eb340_url += '93/41/574d01f352671fbc8589a436167e15a7f3e27ac0aa635d208eb29ee8fd4e/'
eb340_url += 'easybuild-3.4.0.tar.gz#sha256=d870b27211f2224aab89bfd3279834ffb89ff00ad849a0dc2bf5cc1691efa9d2'
@@ -1631,7 +1776,8 @@ def test_pypi_source_urls(self):
# check for Python package that has yanked releases,
# see https://github.com/easybuilders/easybuild-framework/issues/3301
- res = ft.pypi_source_urls('ipython')
+ with self.mocked_stdout_stderr():
+ res = ft.pypi_source_urls('ipython')
self.assertTrue(isinstance(res, list) and res)
prefix = 'https://pypi.python.org/packages'
for entry in res:
@@ -1642,21 +1788,25 @@ def test_derive_alt_pypi_url(self):
"""Test derive_alt_pypi_url() function."""
url = 'https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz'
alturl = url.replace('source/e/easybuild', '5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf')
- self.assertEqual(ft.derive_alt_pypi_url(url), alturl)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.derive_alt_pypi_url(url), alturl)
# test case to ensure that '.' characters in filename are escaped using '\.'
# if not, the alternative URL for tornado-4.5b1.tar.gz is found...
url = 'https://pypi.python.org/packages/source/t/tornado/tornado-4.5.1.tar.gz'
alturl = url.replace('source/t/tornado', 'df/42/a180ee540e12e2ec1007ac82a42b09dd92e5461e09c98bf465e98646d187')
- self.assertEqual(ft.derive_alt_pypi_url(url), alturl)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.derive_alt_pypi_url(url), alturl)
# no crash on non-existing version
url = 'https://pypi.python.org/packages/source/e/easybuild/easybuild-0.0.0.tar.gz'
- self.assertEqual(ft.derive_alt_pypi_url(url), None)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.derive_alt_pypi_url(url), None)
# no crash on non-existing package
url = 'https://pypi.python.org/packages/source/n/nosuchpackageonpypiever/nosuchpackageonpypiever-0.0.0.tar.gz'
- self.assertEqual(ft.derive_alt_pypi_url(url), None)
+ with self.mocked_stdout_stderr():
+ self.assertEqual(ft.derive_alt_pypi_url(url), None)
def test_create_patch_info(self):
"""Test create_patch_info function."""
@@ -1675,25 +1825,15 @@ def test_create_patch_info(self):
self.assertEqual(ft.create_patch_info({'name': 'foo.txt', 'copy': 'subdir', 'alt_location': 'alt'}),
{'name': 'foo.txt', 'copy': 'subdir', 'alt_location': 'alt'})
- self.allow_deprecated_behaviour()
- self.mock_stderr(True)
- self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'})
- stderr = self.get_stderr()
- self.mock_stderr(False)
- self.disallow_deprecated_behaviour()
- expected_warning = "Use of patch file with filename that doesn't end with correct extension: foo.txt "
- expected_warning += "(should be any of: .patch, .patch.bz2, .patch.gz, .patch.xz)"
- fail_msg = "Warning '%s' should appear in stderr output: %s" % (expected_warning, stderr)
- self.assertIn(expected_warning, stderr, fail_msg)
-
- # deprecation warning is treated as an error in context of unit test suite
- expected_error = expected_warning.replace('(', '\\(').replace(')', '\\)')
+ expected_error = r"Wrong patch spec \(foo.txt\), extension type should be any of .patch, .patch.bz2, "
+ expected_error += ".patch.gz, .patch.xz."
self.assertErrorRegex(EasyBuildError, expected_error, ft.create_patch_info, 'foo.txt')
# faulty input
error_msg = "Wrong patch spec"
self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None)
self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'copy': 'subdir'})
+ self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.txt'})
self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.txt', 'random': 'key'})
self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info,
{'name': 'foo.txt', 'copy': 'subdir', 'sourcepath': 'subdir'})
@@ -1707,7 +1847,8 @@ def test_apply_patch(self):
""" Test apply_patch """
testdir = os.path.dirname(os.path.abspath(__file__))
toy_tar_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz')
- path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn)
@@ -1729,7 +1870,8 @@ def test_apply_patch(self):
# This patch is dependent on the previous one
toy_patch_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz')
- self.assertTrue(ft.apply_patch(toy_patch_gz, path))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(ft.apply_patch(toy_patch_gz, path))
patched_gz = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source'))
pattern = "I'm a toy, and very very proud of it"
self.assertIn(pattern, patched_gz)
@@ -1761,10 +1903,9 @@ def test_apply_patch(self):
self.assertEqual(ft.read_file(os.path.join(target_dir, 'subdir', 'target.txt')), '123')
# cleanup and re-extract toy source tarball
- ft.remove_dir(self.test_prefix)
- ft.mkdir(self.test_prefix)
ft.change_dir(self.test_prefix)
- path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
# test applying of patch with git
toy_source_path = os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')
@@ -1789,20 +1930,11 @@ def test_apply_patch(self):
self.assertEqual(ft.read_file(new_file_path), "This is a new file\n")
# cleanup & restore
- ft.remove_dir(path)
- path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
self.assertNotIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path))
- # mock stderr to catch deprecation warning caused by setting 'use_git_am'
- self.allow_deprecated_behaviour()
- self.mock_stderr(True)
- ft.apply_patch(toy_patch, self.test_prefix, use_git_am=True)
- stderr = self.get_stderr()
- self.mock_stderr(False)
- self.assertIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path))
- self.assertIn("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'", stderr)
-
def test_copy_file(self):
"""Test copy_file function."""
testdir = os.path.dirname(os.path.abspath(__file__))
@@ -1932,13 +2064,14 @@ def test_copy_file_xattr(self):
cmd = "xattr -w foo bar %s" % special_file
if cmd:
- (_, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
# need to make file read-only after setting extended attribute
ft.adjust_permissions(special_file, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False)
# only proceed if setting extended attribute worked
- if ec == 0:
+ if res.exit_code == 0:
target = os.path.join(self.test_prefix, 'copy.txt')
ft.copy_file(special_file, target)
self.assertTrue(os.path.exists(target))
@@ -1952,9 +2085,10 @@ def test_copy_file_xattr(self):
cmd = "attr -g foo %s" % target
else:
cmd = "xattr -l %s" % target
- (out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False)
- self.assertEqual(ec, 0)
- self.assertTrue(out.endswith('\nbar\n'))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.endswith('\nbar\n'))
def test_copy_files(self):
"""Test copy_files function."""
@@ -2283,6 +2417,19 @@ def test_copy(self):
self.assertTrue(os.path.isfile(os.path.join(self.test_prefix, 'GCC-4.6.3.eb')))
self.assertEqual(txt, '')
+ def test_get_cwd(self):
+ """Test get_cwd"""
+ toy_dir = os.path.join(self.test_prefix, "test_get_cwd_dir")
+ os.mkdir(toy_dir)
+ os.chdir(toy_dir)
+
+ self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir))
+
+ os.rmdir(toy_dir)
+ self.assertErrorRegex(EasyBuildError, ft.CWD_NOTFOUND_ERROR, ft.get_cwd)
+
+ self.assertEqual(ft.get_cwd(must_exist=False), None)
+
def test_change_dir(self):
"""Test change_dir"""
@@ -2316,7 +2463,8 @@ def test_extract_file(self):
toy_tarball = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz')
self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))
- path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=False)
self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))
self.assertTrue(os.path.samefile(path, self.test_prefix))
# still in same directory as before if change_into_dir is set to False
@@ -2326,7 +2474,8 @@ def test_extract_file(self):
toy_tarball_renamed = os.path.join(self.test_prefix, 'toy_tarball')
shutil.copyfile(toy_tarball, toy_tarball_renamed)
- path = ft.extract_file(toy_tarball_renamed, self.test_prefix, cmd="tar xfvz %s", change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tarball_renamed, self.test_prefix, cmd="tar xfvz %s", change_into_dir=False)
self.assertTrue(os.path.samefile(os.getcwd(), cwd))
self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))
self.assertTrue(os.path.samefile(path, self.test_prefix))
@@ -2347,9 +2496,10 @@ def test_extract_file(self):
self.assertTrue(os.path.samefile(path, self.test_prefix))
self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0'))
- self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt))
+ self.assertTrue(re.search('running shell command "tar xzf .*/toy-0.0.tar.gz"', txt))
- path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False)
self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))
self.assertTrue(os.path.samefile(path, self.test_prefix))
self.assertTrue(os.path.samefile(os.getcwd(), cwd))
@@ -2359,37 +2509,26 @@ def test_extract_file(self):
ft.remove_dir(os.path.join(self.test_prefix, 'toy-0.0'))
- # a deprecation warning is printed (which is an error in this context)
- # if the 'change_into_dir' named argument was left unspecified
- error_pattern = "extract_file function was called without specifying value for change_into_dir"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_file, toy_tarball, self.test_prefix)
- self.allow_deprecated_behaviour()
-
- # make sure we're not in self.test_prefix now (checks below assumes so)
- self.assertFalse(os.path.samefile(os.getcwd(), self.test_prefix))
-
- # by default, extract_file changes to directory in which source file was unpacked
- self.mock_stderr(True)
- path = ft.extract_file(toy_tarball, self.test_prefix)
- stderr = self.get_stderr().strip()
- self.mock_stderr(False)
- self.assertTrue(os.path.samefile(path, self.test_prefix))
- self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix))
- regex = re.compile("^WARNING: .*extract_file function was called without specifying value for change_into_dir")
- self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
-
ft.change_dir(cwd)
self.assertFalse(os.path.samefile(os.getcwd(), self.test_prefix))
- # no deprecation warning when change_into_dir is set to True
- self.mock_stderr(True)
- path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True)
- stderr = self.get_stderr().strip()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
self.assertTrue(os.path.samefile(path, self.test_prefix))
self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix))
self.assertFalse(stderr)
+ self.assertTrue("running shell command" in stdout)
+
+ # check whether disabling trace output works
+ with self.mocked_stdout_stderr():
+ path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True, trace=False)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.assertFalse(stderr)
+ self.assertFalse(stdout)
def test_remove(self):
"""Test remove_file, remove_dir and join remove functions."""
@@ -2474,7 +2613,7 @@ def test_index_functions(self):
# test with specified path with and without trailing '/'s
for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
index = ft.create_index(path)
- self.assertEqual(len(index), 92)
+ self.assertEqual(len(index), 94)
expected = [
os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
@@ -2488,12 +2627,13 @@ def test_index_functions(self):
self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json')
# set up some files to create actual index file for
- ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g'))
+ ecs_dir = os.path.join(self.test_prefix, 'easyconfigs')
+ ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir)
# test dump_index function
- index_fp = ft.dump_index(self.test_prefix)
+ index_fp = ft.dump_index(ecs_dir)
self.assertExists(index_fp)
- self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp)))
+ self.assertTrue(os.path.samefile(ecs_dir, os.path.dirname(index_fp)))
datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+"
expected_header = [
@@ -2501,9 +2641,9 @@ def test_index_functions(self):
"# valid until: " + datestamp_pattern,
]
expected = [
- os.path.join('g', 'gzip', 'gzip-1.4.eb'),
- os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'),
- os.path.join('g', 'gompic', 'gompic-2018a.eb'),
+ os.path.join('gzip', 'gzip-1.4.eb'),
+ os.path.join('GCC', 'GCC-7.3.0-2.30.eb'),
+ os.path.join('gompic', 'gompic-2018a.eb'),
]
index_txt = ft.read_file(index_fp)
for fn in expected_header + expected:
@@ -2513,28 +2653,28 @@ def test_index_functions(self):
# test load_index function
self.mock_stderr(True)
self.mock_stdout(True)
- index = ft.load_index(self.test_prefix)
+ index = ft.load_index(ecs_dir)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertFalse(stderr)
- regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+ regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
- self.assertEqual(len(index), 26)
+ self.assertEqual(len(index), 25)
for fn in expected:
self.assertIn(fn, index)
# dump_index will not overwrite existing index without force
error_pattern = "File exists, not overwriting it without --force"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix)
+ self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, ecs_dir)
ft.remove_file(index_fp)
# test creating index file that's infinitely valid
- index_fp = ft.dump_index(self.test_prefix, max_age_sec=0)
+ index_fp = ft.dump_index(ecs_dir, max_age_sec=0)
index_txt = ft.read_file(index_fp)
expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+"
for fn in expected_header + expected:
@@ -2543,40 +2683,40 @@ def test_index_functions(self):
self.mock_stderr(True)
self.mock_stdout(True)
- index = ft.load_index(self.test_prefix)
+ index = ft.load_index(ecs_dir)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertFalse(stderr)
- regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+ regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
- self.assertEqual(len(index), 26)
+ self.assertEqual(len(index), 25)
for fn in expected:
self.assertIn(fn, index)
ft.remove_file(index_fp)
# test creating index file that's only valid for a (very) short amount of time
- index_fp = ft.dump_index(self.test_prefix, max_age_sec=1)
+ index_fp = ft.dump_index(ecs_dir, max_age_sec=1)
time.sleep(3)
self.mock_stderr(True)
self.mock_stdout(True)
- index = ft.load_index(self.test_prefix)
+ index = ft.load_index(ecs_dir)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertIsNone(index)
self.assertFalse(stdout)
- regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix)
+ regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % ecs_dir)
self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
# check whether load_index takes into account --ignore-index
init_config(build_options={'ignore_index': True})
- self.assertEqual(ft.load_index(self.test_prefix), None)
+ self.assertEqual(ft.load_index(ecs_dir), None)
def test_search_file(self):
"""Test search_file function."""
@@ -2833,7 +2973,7 @@ def test_diff_files(self):
self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res))
@requires_github_access()
- def test_get_source_tarball_from_git(self):
+ def test_github_get_source_tarball_from_git(self):
"""Test get_source_tarball_from_git function."""
target_dir = os.path.join(self.test_prefix, 'target')
@@ -2848,7 +2988,7 @@ def test_get_source_tarball_from_git(self):
def run_check():
"""Helper function to run get_source_tarball_from_git & check dry run output"""
with self.mocked_stdout_stderr():
- res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config)
+ res = ft.get_source_tarball_from_git('test', target_dir, git_config)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.assertEqual(stderr, '')
@@ -2856,113 +2996,142 @@ def run_check():
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
self.assertEqual(os.path.dirname(res), target_dir)
- self.assertEqual(os.path.basename(res), 'test.tar.gz')
+ self.assertEqual(os.path.basename(res), 'test.tar.xz')
git_config = {
'repo_name': 'testrepository',
'url': 'git@github.com:easybuilders',
'tag': 'tag_for_tests',
}
- git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter
+ string_args = {
+ 'git_repo': 'git@github.com:easybuilders/testrepository.git',
+ 'git_clone_cmd': 'git clone --no-checkout',
+ 'test_prefix': self.test_prefix,
+ }
+
expected = '\n'.join([
- r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r' running shell command "{git_clone_cmd} {git_repo}"',
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout refs/tags/tag_for_tests"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
git_config['clone_into'] = 'test123'
expected = '\n'.join([
- r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"',
+ r' running shell command "{git_clone_cmd} {git_repo} test123"',
r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"',
- r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout refs/tags/tag_for_tests"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='test123')
run_check()
del git_config['clone_into']
git_config['recursive'] = True
expected = '\n'.join([
- r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r' running shell command "{git_clone_cmd} {git_repo}"',
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout refs/tags/tag_for_tests"',
+ r" \(in .*/{repo_name}\)",
+ r' running shell command "git submodule update --init --recursive"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
expected = '\n'.join([
- ' running command "git clone --depth 1 --branch tag_for_tests --recursive'
- + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
+ r' running shell command "{git_clone_cmd} {git_repo}"',
r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout refs/tags/tag_for_tests"',
+ r" \(in .*/{repo_name}\)",
+ r' running shell command "git submodule update --init --recursive -- \':!vcflib\' \':!sdsl-lite\'"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
git_config['extra_config_params'] = [
'submodule."fastahack".active=false',
'submodule."sha1".active=false',
]
+ git_cmd_extra = 'git -c submodule."fastahack".active=false -c submodule."sha1".active=false'
expected = '\n'.join([
- ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false'
- + ' clone --depth 1 --branch tag_for_tests --recursive'
- + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r' running shell command "{git_cmd_extra} clone --no-checkout {git_repo}"',
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "{git_cmd_extra} checkout refs/tags/tag_for_tests"',
+ r" \(in .*/{repo_name}\)",
+ r' running shell command "{git_cmd_extra} submodule update --init --recursive --'
+ + ' \':!vcflib\' \':!sdsl-lite\'"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository', git_cmd_extra=git_cmd_extra)
run_check()
del git_config['recurse_submodules']
del git_config['extra_config_params']
- git_config['keep_git_dir'] = True
- expected = '\n'.join([
- r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- r' running command "tar cfvz .*/target/test.tar.gz testrepository"',
- r" \(in .*/tmp.*\)",
- ]) % git_repo
- run_check()
- del git_config['keep_git_dir']
-
del git_config['tag']
git_config['commit'] = '8456f86'
expected = '\n'.join([
- r' running command "git clone --no-checkout %(git_repo)s"',
+ r' running shell command "git clone --no-checkout {git_repo}"',
r" \(in .*/tmp.*\)",
- r' running command "git checkout 8456f86 && git submodule update --init --recursive"',
- r" \(in testrepository\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout 8456f86"',
+ r" \(in .*/{repo_name}\)",
+ r' running shell command "git submodule update --init --recursive"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
expected = '\n'.join([
- r' running command "git clone --no-checkout %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- ' running command "git checkout 8456f86 && git submodule update --init --recursive'
- + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"',
- r" \(in testrepository\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r' running shell command "git clone --no-checkout {git_repo}"',
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git checkout 8456f86"',
+ r" \(in .*/{repo_name}\)",
+ r' running shell command "git submodule update --init --recursive -- \':!vcflib\' \':!sdsl-lite\'"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
del git_config['recursive']
del git_config['recurse_submodules']
expected = '\n'.join([
- r' running command "git clone --no-checkout %(git_repo)s"',
- r" \(in .*/tmp.*\)",
- r' running command "git checkout 8456f86"',
- r" \(in testrepository\)",
- r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in .*/tmp.*\)",
- ]) % git_repo
+ r' running shell command "git clone --no-checkout {git_repo}"',
+ r" \(in .*\)",
+ r' running shell command "git checkout 8456f86"',
+ r" \(in .*/{repo_name}\)",
+ r"Archiving '.*/{repo_name}' into '{test_prefix}/target/test.tar.xz'...",
+ ]).format(**string_args, repo_name='testrepository')
run_check()
+ # tarball formats that are not reproducible
+ bad_filenames = ['test.tar.gz', 'test.tar.bz2']
+ # tarball formats that are reproducible
+ good_filenames = ['test.tar', 'test.tar.xz']
+ # extensionsless filenames get a default archive compression of XZ
+ noext_filename = ['test']
+ for test_filename in bad_filenames + good_filenames + noext_filename:
+ with self.mocked_stdout_stderr():
+ res = ft.get_source_tarball_from_git(test_filename, target_dir, git_config)
+ stderr = self.get_stderr()
+
+ regex = re.compile("Can not create reproducible archive.*")
+ if test_filename in bad_filenames:
+ self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' found in: {stderr}")
+ else:
+ self.assertFalse(regex.search(stderr), f"Pattern '{regex.pattern}' found in: {stderr}")
+
+ ref_filename = f"{test_filename}.tar.xz" if test_filename in noext_filename else test_filename
+ self.assertTrue(res.endswith(ref_filename))
+
+ # non-tarball formats are not supported
+ with self.mocked_stdout_stderr():
+ self.assertRaises(EasyBuildError, ft.get_source_tarball_from_git, 'test.zip', target_dir, git_config)
+
# Test with real data.
init_config()
git_config = {
@@ -2972,48 +3141,109 @@ def run_check():
}
try:
- res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config)
+ res = ft.get_source_tarball_from_git('test', target_dir, git_config)
# (only) tarball is created in specified target dir
- test_file = os.path.join(target_dir, 'test.tar.gz')
+ test_file = os.path.join(target_dir, 'test.tar.xz')
self.assertEqual(res, test_file)
self.assertTrue(os.path.isfile(test_file))
- test_tar_gzs = [os.path.basename(test_file)]
- self.assertEqual(os.listdir(target_dir), ['test.tar.gz'])
+ test_tar_files = [os.path.basename(test_file)]
+ self.assertEqual(os.listdir(target_dir), ['test.tar.xz'])
# Check that we indeed downloaded the right tag
extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
- extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-branch.txt')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
os.remove(test_file)
# use a tag that clashes with a branch name and make sure this is handled correctly
git_config['tag'] = 'tag_for_tests'
with self.mocked_stdout_stderr():
- res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config)
- stderr = self.get_stderr()
- self.assertIn('Tag tag_for_tests was not downloaded in the first try', stderr)
+ res = ft.get_source_tarball_from_git('test', target_dir, git_config)
self.assertEqual(res, test_file)
self.assertTrue(os.path.isfile(test_file))
# Check that we indeed downloaded the tag and not the branch
extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
- extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
del git_config['tag']
git_config['commit'] = '90366ea'
- res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config)
- test_file = os.path.join(target_dir, 'test2.tar.gz')
+ res = ft.get_source_tarball_from_git('test2', target_dir, git_config)
+ test_file = os.path.join(target_dir, 'test2.tar.xz')
self.assertEqual(res, test_file)
self.assertTrue(os.path.isfile(test_file))
- test_tar_gzs.append(os.path.basename(test_file))
- self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs)
+ test_tar_files.append(os.path.basename(test_file))
+ self.assertCountEqual(sorted(os.listdir(target_dir)), test_tar_files)
+ extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'README.md')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
git_config['keep_git_dir'] = True
- res = ft.get_source_tarball_from_git('test3.tar.gz', target_dir, git_config)
- test_file = os.path.join(target_dir, 'test3.tar.gz')
+ res = ft.get_source_tarball_from_git('test3', target_dir, git_config)
+ test_file = os.path.join(target_dir, 'test3.tar.xz')
+ self.assertEqual(res, test_file)
+ self.assertTrue(os.path.isfile(test_file))
+ test_tar_files.append(os.path.basename(test_file))
+ self.assertCountEqual(sorted(os.listdir(target_dir)), test_tar_files)
+ extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'README.md')))
+ self.assertTrue(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
+
+ del git_config['keep_git_dir']
+ git_config['commit'] = '17a551c'
+ git_config['recursive'] = True
+ res = ft.get_source_tarball_from_git('test_recursive', target_dir, git_config)
+ test_file = os.path.join(target_dir, 'test_recursive.tar.xz')
+ self.assertEqual(res, test_file)
+ self.assertTrue(os.path.isfile(test_file))
+ test_tar_files.append(os.path.basename(test_file))
+ self.assertCountEqual(sorted(os.listdir(target_dir)), test_tar_files)
+ extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'README.md')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
+ self.assertTrue(os.path.isdir(os.path.join(extracted_repo_dir, 'easybuilders.github.io')))
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'easybuilders.github.io', 'index.html')))
+
+ git_config['commit'] = '17a551c'
+ git_config['recurse_submodules'] = ['easybuilders.github.io']
+ res = ft.get_source_tarball_from_git('test_submodules', target_dir, git_config)
+ test_file = os.path.join(target_dir, 'test_submodules.tar.xz')
+ self.assertEqual(res, test_file)
+ self.assertTrue(os.path.isfile(test_file))
+ test_tar_files.append(os.path.basename(test_file))
+ self.assertCountEqual(sorted(os.listdir(target_dir)), test_tar_files)
+ extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'README.md')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
+ self.assertTrue(os.path.isdir(os.path.join(extracted_repo_dir, 'easybuilders.github.io')))
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'easybuilders.github.io', 'index.html')))
+
+ git_config['commit'] = '17a551c'
+ git_config['recurse_submodules'] = ['!easybuilders.github.io']
+ res = ft.get_source_tarball_from_git('test_exclude_submodules', target_dir, git_config)
+ test_file = os.path.join(target_dir, 'test_exclude_submodules.tar.xz')
self.assertEqual(res, test_file)
self.assertTrue(os.path.isfile(test_file))
- test_tar_gzs.append(os.path.basename(test_file))
- self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs)
+ test_tar_files.append(os.path.basename(test_file))
+ self.assertCountEqual(sorted(os.listdir(target_dir)), test_tar_files)
+ extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
+ with self.mocked_stdout_stderr():
+ extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False)
+ self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'README.md')))
+ self.assertFalse(os.path.isdir(os.path.join(extracted_repo_dir, '.git')))
+ self.assertTrue(os.path.isdir(os.path.join(extracted_repo_dir, 'easybuilders.github.io')))
+ self.assertFalse(os.path.isfile(os.path.join(extracted_repo_dir, 'easybuilders.github.io', 'index.html')))
except EasyBuildError as err:
if "Network is down" in str(err):
@@ -3026,7 +3256,7 @@ def run_check():
'url': 'git@github.com:easybuilders',
'tag': 'tag_for_tests',
}
- args = ['test.tar.gz', self.test_prefix, git_config]
+ args = ['test', self.test_prefix, git_config]
for key in ['repo_name', 'url', 'tag']:
orig_value = git_config.pop(key)
@@ -3047,10 +3277,97 @@ def run_check():
self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args)
del git_config['unknown']
- args[0] = 'test.txt'
- error_pattern = "git_config currently only supports filename ending in .tar.gz"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args)
- args[0] = 'test.tar.gz'
+ def test_make_archive(self):
+ """Test for make_archive method"""
+ # create fake directories and files to be archived
+ tmpdir = tempfile.mkdtemp()
+ tardir = os.path.join(tmpdir, "test_archive")
+ os.mkdir(tardir)
+ for path in ('bin', 'lib', 'include'):
+ os.mkdir(os.path.join(tardir, path))
+ ft.write_file(os.path.join(tardir, 'README'), 'Dummy readme')
+ ft.write_file(os.path.join(tardir, 'bin', 'executable'), 'Dummy binary')
+ ft.write_file(os.path.join(tardir, 'lib', 'lib.so'), 'Dummy library')
+ ft.write_file(os.path.join(tardir, 'include', 'header.h'), 'Dummy header')
+
+ # default behaviour
+ unreprod_txz = ft.make_archive(tardir, reproducible=False)
+ unreprod_txz_chksum = ft.compute_checksum(unreprod_txz, checksum_type="sha256")
+ self.assertEqual(unreprod_txz, "test_archive.tar.xz")
+ self.assertExists(unreprod_txz)
+ os.remove(unreprod_txz)
+ reprod_txz = ft.make_archive(tardir, reproducible=True)
+ reprod_txz_chksum = ft.compute_checksum(reprod_txz, checksum_type="sha256")
+ self.assertEqual(reprod_txz, "test_archive.tar.xz")
+ self.assertExists(reprod_txz)
+ os.remove(reprod_txz)
+ # custom filenames
+ custom_txz = ft.make_archive(tardir, archive_file="custom_name", reproducible=True)
+ custom_txz_chksum = ft.compute_checksum(custom_txz, checksum_type="sha256")
+ self.assertEqual(custom_txz, "custom_name.tar.xz")
+ self.assertExists(custom_txz)
+ os.remove(custom_txz)
+ customdir_txz = ft.make_archive(tardir, archive_file="custom_name", archive_dir=tmpdir, reproducible=True)
+ customdir_txz_chksum = ft.compute_checksum(customdir_txz, checksum_type="sha256")
+ self.assertEqual(customdir_txz, os.path.join(tmpdir, "custom_name.tar.xz"))
+ self.assertExists(customdir_txz)
+ os.remove(customdir_txz)
+ # custom .tar
+ reprod_tar = ft.make_archive(tardir, archive_file="custom_name.tar", reproducible=True)
+ reprod_tar_chksum = ft.compute_checksum(reprod_tar, checksum_type="sha256")
+ self.assertEqual(reprod_tar, "custom_name.tar")
+ self.assertExists(reprod_tar)
+ os.remove(reprod_tar)
+ unreprod_tar = ft.make_archive(tardir, archive_file="custom_name.tar", reproducible=False)
+ unreprod_tar_chksum = ft.compute_checksum(unreprod_tar, checksum_type="sha256")
+ self.assertEqual(unreprod_tar, "custom_name.tar")
+ self.assertExists(unreprod_tar)
+ os.remove(unreprod_tar)
+
+ # custom .tar.gz
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ custom_tgz = ft.make_archive(tardir, archive_file="custom_name.tar.gz", reproducible=True)
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+
+ warning_msg = "WARNING: Can not create reproducible archive due to unsupported file compression (gz)"
+ self.assertIn(warning_msg, stderr)
+
+ custom_tgz_chksum = ft.compute_checksum(custom_tgz, checksum_type="sha256")
+ self.assertEqual(custom_tgz, "custom_name.tar.gz")
+ self.assertExists(custom_tgz)
+ os.remove(custom_tgz)
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ custom_tgz = ft.make_archive(tardir, archive_file="custom_name.tar.gz", reproducible=False)
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+
+ self.assertNotIn(warning_msg, stderr)
+
+ custom_tgz_chksum = ft.compute_checksum(custom_tgz, checksum_type="sha256")
+ self.assertEqual(custom_tgz, "custom_name.tar.gz")
+ self.assertExists(custom_tgz)
+ os.remove(custom_tgz)
+
+ self.assertErrorRegex(EasyBuildError, "Unsupported archive format.*", ft.make_archive, tardir, "unknown.ext")
+
+ reference_checksum_txz = "ec0f91a462c2743b19b428f4c177d7109d2ccc018dcdedc12570d9d735d6fb1b"
+ reference_checksum_tar = "6e902e77925ab2faeef8377722434d4482f1fcc74af958c984c3f22509ae5084"
+
+ if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
+ # checksums of tarballs made by EB cannot be reliably checked prior to Python 3.9
+ # due to changes introduced in python/cpython#90021
+ self.assertNotEqual(unreprod_txz_chksum, reference_checksum_txz)
+ self.assertEqual(reprod_txz_chksum, reference_checksum_txz)
+ self.assertEqual(custom_txz_chksum, reference_checksum_txz)
+ self.assertEqual(customdir_txz_chksum, reference_checksum_txz)
+ self.assertNotEqual(unreprod_tar_chksum, reference_checksum_tar)
+ self.assertEqual(reprod_tar_chksum, reference_checksum_tar)
+ self.assertNotEqual(custom_tgz_chksum, reference_checksum_txz)
def test_is_sha256_checksum(self):
"""Test for is_sha256_checksum function."""
@@ -3142,6 +3459,18 @@ def test_is_generic_easyblock(self):
for name in ['EB_bzip2', 'EB_DL_underscore_POLY_underscore_Classic', 'EB_GCC', 'EB_WRF_minus_Fire']:
self.assertFalse(ft.is_generic_easyblock(name))
+ def test_load_source(self):
+ """Test for load_source function."""
+ txt = textwrap.dedent("""
+ def foobar():
+ pass
+ """)
+ fp = os.path.join(self.test_prefix, 'foobar.py')
+ ft.write_file(fp, txt)
+ foobar = ft.load_source('foobar', fp)
+ self.assertTrue(isinstance(foobar, types.ModuleType))
+ self.assertTrue(isinstance(foobar.foobar, types.FunctionType))
+
def test_get_easyblock_class_name(self):
"""Test for get_easyblock_class_name function."""
@@ -3511,16 +3840,29 @@ def test_set_gid_sticky_bits(self):
self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
- def test_compat_makedirs(self):
- """Test compatibility layer for Python3 os.makedirs"""
- name = os.path.join(self.test_prefix, 'folder')
- self.assertNotExists(name)
- py2vs3.makedirs(name)
- self.assertExists(name)
- # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3)
- self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name)
- py2vs3.makedirs(name, exist_ok=True) # No error
- self.assertExists(name)
+ def test_get_first_non_existing_parent_path(self):
+ """Test get_first_non_existing_parent_path function."""
+ root_path = os.path.join(self.test_prefix, 'a')
+ target_path = os.path.join(self.test_prefix, 'a', 'b', 'c')
+ ft.mkdir(root_path, parents=True)
+ first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path)
+ self.assertEqual(first_non_existing_parent, os.path.join(self.test_prefix, 'a', 'b'))
+ ft.remove_dir(root_path)
+
+ # Use a reflexive parent relation
+ root_path = os.path.join(self.test_prefix, 'a', 'b')
+ target_path = os.path.join(self.test_prefix, 'a', 'b', 'c')
+ ft.mkdir(root_path, parents=True)
+ first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path)
+ self.assertEqual(first_non_existing_parent, os.path.join(self.test_prefix, 'a', 'b', 'c'))
+ ft.remove_dir(root_path)
+
+ root_path = os.path.join(self.test_prefix, 'a', 'b', 'c')
+ target_path = os.path.join(self.test_prefix, 'a', 'b', 'c')
+ ft.mkdir(root_path, parents=True)
+ first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path)
+ self.assertEqual(first_non_existing_parent, None)
+ ft.remove_dir(root_path)
def test_create_unused_dir(self):
"""Test create_unused_dir function."""
@@ -3560,6 +3902,234 @@ def test_create_unused_dir(self):
self.assertEqual(path, os.path.join(self.test_prefix, 'file_0'))
self.assertExists(path)
+ def test_create_non_existing_paths(self):
+ """Test create_non_existing_paths function."""
+ test_root = os.path.join(self.test_prefix, 'test_create_non_existing_paths')
+
+ ft.mkdir(test_root)
+ requested_paths = [
+ os.path.join(test_root, 'folder_a'),
+ os.path.join(test_root, 'folder_b'),
+ ]
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, requested_paths)
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Repeat with existing folder(s) should create new ones
+ ft.mkdir(test_root)
+ requested_paths = [
+ os.path.join(test_root, 'folder_a'),
+ os.path.join(test_root, 'folder_b'),
+ ]
+ for path in requested_paths:
+ ft.mkdir(path)
+ for i in range(10):
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, [f'{p}_{i}' for p in requested_paths])
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Add a suffix in both directories if a suffix already exists
+ ft.mkdir(test_root)
+ requested_paths = [
+ os.path.join(test_root, 'existing_a'),
+ os.path.join(test_root, 'existing_b'),
+ ]
+ ft.mkdir(os.path.join(test_root, 'existing_b'))
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, [f'{p}_0' for p in requested_paths])
+ self.assertNotExists(os.path.join(test_root, 'existing_a'))
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Skip suffix if a directory with the suffix already exists
+ ft.mkdir(test_root)
+ existing_suffix = 1
+ requested_paths = [
+ os.path.join(test_root, 'existing_suffix_a'),
+ os.path.join(test_root, 'existing_suffix_b'),
+ ]
+
+ ft.mkdir(os.path.join(test_root, f'existing_suffix_b_{existing_suffix}'))
+
+ def expected_suffix(n_calls_to_create_non_existing_paths):
+ if n_calls_to_create_non_existing_paths == 0:
+ return ""
+ new_suffix = n_calls_to_create_non_existing_paths - 1
+ if n_calls_to_create_non_existing_paths > existing_suffix:
+ new_suffix += 1
+ return f"_{new_suffix}"
+
+ for i in range(3):
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, [p + expected_suffix(i) for p in requested_paths])
+ self.assertAllExist(paths)
+ self.assertNotExists(os.path.join(test_root, f'existing_suffix_a_{existing_suffix}'))
+ self.assertExists(os.path.join(test_root, f'existing_suffix_b_{existing_suffix}'))
+
+ ft.remove_dir(test_root)
+
+ # Support creation of parent directories
+ ft.mkdir(test_root)
+ requested_paths = [os.path.join(test_root, 'parent_folder', 'folder')]
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, requested_paths)
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Not influenced by similar folder
+ ft.mkdir(test_root)
+ requested_paths = [os.path.join(test_root, 'folder_a2')]
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, requested_paths)
+ self.assertAllExist(paths)
+ for i in range(10):
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, [f'{p}_{i}' for p in requested_paths])
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Fail cleanly if passed a readonly folder
+ ft.mkdir(test_root)
+ readonly_dir = os.path.join(test_root, 'ro_folder')
+ ft.mkdir(readonly_dir)
+ old_perms = os.lstat(readonly_dir)[stat.ST_MODE]
+ ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False)
+ requested_path = [os.path.join(readonly_dir, 'new_folder')]
+ try:
+ self.assertErrorRegex(
+ EasyBuildError, "Failed to create directory",
+ ft.create_non_existing_paths, requested_path
+ )
+ finally:
+ ft.adjust_permissions(readonly_dir, old_perms, relative=False)
+ ft.remove_dir(test_root)
+
+ # Fail if the number of attempts to create the directory is exceeded
+ ft.mkdir(test_root)
+ requested_paths = [os.path.join(test_root, 'attempt')]
+ ft.mkdir(os.path.join(test_root, 'attempt'))
+ ft.mkdir(os.path.join(test_root, 'attempt_0'))
+ ft.mkdir(os.path.join(test_root, 'attempt_1'))
+ ft.mkdir(os.path.join(test_root, 'attempt_2'))
+ ft.mkdir(os.path.join(test_root, 'attempt_3'))
+ max_tries = 4
+ self.assertErrorRegex(
+ EasyBuildError,
+ rf"Exceeded maximum number of attempts \({max_tries}\) to generate non-existing paths",
+ ft.create_non_existing_paths,
+ requested_paths, max_tries=max_tries
+ )
+ ft.remove_dir(test_root)
+
+ # Ignore files same as folders. So first just create a file with no contents
+ ft.mkdir(test_root)
+ requested_path = os.path.join(test_root, 'file')
+ ft.write_file(requested_path, '')
+ paths = ft.create_non_existing_paths([requested_path])
+ self.assertEqual(paths, [requested_path + '_0'])
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Deny creation of nested directories
+ requested_paths = [
+ os.path.join(test_root, 'foo/bar'),
+ os.path.join(test_root, 'foo/bar/baz'),
+ ]
+ self.assertErrorRegex(
+ EasyBuildError,
+ "Path '.*/foo/bar' is a parent path of '.*/foo/bar/baz'",
+ ft.create_non_existing_paths,
+ requested_paths
+ )
+ self.assertNotExists(test_root) # Fail early, do not create intermediate directories
+
+ requested_paths = [
+ os.path.join(test_root, 'foo/bar/baz'),
+ os.path.join(test_root, 'foo/bar'),
+ ]
+ self.assertErrorRegex(
+ EasyBuildError,
+ "Path '.*/foo/bar' is a parent path of '.*/foo/bar/baz'",
+ ft.create_non_existing_paths,
+ requested_paths
+ )
+ self.assertNotExists(test_root) # Fail early, do not create intermediate directories
+
+ requested_paths = [
+ os.path.join(test_root, 'foo/bar'),
+ os.path.join(test_root, 'foo/bar'),
+ ]
+ self.assertErrorRegex(
+ EasyBuildError,
+ "Path '.*/foo/bar' is a parent path of '.*/foo/bar'",
+ ft.create_non_existing_paths,
+ requested_paths
+ )
+ self.assertNotExists(test_root) # Fail early, do not create intermediate directories
+
+ # Allow creation of non-nested directories
+ ft.mkdir(test_root)
+ requested_paths = [
+ os.path.join(test_root, 'nested/foo/bar'),
+ os.path.join(test_root, 'nested/foo/baz'),
+ os.path.join(test_root, 'nested/buz'),
+ ]
+ paths = ft.create_non_existing_paths(requested_paths)
+ self.assertEqual(paths, requested_paths)
+ self.assertAllExist(paths)
+ ft.remove_dir(test_root)
+
+ # Test that permissions are set in single directories
+ ft.mkdir(test_root, set_gid=False, sticky=False)
+ init_config(build_options={'set_gid_bit': True, 'sticky_bit': True})
+ requested_path = os.path.join(test_root, 'directory')
+ paths = ft.create_non_existing_paths([requested_path])
+ self.assertEqual(len(paths), 1)
+ dir_perms = os.lstat(paths[0])[stat.ST_MODE]
+ self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+ self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+ init_config(build_options={'set_gid_bit': None, 'sticky_bit': None})
+ ft.remove_dir(test_root)
+
+ # Test that permissions are set correctly across a whole path
+ ft.mkdir(test_root, set_gid=False, sticky=False)
+ init_config(build_options={'set_gid_bit': True, 'sticky_bit': True})
+ requested_path = os.path.join(test_root, 'directory', 'subdirectory')
+ paths = ft.create_non_existing_paths([requested_path])
+ self.assertEqual(len(paths), 1)
+ tested_paths = [
+ os.path.join(test_root, 'directory'),
+ os.path.join(test_root, 'directory', 'subdirectory'),
+ ]
+ for path in tested_paths:
+ dir_perms = os.lstat(path)[stat.ST_MODE]
+ self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID, f"set_gid bit should be set for {path}")
+ self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX, f"sticky bit should be set for {path}")
+ init_config(build_options={'set_gid_bit': None, 'sticky_bit': None})
+ ft.remove_dir(test_root)
+
+ # Test that existing directory permissions are not modified
+ ft.mkdir(test_root)
+ init_config(build_options={'set_gid_bit': True, 'sticky_bit': True})
+ existing_parent = os.path.join(test_root, 'directory')
+ requested_path = os.path.join(existing_parent, 'subdirectory')
+
+ ft.mkdir(existing_parent, set_gid=False, sticky=False)
+ paths = ft.create_non_existing_paths([requested_path])
+ self.assertEqual(len(paths), 1)
+
+ dir_perms = os.lstat(paths[0])[stat.ST_MODE]
+ self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID, f"set_gid bit should be set for {path}")
+ self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX, f"sticky bit should be set for {path}")
+
+ dir_perms = os.lstat(existing_parent)[stat.ST_MODE]
+ self.assertEqual(dir_perms & stat.S_ISGID, 0, f"set_gid bit should not be set for {path}")
+ self.assertEqual(dir_perms & stat.S_ISVTX, 0, f"sticky bit should not be set for {path}")
+ init_config(build_options={'set_gid_bit': None, 'sticky_bit': None})
+ ft.remove_dir(test_root)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py
index e0add188b3..fc028f7299 100644
--- a/test/framework/format_convert.py
+++ b/test/framework/format_convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/general.py b/test/framework/general.py
index 3837cecc87..e3b15129cc 100644
--- a/test/framework/general.py
+++ b/test/framework/general.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/github.py b/test/framework/github.py
index d899ad06a1..ea862afb05 100644
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,9 +36,11 @@
import sys
import textwrap
import unittest
+from string import ascii_letters
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from time import gmtime
from unittest import TextTestRunner
+from urllib.request import HTTPError, URLError
import easybuild.tools.testing
from easybuild.base.rest import RestClient
@@ -53,7 +55,6 @@
from easybuild.tools.github import det_pr_title, fetch_easyconfigs_from_commit, fetch_files_from_commit
from easybuild.tools.github import is_patch_for, pick_default_branch
from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state
-from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters
import easybuild.tools.github as gh
try:
@@ -222,7 +223,8 @@ def test_github_read(self):
return
try:
- fp = self.ghfs.read("a_directory/a_file.txt", api=False)
+ with self.mocked_stdout_stderr():
+ fp = self.ghfs.read("a_directory/a_file.txt", api=False)
self.assertEqual(read_file(fp).strip(), "this is a line of text")
os.remove(fp)
except (IOError, OSError):
@@ -250,25 +252,21 @@ def test_github_add_pr_labels(self):
build_options['pr_target_repo'] = GITHUB_EASYCONFIGS_REPO
init_config(build_options=build_options)
- # PR #11262 includes easyconfigs that use 'dummy' toolchain,
- # so we need to allow triggering deprecated behaviour
- self.allow_deprecated_behaviour()
-
self.mock_stdout(True)
self.mock_stderr(True)
- gh.add_pr_labels(11262)
+ gh.add_pr_labels(22380)
stdout = self.get_stdout()
self.mock_stdout(False)
self.mock_stderr(False)
- self.assertIn("Could not determine any missing labels for PR #11262", stdout)
+ self.assertIn("Could not determine any missing labels for PR #22380", stdout)
self.mock_stdout(True)
self.mock_stderr(True)
- gh.add_pr_labels(8006) # closed, unmerged, unlabeled PR
+ gh.add_pr_labels(22088) # closed, unmerged, unlabeled PR
stdout = self.get_stdout()
self.mock_stdout(False)
self.mock_stderr(False)
- self.assertIn("PR #8006 should be labelled 'update'", stdout)
+ self.assertIn("Could not determine any missing labels for PR #22088", stdout)
def test_github_fetch_pr_data(self):
"""Test fetch_pr_data function."""
@@ -408,19 +406,21 @@ def test_github_fetch_easyblocks_from_pr(self):
'pr_target_account': gh.GITHUB_EB_MAIN,
})
+ # TODO: no 5.x PRs for new easyblocks
# PR with new easyblock plus non-easyblock file
- all_ebs_pr1964 = ['lammps.py']
+ # all_ebs_pr1964 = ['lammps.py']
# PR with changed easyblock
- all_ebs_pr1967 = ['siesta.py']
+ all_ebs_pr3631 = ['root.py']
# PR with more than one easyblock
- all_ebs_pr1949 = ['configuremake.py', 'rpackage.py']
+ all_ebs_pr3596 = ['wps.py', 'wrf.py']
- for pr, all_ebs in [(1964, all_ebs_pr1964), (1967, all_ebs_pr1967), (1949, all_ebs_pr1949)]:
+ for pr, all_ebs in [(3631, all_ebs_pr3631), (3596, all_ebs_pr3596)]:
try:
tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr)
- eb_files = gh.fetch_easyblocks_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
+ with self.mocked_stdout_stderr():
+ eb_files = gh.fetch_easyblocks_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(sorted(all_ebs), sorted([os.path.basename(f) for f in eb_files]))
except URLError as err:
print("Ignoring URLError '%s' in test_fetch_easyblocks_from_pr" % err)
@@ -435,43 +435,30 @@ def test_github_fetch_easyconfigs_from_pr(self):
'pr_target_account': gh.GITHUB_EB_MAIN,
})
- # PR for rename of arrow to Arrow,
- # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007/files
- all_ecs_pr8007 = [
- 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb',
- 'bat-0.3.3-fix-pyspark.patch',
- 'bat-0.3.3-intel-2017b-Python-3.6.3.eb',
+ # PR for XCrySDen,
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227/files
+ all_ecs_pr22227 = [
+ 'bwidget-1.10.1-GCCcore-13.3.0.eb',
+ 'quarto-1.5.57-x64.eb',
+ 'Sabre-2013-09-28-GCC-13.3.0.eb',
+ 'Togl-2.0-GCCcore-13.3.0.eb',
+ 'XCrySDen-1.6.2-foss-2024a.eb',
]
- # PR where also files are patched in test/
- # see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files
- all_ecs_pr6587 = [
- 'WIEN2k-18.1-foss-2018a.eb',
- 'WIEN2k-18.1-gimkl-2017a.eb',
- 'WIEN2k-18.1-intel-2018a.eb',
- 'libxc-4.2.3-foss-2018a.eb',
- 'libxc-4.2.3-gimkl-2017a.eb',
- 'libxc-4.2.3-intel-2018a.eb',
+ # PR where only files are patched in test/
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22061/files
+ all_ecs_pr22061 = [
]
- # PR where files are renamed
- # see https://github.com/easybuilders/easybuild-easyconfigs/pull/7159/files
- all_ecs_pr7159 = [
- 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
- 'OpenFOAM-5.0-20180108-foss-2018a.eb',
- 'OpenFOAM-5.0-20180108-intel-2018a.eb',
- 'OpenFOAM-6-foss-2018b.eb',
- 'OpenFOAM-6-intel-2018a.eb',
- 'OpenFOAM-v1806-foss-2018b.eb',
- 'PETSc-3.9.3-foss-2018a.eb',
- 'SCOTCH-6.0.6-foss-2018a.eb',
- 'SCOTCH-6.0.6-foss-2018b.eb',
- 'SCOTCH-6.0.6-intel-2018a.eb',
- 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
+ # PR where files are unarchived
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19834/files
+ all_ecs_pr19834 = [
+ 'Gblocks-0.91b.eb',
]
- for pr, all_ecs in [(8007, all_ecs_pr8007), (6587, all_ecs_pr6587), (7159, all_ecs_pr7159)]:
+ for pr, all_ecs in [(22227, all_ecs_pr22227), (22061, all_ecs_pr22061), (19834, all_ecs_pr19834)]:
try:
tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr)
- ec_files = gh.fetch_easyconfigs_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
+ with self.mocked_stdout_stderr():
+ ec_files = gh.fetch_easyconfigs_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(sorted(all_ecs), sorted([os.path.basename(f) for f in ec_files]))
except URLError as err:
print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err)
@@ -490,35 +477,30 @@ def test_github_fetch_files_from_pr_cache(self):
gh.fetch_files_from_pr.clear_cache()
self.assertFalse(gh.fetch_files_from_pr._cache)
- pr7159_filenames = [
- 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
- 'OpenFOAM-5.0-20180108-foss-2018a.eb',
- 'OpenFOAM-5.0-20180108-intel-2018a.eb',
- 'OpenFOAM-6-foss-2018b.eb',
- 'OpenFOAM-6-intel-2018a.eb',
- 'OpenFOAM-v1806-foss-2018b.eb',
- 'PETSc-3.9.3-foss-2018a.eb',
- 'SCOTCH-6.0.6-foss-2018a.eb',
- 'SCOTCH-6.0.6-foss-2018b.eb',
- 'SCOTCH-6.0.6-intel-2018a.eb',
- 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
+ pr22227_filenames = [
+ 'bwidget-1.10.1-GCCcore-13.3.0.eb',
+ 'quarto-1.5.57-x64.eb',
+ 'Sabre-2013-09-28-GCC-13.3.0.eb',
+ 'Togl-2.0-GCCcore-13.3.0.eb',
+ 'XCrySDen-1.6.2-foss-2024a.eb',
]
- pr7159_files = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
- self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files))
+ with self.mocked_stdout_stderr():
+ pr22227_files = gh.fetch_easyconfigs_from_pr(22227, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
+ self.assertEqual(sorted(pr22227_filenames), sorted(os.path.basename(f) for f in pr22227_files))
- # check that cache has been populated for PR 7159
+ # check that cache has been populated for PR 22227
self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1)
# github_account value is None (results in using default 'easybuilders')
- cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix)
+ cache_key = (22227, None, 'easybuild-easyconfigs', self.test_prefix)
self.assertIn(cache_key, gh.fetch_files_from_pr._cache.keys())
cache_entry = gh.fetch_files_from_pr._cache[cache_key]
- self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames))
+ self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr22227_filenames))
# same query should return result from cache entry
- res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
- self.assertEqual(res, pr7159_files)
+ res = gh.fetch_easyconfigs_from_pr(22227, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
+ self.assertEqual(res, pr22227_files)
# inject entry in cache and check result of matching query
pr_id = 12345
@@ -615,6 +597,7 @@ def test_github_download_repo(self):
return
cwd = os.getcwd()
+ self.mock_stdout(True)
# default: download tarball for master branch of easybuilders/easybuild-easyconfigs repo
path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
@@ -649,6 +632,7 @@ def test_github_download_repo(self):
self.assertIn('easybuild', os.listdir(repodir))
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertExists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py'))
+ self.mock_stdout(False)
def test_github_download_repo_commit(self):
"""Test downloading repo at specific commit (which does not require any GitHub token)"""
@@ -745,7 +729,8 @@ def test_github_find_easybuild_easyconfig(self):
if self.skip_github_tests:
print("Skipping test_find_easybuild_easyconfig, no GitHub token available?")
return
- path = gh.find_easybuild_easyconfig(github_user=GITHUB_TEST_ACCOUNT)
+ with self.mocked_stdout_stderr():
+ path = gh.find_easybuild_easyconfig(github_user=GITHUB_TEST_ACCOUNT)
expected = os.path.join('e', 'EasyBuild', r'EasyBuild-[1-9]+\.[0-9]+\.[0-9]+\.eb')
regex = re.compile(expected)
self.assertTrue(regex.search(path), "Pattern '%s' found in '%s'" % (regex.pattern, path))
@@ -773,15 +758,17 @@ def test_github_find_patches(self):
reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked')
self.assertTrue(re.search(reg, txt))
+ self.mock_stdout(True)
self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None)
+ self.mock_stdout(False)
- # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present (only with Python 3)
- if sys.version_info[0] >= 3:
- non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch')
- with open(non_utf8_patch, 'wb') as fp:
- fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1'))
+ non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch')
+ with open(non_utf8_patch, 'wb') as fp:
+ fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1'))
- self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None)
+ self.mock_stdout(True)
+ self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None)
+ self.mock_stdout(False)
def test_github_det_commit_status(self):
"""Test det_commit_status function."""
diff --git a/test/framework/hooks.py b/test/framework/hooks.py
index 5e810ac0f9..052df5ffdd 100644
--- a/test/framework/hooks.py
+++ b/test/framework/hooks.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2017-2024 Ghent University
+# Copyright 2017-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -29,6 +29,7 @@
"""
import os
import sys
+import textwrap
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
@@ -79,6 +80,9 @@ def setUp(self):
'',
'def fail_hook(err):',
' print("EasyBuild FAIL: %s" % err)',
+ '',
+ 'def crash_hook(err):',
+ ' print("EasyBuild CRASHED, oh no! => %s" % err)',
])
write_file(self.test_hooks_pymod, test_hooks_pymod_txt)
@@ -97,8 +101,9 @@ def test_load_hooks(self):
hooks = load_hooks(self.test_hooks_pymod)
- self.assertEqual(len(hooks), 8)
+ self.assertEqual(len(hooks), 9)
expected = [
+ 'crash_hook',
'fail_hook',
'parse_hook',
'post_configure_hook',
@@ -140,6 +145,7 @@ def test_find_hook(self):
pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0]
start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0]
pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0]
+ crash_hook = [hooks[k] for k in hooks if k == 'crash_hook'][0]
fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0]
pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0]
@@ -175,6 +181,10 @@ def test_find_hook(self):
self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None)
self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None)
+ self.assertEqual(find_hook('crash', hooks), crash_hook)
+ self.assertEqual(find_hook('crash', hooks, pre_step_hook=True), None)
+ self.assertEqual(find_hook('crash', hooks, post_step_hook=True), None)
+
hook_name = 'build_and_install_loop'
self.assertEqual(find_hook(hook_name, hooks), None)
self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook)
@@ -209,6 +219,7 @@ def run_hooks():
run_hook('single_extension', hooks, post_step_hook=True, args=[None])
run_hook('extensions', hooks, post_step_hook=True, args=[None])
run_hook('fail', hooks, args=[EasyBuildError('oops')])
+ run_hook('crash', hooks, args=[RuntimeError('boom!')])
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
@@ -244,6 +255,8 @@ def run_hooks():
"this is run before installing an extension",
"== Running fail hook...",
"EasyBuild FAIL: 'oops'",
+ "== Running crash hook...",
+ "EasyBuild CRASHED, oh no! => boom!",
]
expected_stdout = '\n'.join(expected_stdout_lines)
@@ -268,22 +281,24 @@ def test_verify_hooks(self):
self.assertEqual(verify_hooks(hooks), None)
test_broken_hooks_pymod = os.path.join(self.test_prefix, 'test_broken_hooks.py')
- test_hooks_txt = '\n'.join([
- '',
- 'def there_is_no_such_hook():',
- ' pass',
- 'def stat_hook(self):',
- ' pass',
- 'def post_source_hook(self):',
- ' pass',
- 'def install_hook(self):',
- ' pass',
- ])
+ test_hooks_txt = textwrap.dedent("""
+ def there_is_no_such_hook():
+ pass
+ def stat_hook(self):
+ pass
+ def post_source_hook(self):
+ pass
+ def post_extract_hook(self):
+ pass
+ def install_hook(self):
+ pass
+ """)
write_file(test_broken_hooks_pymod, test_hooks_txt)
error_msg_pattern = r"Found one or more unknown hooks:\n"
error_msg_pattern += r"\* install_hook \(did you mean 'pre_install_hook', or 'post_install_hook'\?\)\n"
+ error_msg_pattern += r"\* post_source_hook \(did you mean .*'\?\)\n"
error_msg_pattern += r"\* stat_hook \(did you mean 'start_hook'\?\)\n"
error_msg_pattern += r"\* there_is_no_such_hook\n\n"
error_msg_pattern += r"Run 'eb --avail-hooks' to get an overview of known hooks"
diff --git a/test/framework/include.py b/test/framework/include.py
index 09a5925a2f..ef20e682fd 100644
--- a/test/framework/include.py
+++ b/test/framework/include.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/lib.py b/test/framework/lib.py
index 7930e00567..ca95c0ab47 100644
--- a/test/framework/lib.py
+++ b/test/framework/lib.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2018-2024 Ghent University
+# Copyright 2018-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -43,7 +43,7 @@
from easybuild.tools.options import set_up_configuration
from easybuild.tools.filetools import mkdir
from easybuild.tools.modules import modules_tool
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd, run_cmd
class EasyBuildLibTest(TestCase):
@@ -67,31 +67,42 @@ def tearDown(self):
def configure(self):
"""Utility function to set up EasyBuild configuration."""
-
- # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called
- if BuildOptions in BuildOptions._instances:
- del BuildOptions._instances[BuildOptions]
-
- self.assertNotIn(BuildOptions, BuildOptions._instances)
- set_up_configuration(silent=True)
- self.assertIn(BuildOptions, BuildOptions._instances)
+ set_up_configuration(silent=True, reconfigure=True)
def test_run_cmd(self):
"""Test use of run_cmd function in the context of using EasyBuild framework as a library."""
error_pattern = r"Undefined build option: .*"
error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
- self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello")
self.configure()
# run_cmd works fine if set_up_configuration was called first
- (out, ec) = run_cmd("echo hello")
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello")
self.assertEqual(ec, 0)
self.assertEqual(out, 'hello\n')
+ def test_run_shell_cmd(self):
+ """Test use of run_shell_cmd function in the context of using EasyBuild framework as a library."""
+
+ error_pattern = r"Undefined build option: .*"
+ error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, "echo hello")
+
+ self.configure()
+
+ # runworks fine if set_up_configuration was called first
+ self.mock_stdout(True)
+ res = run_shell_cmd("echo hello")
+ self.mock_stdout(False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'hello\n')
+
def test_mkdir(self):
- """Test use of run_cmd function in the context of using EasyBuild framework as a library."""
+ """Test use of mkdir function in the context of using EasyBuild framework as a library."""
test_dir = os.path.join(self.tmpdir, 'test123')
diff --git a/test/framework/license.py b/test/framework/license.py
index 6dc79ac6a4..66e5938ab7 100644
--- a/test/framework/license.py
+++ b/test/framework/license.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,7 +33,6 @@
from unittest import TextTestRunner
from easybuild.framework.easyconfig.licenses import License, LicenseVeryRestrictive, what_licenses
-from easybuild.tools.py2vs3 import string_type
class LicenseTest(EnhancedTestCase):
@@ -62,7 +61,7 @@ def test_licenses(self):
"""Test format of available licenses."""
lics = what_licenses()
for lic in lics:
- self.assertIsInstance(lic, string_type)
+ self.assertIsInstance(lic, str)
self.assertTrue(lic.startswith('License'))
self.assertTrue(issubclass(lics[lic], License))
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index 075cc3adbd..c5fc23fe06 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -266,44 +266,57 @@ def test_load(self):
"""Test load part in generated module file."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- # default: guarded module load (which implies no recursive unloading)
- expected = '\n'.join([
- '',
- "if { ![ is-loaded mod_name ] } {",
- " module load mod_name",
- "}",
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ # default: guarded module load (which implies no recursive unloading)
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded mod_name ] } {",
+ " module load mod_name",
+ "}",
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "module load mod_name",
+ '',
+ ])
self.assertEqual(expected, self.modgen.load_module("mod_name"))
- # with recursive unloading: no if is-loaded guard
- expected = '\n'.join([
- '',
- "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {",
- " module load mod_name",
- "}",
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ # with recursive unloading: no if is-loaded guard
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {",
+ " module load mod_name",
+ "}",
+ '',
+ ])
self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload=True))
init_config(build_options={'recursive_mod_unload': True})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
# Lmod 7.6+ depends-on
+
+ self.allow_deprecated_behaviour()
+
if self.modtool.supports_depends_on:
expected = '\n'.join([
'',
"depends-on mod_name",
'',
])
- self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True))
+ with self.mocked_stdout_stderr():
+ txt = self.modgen.load_module("mod_name", depends_on=True)
+ self.assertEqual(expected, txt)
init_config(build_options={'mod_depends_on': 'True'})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
else:
expected = "depends-on statements in generated module are not supported by modules tool"
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True)
- init_config(build_options={'mod_depends_on': 'True'})
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected,
+ self.modgen.load_module, "mod_name", depends_on=True)
else:
# default: guarded module load (which implies no recursive unloading)
expected = '\n'.join([
@@ -330,20 +343,26 @@ def test_load(self):
self.assertEqual(expected, self.modgen.load_module("mod_name"))
# Lmod 7.6+ depends_on
+
+ self.allow_deprecated_behaviour()
+
if self.modtool.supports_depends_on:
expected = '\n'.join([
'',
'depends_on("mod_name")',
'',
])
- self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True))
+ with self.mocked_stdout_stderr():
+ txt = self.modgen.load_module("mod_name", depends_on=True)
+
+ self.assertEqual(expected, txt)
init_config(build_options={'mod_depends_on': 'True'})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
else:
expected = "depends_on statements in generated module are not supported by modules tool"
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True)
- init_config(build_options={'mod_depends_on': 'True'})
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected,
+ self.modgen.load_module, "mod_name", depends_on=True)
def test_load_multi_deps(self):
"""Test generated load statement when multi_deps is involved."""
@@ -353,13 +372,24 @@ def test_load_multi_deps(self):
res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {",
- " module load Python/3.7.4",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {",
+ " module load Python/3.7.4",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded Python/2.7.16 ] } {",
+ " module load Python",
+ '} else {',
+ " module load Python/3.7.4",
+ '}',
+ '',
+ ])
else: # Lua syntax
expected = '\n'.join([
'',
@@ -371,8 +401,12 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
+
+ self.allow_deprecated_behaviour()
+
# two versions with depends_on
- res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods, depends_on=True)
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods, depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\n'.join([
@@ -396,19 +430,33 @@ def test_load_multi_deps(self):
])
self.assertEqual(expected, res)
+ self.disallow_deprecated_behaviour()
+
# now test with more than two versions...
multi_dep_mods = ['foo/1.2.3', 'foo/2.3.4', 'foo/3.4.5', 'foo/4.5.6']
res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " +
- "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {",
- " module load foo/1.2.3",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " +
+ "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {",
+ " module load foo/1.2.3",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded foo/2.3.4 ] || [ is-loaded foo/3.4.5 ] " +
+ "|| [ is-loaded foo/4.5.6 ] } {",
+ " module load foo",
+ "} else {",
+ " module load foo/1.2.3",
+ '}',
+ '',
+ ])
else: # Lua syntax
expected = '\n'.join([
'',
@@ -421,8 +469,12 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
+
+ self.allow_deprecated_behaviour()
+
# more than two versions, with depends_on
- res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods, depends_on=True)
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods, depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\n'.join([
@@ -448,18 +500,23 @@ def test_load_multi_deps(self):
])
self.assertEqual(expected, res)
+ self.disallow_deprecated_behaviour()
+
# what if we only list a single version?
# see https://github.com/easybuilders/easybuild-framework/issues/3080
res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'])
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded one/1.0 ] } {",
- " module load one/1.0",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded one/1.0 ] } {",
+ " module load one/1.0",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\nmodule load one/1.0\n'
else: # Lua syntax
expected = '\n'.join([
'',
@@ -471,7 +528,11 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
- res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'], depends_on=True)
+
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'], depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\ndepends-on one/1.0\n'
@@ -528,10 +589,9 @@ def test_modulerc(self):
# loading of module with symbolic version works
self.modtool.load(['test/1.2.3'])
- # test/1.2.3.4.5 is actually loaded (rather than test/1.2.3)
+ # test/1.2.3.4.5 is actually loaded
res = self.modtool.list()
- self.assertEqual(len(res), 1)
- self.assertEqual(res[0]['mod_name'], 'test/1.2.3.4.5')
+ self.assertTrue(any(x['mod_name'] == 'test/1.2.3.4.5' for x in res))
# if same symbolic version is added again, nothing changes
self.modgen.modulerc(mod_ver_spec, filepath=modulerc_path)
@@ -650,7 +710,7 @@ def test_swap(self):
# create tiny test Tcl module to make sure that tested modules tools support single-argument swap
# see https://github.com/easybuilders/easybuild-framework/issues/3396;
- # this is known to fail with the ancient Tcl-only implementation of environment modules,
+ # this is known to fail with the ancient Tcl-only implementation of Environment Modules,
# but that's considered to be a non-issue (since this is mostly relevant for Cray systems,
# which are either using EnvironmentModulesC (3.2.10), EnvironmentModules (4.x) or Lmod...
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl and self.modtool.__class__ != EnvironmentModulesTcl:
@@ -721,13 +781,16 @@ def append_paths(*args, **kwargs):
# check for warning that is printed when same path is added multiple times
with self.modgen.start_module_creation():
self.modgen.append_paths('TEST', 'path1')
- self.mock_stderr(True)
- self.modgen.append_paths('TEST', 'path1')
- stderr = self.get_stderr()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ self.modgen.append_paths('TEST', 'path1')
+ stderr = self.get_stderr()
expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
expected_warning += "as they were already added: path1\n\n"
self.assertEqual(stderr, expected_warning)
+ with self.mocked_stdout_stderr():
+ self.modgen.append_paths('TEST', 'path1', warn_exists=False)
+ stderr = self.get_stderr()
+ self.assertEqual(stderr, '')
def test_module_extensions(self):
"""test the extensions() for extensions"""
@@ -821,14 +884,18 @@ def prepend_paths(*args, **kwargs):
# check for warning that is printed when same path is added multiple times
with self.modgen.start_module_creation():
self.modgen.prepend_paths('TEST', 'path1')
- self.mock_stderr(True)
- self.modgen.prepend_paths('TEST', 'path1')
- stderr = self.get_stderr()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ self.modgen.prepend_paths('TEST', 'path1')
+ stderr = self.get_stderr()
expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
expected_warning += "as they were already added: path1\n\n"
self.assertEqual(stderr, expected_warning)
+ with self.mocked_stdout_stderr():
+ self.modgen.prepend_paths('TEST', 'path1', warn_exists=False)
+ stderr = self.get_stderr()
+ self.assertEqual(stderr, '')
+
def test_det_user_modpath(self):
"""Test for generic det_user_modpath method."""
# None by default
@@ -844,7 +911,10 @@ def test_det_user_modpath(self):
init_config(build_options={'suffix_modules_path': ''})
user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules'
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"')
+ if self.modtool.supports_tcl_getenv:
+ self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" [getenv TEST123] "modules"')
+ else:
+ self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"')
else:
self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"')
@@ -902,12 +972,20 @@ def test_getenv_cmd(self):
# otherwise we won't get the output produced by the test module file...
os.environ.pop('LMOD_QUIET', None)
- self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME'))
- self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME'))
+ if self.modtool.supports_tcl_getenv:
+ self.assertEqual('[getenv HOSTNAME]', self.modgen.getenv_cmd('HOSTNAME'))
+ self.assertEqual('[getenv HOME]', self.modgen.getenv_cmd('HOME'))
- expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]'
- getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
- self.assertEqual(getenv_txt, expected)
+ expected = '[getenv TEST "foobar"]'
+ getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
+ self.assertEqual(getenv_txt, expected)
+ else:
+ self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME'))
+ self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME'))
+
+ expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]'
+ getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
+ self.assertEqual(getenv_txt, expected)
write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt)
else:
@@ -1597,6 +1675,28 @@ def test_generated_module_file_swap(self):
# one/1.0 module was swapped for one/1.1
self.assertEqual(loaded_mods[-2]['mod_name'], 'one/1.1')
+ def test_check_group(self):
+ """Test check_group method."""
+ if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
+ if self.modtool.supports_tcl_check_group:
+ expected = '\n'.join([
+ "if { ![ module-info usergroups group_name ] } {",
+ " error \"mesg\"",
+ "}",
+ '',
+ ])
+ self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg"))
+ else:
+ self.assertEqual('', self.modgen.check_group("group_name", error_msg="mesg"))
+ else:
+ expected = '\n'.join([
+ 'if not ( userInGroup("group_name") ) then',
+ ' LmodError("mesg")',
+ 'end',
+ '',
+ ])
+ self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg"))
+
class TclModuleGeneratorTest(ModuleGeneratorTest):
"""Test for module_generator module for Tcl syntax."""
diff --git a/test/framework/modules.py b/test/framework/modules.py
index 8ba4f4ac65..6080366fa6 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -42,7 +42,7 @@
import easybuild.tools.modules as mod
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig.easyconfig import EasyConfig
-from easybuild.tools import StrictVersion
+from easybuild.tools import LooseVersion
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.environment import modify_env
from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir
@@ -50,7 +50,7 @@
from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool
from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
@@ -122,10 +122,12 @@ def test_run_module(self):
error_pattern = "Module command '.*thisdoesnotmakesense' failed with exit code [1-9]"
self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense')
- # we need to use a different error pattern here with EnvironmentModulesC,
- # because a load of a non-existing module doesnt' trigger a non-zero exit code...
- # it will still fail though, just differently
- if isinstance(self.modtool, EnvironmentModulesC):
+ # we need to use a different error pattern here with EnvironmentModulesC and
+ # EnvironmentModules <5.5, because a load of a non-existing module doesnt' trigger a
+ # non-zero exit code. it will still fail though, just differently
+ version = LooseVersion(self.modtool.version)
+ if (isinstance(self.modtool, EnvironmentModulesC)
+ or (isinstance(self.modtool, EnvironmentModules) and version < '5.5')):
error_pattern = "Unable to locate a modulefile for 'nosuchmodule/1.2.3'"
else:
error_pattern = "Module command '.*load nosuchmodule/1.2.3' failed with exit code [1-9]"
@@ -191,6 +193,21 @@ def test_run_module(self):
regex = re.compile(r'^os\.environ\[', re.M)
self.assertFalse(regex.search(out), "Pattern '%s' should not be found in: %s" % (regex.pattern, out))
+ def test_list(self):
+ """
+ Test running 'module list' via ModulesTool instance.
+ """
+ # make very sure no modules are currently loaded
+ self.modtool.run_module('purge', '--force')
+
+ out = self.modtool.list()
+ self.assertEqual(out, [])
+
+ mods = ['GCC/7.3.0-2.30']
+ self.modtool.load(mods)
+ out = self.modtool.list()
+ self.assertEqual([x['mod_name'] for x in out], mods)
+
def test_avail(self):
"""Test if getting a (restricted) list of available modules works."""
self.init_testmods()
@@ -198,10 +215,8 @@ def test_avail(self):
# test modules include 3 GCC modules and one GCCcore module
ms = self.modtool.available('GCC')
expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30']
- # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching
- # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl.,
- # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules
- if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules):
+ # ancient Tcl-only Environment Modules tool does an exact match on module name, others do prefix matching
+ if not isinstance(self.modtool, EnvironmentModulesTcl):
expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0'])
self.assertEqual(ms, expected)
@@ -211,15 +226,16 @@ def test_avail(self):
# all test modules are accounted for
ms = self.modtool.available()
+ version = LooseVersion(self.modtool.version)
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('5.7.5'):
+ if isinstance(self.modtool, Lmod) and version >= '5.7.5' and not version.is_prerelease('5.7.5', ['rc']):
# with recent versions of Lmod, also the hidden modules are included in the output of 'avail'
self.assertEqual(len(ms), TEST_MODULES_COUNT + 3)
self.assertIn('bzip2/.1.0.6', ms)
self.assertIn('toy/.0.0-deps', ms)
self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms)
elif (isinstance(self.modtool, EnvironmentModules)
- and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')):
+ and version >= '4.6.0' and not version.is_prerelease('4.6.0', ['-beta'])):
# bzip2/.1.0.6 is not there, since that's a module file in Lua syntax
self.assertEqual(len(ms), TEST_MODULES_COUNT + 2)
self.assertIn('toy/.0.0-deps', ms)
@@ -299,7 +315,8 @@ def test_exist(self):
avail_mods = self.modtool.available()
self.assertIn('Java/1.8.0_181', avail_mods)
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'):
+ version = LooseVersion(self.modtool.version)
+ if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']):
self.assertIn('Java/1.8', avail_mods)
self.assertIn('Java/site_default', avail_mods)
self.assertIn('JavaAlias', avail_mods)
@@ -326,12 +343,12 @@ def test_exist(self):
easybuild.tools.modules.MODULE_SHOW_CACHE.clear()
self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True])
- # mimic more verbose stderr output produced by old Tmod version,
- # including a warning produced when multiple .modulerc files are being picked up
+ # mimic "module-*" output produced by EnvironmentModulesC or EnvironmentModulesTcl
+ # mimic warning produced by Environment Modules when a symbol is defined multiple times
# see https://github.com/easybuilders/easybuild-framework/issues/3376
ml_show_java18_stderr = '\n'.join([
"module-version Java/1.8.0_181 1.8",
- "WARNING: Duplicate version symbol '1.8' found",
+ "WARNING: Symbolic version 'Java/1.8' already defined",
"module-version Java/1.8.0_181 1.8",
"-------------------------------------------------------------------",
"/modulefiles/lang/Java/1.8.0_181:",
@@ -359,7 +376,7 @@ def test_exist(self):
self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True])
# also check with .modulerc.lua for Lmod 7.8 or newer
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'):
+ if isinstance(self.modtool, Lmod) and version >= '7.8' and not version.is_prerelease('7.8', ['rc']):
shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir)
reset_module_caches()
@@ -391,7 +408,7 @@ def test_exist(self):
self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True])
# Test alias in home directory .modulerc
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'):
+ if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']):
# Required or temporary HOME would be in MODULEPATH already
self.init_testmods()
# Sanity check: Module aliases don't exist yet
@@ -443,7 +460,7 @@ def test_load(self):
# if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last
self.modtool.load(['GCC/6.4.0-2.28'])
- # environment modules v4+ does not reload already loaded modules
+ # Environment Modules v4+ does not reload already loaded modules
if not isinstance(self.modtool, EnvironmentModules):
self.assertTrue(os.environ.get('EBROOTGCC'))
@@ -671,6 +688,7 @@ def test_get_software_root_version_libdir(self):
self.assertEqual(get_software_root(name), root)
self.assertEqual(get_software_version(name), version)
self.assertEqual(get_software_libdir(name), 'lib')
+ self.assertEqual(get_software_libdir(name, full_path=True), os.path.join(root, 'lib'))
os.environ.pop('EBROOT%s' % env_var_name)
os.environ.pop('EBVERSION%s' % env_var_name)
@@ -679,30 +697,39 @@ def test_get_software_root_version_libdir(self):
root = os.path.join(tmpdir, name)
mkdir(os.path.join(root, 'lib64'))
os.environ['EBROOT%s' % env_var_name] = root
+
+ def check_get_software_libdir(expected, **additional_args):
+ self.assertEqual(get_software_libdir(name, **additional_args), expected)
+ if isinstance(expected, list):
+ expected = [os.path.join(root, d) for d in expected]
+ elif expected:
+ expected = os.path.join(root, expected)
+ self.assertEqual(get_software_libdir(name, full_path=True, **additional_args), expected)
+
write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo')
- self.assertEqual(get_software_libdir(name), 'lib')
+ check_get_software_libdir('lib')
remove_file(os.path.join(root, 'lib', 'libfoo.a'))
# also check vice versa with *shared* library in lib64
shlib_ext = get_shared_lib_ext()
write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo')
- self.assertEqual(get_software_libdir(name), 'lib64')
+ check_get_software_libdir('lib64')
remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext))
# check expected result of get_software_libdir with multiple lib subdirs
self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name)
- self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])
+ check_get_software_libdir(only_one=False, expected=['lib', 'lib64'])
# only directories containing files in specified list should be retained
write_file(os.path.join(root, 'lib64', 'foo'), 'foo')
- self.assertEqual(get_software_libdir(name, fs=['foo']), 'lib64')
+ check_get_software_libdir(fs=['foo'], expected='lib64')
# duplicate paths due to symlink get filtered
remove_dir(os.path.join(root, 'lib64'))
symlink(os.path.join(root, 'lib'), os.path.join(root, 'lib64'))
- self.assertEqual(get_software_libdir(name), 'lib')
+ check_get_software_libdir('lib')
# same goes for lib symlinked to lib64
remove_file(os.path.join(root, 'lib64'))
@@ -710,19 +737,20 @@ def test_get_software_root_version_libdir(self):
mkdir(os.path.join(root, 'lib64'))
symlink(os.path.join(root, 'lib64'), os.path.join(root, 'lib'))
# still returns 'lib' because that's the first subdir considered
- self.assertEqual(get_software_libdir(name), 'lib')
+ check_get_software_libdir('lib')
# clean up for previous tests
os.environ.pop('EBROOT%s' % env_var_name)
# if root/version for specified software package can not be found, these functions should return None
- self.assertEqual(get_software_root('foo'), None)
- self.assertEqual(get_software_version('foo'), None)
- self.assertEqual(get_software_libdir('foo'), None)
+ self.assertEqual(get_software_root(name), None)
+ self.assertEqual(get_software_version(name), None)
+ check_get_software_libdir(None)
# if no library subdir is found, get_software_libdir should return None
os.environ['EBROOTFOO'] = tmpdir
self.assertEqual(get_software_libdir('foo'), None)
+ self.assertEqual(get_software_libdir('foo', full_path=True), None)
os.environ.pop('EBROOTFOO')
shutil.rmtree(tmpdir)
@@ -1331,9 +1359,10 @@ def test_module_use_bash(self):
modulepath = os.environ['MODULEPATH']
self.assertIn(modules_dir, modulepath)
- out, _ = run_cmd("bash -c 'echo MODULEPATH: $MODULEPATH'", simple=False)
- self.assertEqual(out.strip(), "MODULEPATH: %s" % modulepath)
- self.assertIn(modules_dir, out)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("bash -c 'echo MODULEPATH: $MODULEPATH'")
+ self.assertEqual(res.output.strip(), f"MODULEPATH: {modulepath}")
+ self.assertIn(modules_dir, res.output)
def test_load_in_hierarchy(self):
"""Test whether loading a module in a module hierarchy results in loading the correct module."""
@@ -1395,7 +1424,7 @@ def test_exit_code_check(self):
if isinstance(self.modtool, Lmod):
error_pattern = "Module command '.*load nosuchmoduleavailableanywhere' failed with exit code"
else:
- # Tcl implementations exit with 0 even when a non-existing module is loaded...
+ # Environment Modules exits with 0 even when a non-existing module is loaded...
error_pattern = "Unable to locate a modulefile for 'nosuchmoduleavailableanywhere'"
self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.load, ['nosuchmoduleavailableanywhere'])
@@ -1537,8 +1566,10 @@ def test_modulecmd_strip_source(self):
os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH'))
- modtool = EnvironmentModulesC()
- modtool.run_module('load', 'test123')
+ self.allow_deprecated_behaviour()
+ with self.mocked_stdout_stderr():
+ modtool = EnvironmentModulesC()
+ modtool.run_module('load', 'test123')
self.assertEqual(os.getenv('TEST123'), 'test123')
def test_get_setenv_value_from_modulefile(self):
@@ -1554,7 +1585,8 @@ def test_get_setenv_value_from_modulefile(self):
write_file(test_ec, "\nmodextravars = {'FOO': 'value with spaces'}", append=True)
toy_eb = EasyBlock(EasyConfig(test_ec))
- toy_eb.make_module_step()
+ with self.mocked_stdout_stderr():
+ toy_eb.make_module_step()
expected_root = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
ebroot = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'EBROOTTOY')
@@ -1569,6 +1601,247 @@ def test_get_setenv_value_from_modulefile(self):
res = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'NO_SUCH_VARIABLE_SET')
self.assertEqual(res, None)
+ def test_module_environment_variable(self):
+ """Test for ModuleEnvironmentVariable object"""
+ test_paths = ['lib', 'lib64']
+ mod_envar = mod.ModuleEnvironmentVariable(test_paths)
+ self.assertTrue(hasattr(mod_envar, 'contents'))
+ self.assertTrue(hasattr(mod_envar, 'type'))
+ self.assertTrue(hasattr(mod_envar, 'delimiter'))
+ self.assertEqual(mod_envar.contents, test_paths)
+ self.assertEqual(repr(mod_envar), repr(test_paths))
+ self.assertEqual(str(mod_envar), 'lib:lib64')
+
+ mod_envar_custom_delim = mod.ModuleEnvironmentVariable(test_paths, delimiter='|')
+ self.assertEqual(mod_envar_custom_delim.contents, test_paths)
+ self.assertEqual(repr(mod_envar_custom_delim), repr(test_paths))
+ self.assertEqual(str(mod_envar_custom_delim), 'lib|lib64')
+
+ mod_envar_custom_type = mod.ModuleEnvironmentVariable(test_paths, var_type='STRING')
+ self.assertEqual(mod_envar_custom_type.contents, test_paths)
+ self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.STRING)
+ self.assertEqual(mod_envar_custom_type.is_path, False)
+ mod_envar_custom_type.type = 'PATH'
+ self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH)
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ mod_envar_custom_type.type = mod.ModEnvVarType.PATH
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ mod_envar_custom_type.type = 'PATH_WITH_FILES'
+ self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ mod_envar_custom_type.type = mod.ModEnvVarType.PATH_WITH_FILES
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ mod_envar_custom_type.type = 'PATH_WITH_TOP_FILES'
+ self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH_WITH_TOP_FILES)
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ mod_envar_custom_type.type = mod.ModEnvVarType.PATH_WITH_TOP_FILES
+ self.assertEqual(mod_envar_custom_type.is_path, True)
+
+ self.assertRaises(EasyBuildError, setattr, mod_envar_custom_type, 'type', 'NONEXISTENT')
+ self.assertRaises(EasyBuildError, mod.ModuleEnvironmentVariable, test_paths, var_type='NONEXISTENT')
+
+ mod_envar.contents = []
+ self.assertEqual(mod_envar.contents, [])
+ self.assertRaises(TypeError, setattr, mod_envar, 'contents', None)
+ mod_envar.contents = (1, 3, 2, 3)
+ self.assertEqual(mod_envar.contents, ['1', '3', '2'])
+ mod_envar.contents = 'include'
+ self.assertEqual(mod_envar.contents, ['include'])
+
+ mod_envar.append('share')
+ self.assertEqual(mod_envar.contents, ['include', 'share'])
+ mod_envar.append('share')
+ self.assertEqual(mod_envar.contents, ['include', 'share'])
+ self.assertRaises(TypeError, mod_envar.append, 'arg1', 'arg2')
+
+ mod_envar.extend(test_paths)
+ self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64'])
+ mod_envar.extend(test_paths)
+ self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64'])
+ mod_envar.extend(test_paths + ['lib128'])
+ self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64', 'lib128'])
+ self.assertRaises(TypeError, mod_envar.append, ['list1'], ['list2'])
+
+ mod_envar.remove('lib128')
+ self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64'])
+ mod_envar.remove('nonexistent')
+ self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64'])
+ self.assertRaises(TypeError, mod_envar.remove, 'arg1', 'arg2')
+
+ mod_envar.prepend('bin')
+ self.assertEqual(mod_envar.contents, ['bin', 'include', 'share', 'lib', 'lib64'])
+
+ mod_envar.update('new_path')
+ self.assertEqual(mod_envar.contents, ['new_path'])
+ mod_envar.update(['new_path_1', 'new_path_2'])
+ self.assertEqual(mod_envar.contents, ['new_path_1', 'new_path_2'])
+ self.assertRaises(TypeError, mod_envar.update, 'arg1', 'arg2')
+
+ def test_module_load_environment(self):
+ """Test for ModuleLoadEnvironment object"""
+ mod_load_env = mod.ModuleLoadEnvironment()
+
+ # test setting attributes
+ test_contents = ['lib', 'lib64']
+ mod_load_env.TEST_VAR = test_contents
+ self.assertTrue(hasattr(mod_load_env, 'TEST_VAR'))
+ self.assertEqual(mod_load_env.TEST_VAR.contents, test_contents)
+
+ error_pattern = "Name of ModuleLoadEnvironment attribute does not conform to shell naming rules.*'test_lower'"
+ self.assertErrorRegex(EasyBuildError, error_pattern, setattr, mod_load_env, 'test_lower', test_contents)
+
+ mod_load_env.TEST_STR = 'some/path'
+ self.assertTrue(hasattr(mod_load_env, 'TEST_STR'))
+ self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path'])
+
+ mod_load_env.TEST_VARTYPE = {'contents': test_contents, 'var_type': "STRING"}
+ self.assertTrue(hasattr(mod_load_env, 'TEST_VARTYPE'))
+ self.assertEqual(mod_load_env.TEST_VARTYPE.contents, test_contents)
+ self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.STRING)
+
+ mod_load_env.TEST_VARTYPE.type = "PATH"
+ self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH)
+ env_wrong_params = {'contents': test_contents, 'unkown_param': True}
+ self.assertRaises(EasyBuildError, setattr, mod_load_env, 'TEST_UNKNONW', env_wrong_params)
+
+ mod_load_env._UNDERSCORE_VAR = test_contents
+ self.assertTrue(hasattr(mod_load_env, '_UNDERSCORE_VAR'))
+ self.assertEqual(mod_load_env._UNDERSCORE_VAR.contents, test_contents)
+
+ mod_load_env.__DOUBLE_UNDERSCORE_VAR = test_contents
+ self.assertTrue(hasattr(mod_load_env, '__DOUBLE_UNDERSCORE_VAR'))
+ self.assertEqual(mod_load_env.__DOUBLE_UNDERSCORE_VAR.contents, test_contents)
+
+ mod_load_env.___TRIPLE__UNDERSCORE_VAR = test_contents
+ self.assertTrue(hasattr(mod_load_env, '___TRIPLE__UNDERSCORE_VAR'))
+ self.assertEqual(mod_load_env.___TRIPLE__UNDERSCORE_VAR.contents, test_contents)
+
+ # test retrieval of environment
+ # use copy of public attributes as reference
+ ref_load_env = mod_load_env._env_vars.copy()
+ self.assertCountEqual(list(mod_load_env), ref_load_env.keys())
+
+ ref_load_env_item_list = list(ref_load_env.items())
+ self.assertCountEqual(list(mod_load_env.items()), ref_load_env_item_list)
+
+ ref_load_env_item_list = dict(ref_load_env.items())
+ self.assertCountEqual(mod_load_env.as_dict, ref_load_env_item_list)
+
+ ref_load_env_environ = {key: str(value) for key, value in ref_load_env.items()}
+ self.assertDictEqual(mod_load_env.environ, ref_load_env_environ)
+
+ # test updating environment
+ new_test_env = {
+ 'TEST_VARTYPE': 'replaced_path',
+ 'TEST_NEW_VAR': ['new_path1', 'new_path2'],
+ }
+ mod_load_env.update(new_test_env)
+ self.assertTrue(hasattr(mod_load_env, 'TEST_VARTYPE'))
+ self.assertEqual(mod_load_env.TEST_VARTYPE.contents, ['replaced_path'])
+ self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ self.assertTrue(hasattr(mod_load_env, 'TEST_NEW_VAR'))
+ self.assertEqual(mod_load_env.TEST_NEW_VAR.contents, ['new_path1', 'new_path2'])
+ self.assertEqual(mod_load_env.TEST_NEW_VAR.type, mod.ModEnvVarType.PATH_WITH_FILES)
+
+ # check that previous variables still exist
+ self.assertTrue(hasattr(mod_load_env, 'TEST_VAR'))
+ self.assertEqual(mod_load_env.TEST_VAR.contents, test_contents)
+ self.assertTrue(hasattr(mod_load_env, 'TEST_STR'))
+ self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path'])
+
+ # test removal of envars
+ mod_load_env.REMOVABLE_VAR = test_contents
+ self.assertTrue('REMOVABLE_VAR' in mod_load_env.vars)
+ mod_load_env.remove('REMOVABLE_VAR')
+ self.assertFalse('REMOVABLE_VAR' in mod_load_env.vars)
+ self.assertFalse('NONEXISTENT' in mod_load_env.vars)
+ mod_load_env.remove('NONEXISTENT')
+ self.assertFalse('NONEXISTENT' in mod_load_env.vars)
+
+ # test removal with delattr
+ mod_load_env.REMOVABLE_VAR = test_contents
+ self.assertTrue('REMOVABLE_VAR' in mod_load_env.vars)
+ delattr(mod_load_env, 'REMOVABLE_VAR')
+ self.assertFalse('REMOVABLE_VAR' in mod_load_env.vars)
+ mod_load_env._REMOVABLE_VAR = test_contents
+ self.assertTrue('_REMOVABLE_VAR' in mod_load_env.vars)
+ delattr(mod_load_env, '_REMOVABLE_VAR')
+ self.assertFalse('_REMOVABLE_VAR' in mod_load_env.vars)
+ mod_load_env.__REMOVABLE_VAR = test_contents
+ self.assertTrue('__REMOVABLE_VAR' in mod_load_env.vars)
+ delattr(mod_load_env, '__REMOVABLE_VAR')
+ self.assertFalse('__REMOVABLE_VAR' in mod_load_env.vars)
+ self.assertRaises(EasyBuildError, delattr, mod_load_env, 'NONEXISTENT')
+
+ # test replacing of env vars
+ env_vars = sorted(mod_load_env.as_dict.keys())
+ expected = ['ACLOCAL_PATH', 'CLASSPATH', 'CMAKE_LIBRARY_PATH', 'CMAKE_PREFIX_PATH', 'GI_TYPELIB_PATH',
+ 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'MANPATH', 'PATH', 'PKG_CONFIG_PATH', 'TEST_NEW_VAR',
+ 'TEST_STR', 'TEST_VAR', 'TEST_VARTYPE', 'XDG_DATA_DIRS', '_UNDERSCORE_VAR',
+ '__DOUBLE_UNDERSCORE_VAR', '___TRIPLE__UNDERSCORE_VAR']
+ self.assertEqual(env_vars, expected)
+
+ mod_load_env.replace({'FOO': 'foo', 'BAR': 'bar'})
+ env_vars = sorted(mod_load_env.as_dict.keys())
+ self.assertEqual(env_vars, ['BAR', 'FOO'])
+ self.assertEqual(mod_load_env.BAR.contents, ['bar'])
+ self.assertEqual(mod_load_env.FOO.contents, ['foo'])
+
+ # test aliases
+ aliases = {
+ 'ALIAS1': ['ALIAS_VAR11', 'ALIAS_VAR12'],
+ 'ALIAS2': ['ALIAS_VAR21'],
+ }
+ alias_load_env = mod.ModuleLoadEnvironment(aliases=aliases)
+ self.assertEqual(alias_load_env._aliases, aliases)
+ self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS1')), ['ALIAS_VAR11', 'ALIAS_VAR12'])
+ self.assertEqual(alias_load_env.alias_vars('ALIAS2'), ['ALIAS_VAR21'])
+ # set a known alias
+ alias_load_env.set_alias_vars('ALIAS1', 'alias1_path')
+ self.assertTrue('ALIAS_VAR11' in alias_load_env.vars)
+ self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ self.assertTrue('ALIAS_VAR12' in alias_load_env.vars)
+ self.assertEqual(alias_load_env.ALIAS_VAR12.contents, ['alias1_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR12.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ self.assertFalse('ALIAS_VAR21' in alias_load_env.vars)
+ for envar in alias_load_env.alias('ALIAS1'):
+ self.assertEqual(envar.contents, ['alias1_path'])
+ self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ # set a second known alias
+ alias_load_env.set_alias_vars('ALIAS2', 'alias2_path')
+ self.assertTrue('ALIAS_VAR11' in alias_load_env.vars)
+ self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ self.assertTrue('ALIAS_VAR21' in alias_load_env.vars)
+ self.assertEqual(alias_load_env.ALIAS_VAR21.contents, ['alias2_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR21.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ # add a new alias
+ alias_load_env.update_alias('ALIAS3', 'ALIAS_VAR31')
+ self.assertEqual(alias_load_env.alias_vars('ALIAS3'), ['ALIAS_VAR31'])
+ alias_load_env.update_alias('ALIAS3', ['ALIAS_VAR31', 'ALIAS_VAR32'])
+ self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS3')), ['ALIAS_VAR31', 'ALIAS_VAR32'])
+ alias_load_env.set_alias_vars('ALIAS3', 'alias3_path')
+ for envar in alias_load_env.alias('ALIAS3'):
+ self.assertEqual(envar.contents, ['alias3_path'])
+ self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES)
+ # append path to existing alias
+ for envar in alias_load_env.alias('ALIAS3'):
+ envar.append('new_path')
+ self.assertEqual(sorted(envar.contents), ['alias3_path', 'new_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR31.contents, ['alias3_path', 'new_path'])
+ self.assertEqual(alias_load_env.ALIAS_VAR32.contents, ['alias3_path', 'new_path'])
+
+ error_pattern = "Wrong format for aliases defitions passed to ModuleLoadEnvironment"
+ self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=False)
+ self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases='wrong')
+ self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=['some', 'list'])
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py
index 3cf4493f02..594b8b59f4 100644
--- a/test/framework/modulestool.py
+++ b/test/framework/modulestool.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,7 @@
from unittest import TextTestRunner
from easybuild.base import fancylogger
-from easybuild.tools import modules, StrictVersion
+from easybuild.tools import modules, LooseVersion
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file, which, write_file
from easybuild.tools.modules import EnvironmentModules, Lmod
@@ -76,7 +76,7 @@ def test_mock(self):
mmt = MockModulesTool(mod_paths=[], testing=True)
# the version of the MMT is the commandline option
- self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION))
+ self.assertEqual(mmt.version, LooseVersion(MockModulesTool.VERSION_OPTION))
cmd_abspath = which(MockModulesTool.COMMAND)
@@ -100,7 +100,7 @@ def test_environment_command(self):
bmmt = BrokenMockModulesTool(mod_paths=[], testing=True)
cmd_abspath = which(MockModulesTool.COMMAND)
- self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION))
+ self.assertEqual(bmmt.version, LooseVersion(MockModulesTool.VERSION_OPTION))
self.assertEqual(bmmt.cmd, cmd_abspath)
# clean it up
@@ -209,12 +209,36 @@ def test_environment_modules_specific(self):
mt = EnvironmentModules(testing=True)
self.assertIsInstance(mt.loaded_modules(), list) # dummy usage
+ # test updating module cache
+ test_modulepath = os.path.join(self.test_installpath, 'modules', 'all')
+ os.environ['MODULEPATH'] = test_modulepath
+ test_module_dir = os.path.join(test_modulepath, 'test')
+ test_module_file = os.path.join(test_module_dir, '1.2.3')
+ write_file(test_module_file, '#%Module')
+ build_options = {
+ 'update_modules_tool_cache': True,
+ }
+ init_config(build_options=build_options)
+ mt = EnvironmentModules(testing=True)
+ out = mt.update()
+ os.remove(test_module_file)
+ os.rmdir(test_module_dir)
+
+ # test cache file has been created if module tool supports it
+ if LooseVersion(mt.version) >= LooseVersion('5.3.0'):
+ cache_fp = os.path.join(test_modulepath, '.modulecache')
+ expected = "Creating %s\n" % cache_fp
+ self.assertEqual(expected, out, "Module cache created")
+ self.assertTrue(os.path.exists(cache_fp))
+ os.remove(cache_fp)
+
# initialize Environment Modules tool with non-official version number
# pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD
fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl')
fake_modulecmd_txt = '\n'.join([
- 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}',
- "puts {os.environ['FOO'] = 'foo'}",
+ '#!/bin/bash',
+ 'echo "Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)" >&2',
+ 'echo "os.environ[\'FOO\'] = \'foo\'"',
])
write_file(fake_path, fake_modulecmd_txt)
os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR)
diff --git a/test/framework/options.py b/test/framework/options.py
index 2208465db0..f54bfc4f1a 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,9 +36,9 @@
import sys
import tempfile
import textwrap
-import warnings
-from easybuild.tools import LooseVersion
+from importlib import reload
from unittest import TextTestRunner
+from urllib.request import URLError
import easybuild.main
import easybuild.tools.build_log
@@ -63,20 +63,16 @@
from easybuild.tools.modules import Lmod
from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata
from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color
-from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions
from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX
-from easybuild.tools.run import run_cmd
-from easybuild.tools.systemtools import HAVE_ARCHSPEC
+from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type
from easybuild.tools.version import VERSION
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
+ pass
EXTERNAL_MODULES_METADATA = """[foobar/1.2.3]
@@ -203,7 +199,8 @@ def test_help_rst(self):
def test_no_args(self):
"""Test using no arguments."""
- outtxt = self.eb_main([])
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main([])
error_msg = "ERROR.* Please provide one or multiple easyconfig files,"
error_msg += " or use software build options to make EasyBuild search for easyconfigs"
@@ -218,7 +215,8 @@ def test_debug(self):
'nosuchfile.eb',
debug_arg,
]
- outtxt = self.eb_main(args)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args)
for log_msg_type in ['DEBUG', 'INFO', 'ERROR']:
res = re.search(' %s ' % log_msg_type, outtxt)
@@ -232,7 +230,8 @@ def test_info(self):
'nosuchfile.eb',
info_arg,
]
- outtxt = self.eb_main(args)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args)
error_tmpl = "%s log messages are included when using %s ( out: %s)"
for log_msg_type in ['INFO', 'ERROR']:
@@ -247,7 +246,8 @@ def test_quiet(self):
"""Test enabling quiet logging (errors only)."""
for quiet_arg in ['--quiet']:
args = ['nosuchfile.eb', quiet_arg]
- out = self.eb_main(args)
+ with self.mocked_stdout_stderr():
+ out = self.eb_main(args)
for log_msg_type in ['ERROR']:
res = re.search(' %s ' % log_msg_type, out)
@@ -264,13 +264,15 @@ def test_force(self):
# use GCC-4.6.3.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb')
+ self.mock_stdout(True)
# check log message without --force
args = [
eb_file,
'--debug',
]
- outtxt, error_thrown = self.eb_main(args, return_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt, error_thrown = self.eb_main(args, return_error=True)
error_msg = "No error is thrown if software is already installed (error_thrown: %s)" % error_thrown
self.assertTrue(not error_thrown, error_msg)
@@ -284,12 +286,13 @@ def test_force(self):
# check that --force and --rebuild work
for arg in ['--force', '--rebuild']:
- outtxt = self.eb_main([eb_file, '--debug', arg])
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main([eb_file, '--debug', arg])
self.assertTrue(not re.search(already_msg, outtxt), "Already installed message not there with %s" % arg)
+ self.mock_stdout(False)
def test_skip(self):
"""Test skipping installation of module (--skip, -k)."""
-
# use toy-0.0.eb easyconfig file that comes with the tests
topdir = os.path.abspath(os.path.dirname(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
@@ -297,13 +300,11 @@ def test_skip(self):
# check log message with --skip for existing module
args = [
toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--force',
'--debug',
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
args.append('--skip')
self.mock_stdout(True)
@@ -321,24 +322,20 @@ def test_skip(self):
# check log message with --skip for non-existing module
args = [
toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--try-software-version=1.2.3.4.5.6.7.8.9',
'--try-amend=sources=toy-0.0.tar.gz,toy-0.0.tar.gz', # hackish, but fine
'--force',
'--debug',
'--skip',
]
- outtxt = self.eb_main(args, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True)
- found_msg = "Module toy/1.2.3.4.5.6.7.8.9 found."
- found = re.search(found_msg, outtxt)
- self.assertTrue(not found, "Module found message not there with --skip for non-existing modules: %s" % outtxt)
+ self.assertNotIn("Module toy/1.2.3.4.5.6.7.8.9 found.", outtxt,
+ "Module found message should not be there with --skip for non-existing modules")
- not_found_msg = "No module toy/1.2.3.4.5.6.7.8.9 found. Not skipping anything."
- not_found = re.search(not_found_msg, outtxt)
- self.assertTrue(not_found, "Module not found message there with --skip for non-existing modules: %s" % outtxt)
+ self.assertIn("No module toy/1.2.3.4.5.6.7.8.9 found. Not skipping anything.", outtxt,
+ "Module not found message should be there with --skip for non-existing modules")
toy_mod_glob = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '*')
for toy_mod in glob.glob(toy_mod_glob):
@@ -357,26 +354,26 @@ def test_skip(self):
'--force',
]
error_pattern = "Sanity check failed: no file found at 'bin/nosuchfile'"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.mocked_main, args, do_build=True, raise_error=True)
- # check use of skipsteps to skip sanity check
- test_ec_txt += "\nskipsteps = ['sanitycheck']\n"
- write_file(test_ec, test_ec_txt)
- self.eb_main(args, do_build=True, raise_error=True)
+ def test_module_only_param(self):
+ """check use of module_only parameter"""
+ topdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
- self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "\nmodule_only=True\n"
+ test_ec_txt += "\nskipsteps = ['sanitycheck']\n" # Software does not exist, so sanity check would fail
+ write_file(test_ec, test_ec_txt)
- # check use of module_only parameter
- remove_dir(os.path.join(self.test_installpath, 'modules', 'all', 'toy'))
- remove_dir(os.path.join(self.test_installpath, 'software', 'toy', '0.0'))
args = [
test_ec,
'--rebuild',
]
- test_ec_txt += "\nmodule_only = True\n"
- write_file(test_ec, test_ec_txt)
- self.eb_main(args, do_build=True, raise_error=True)
+ self.mocked_main(args, do_build=True, raise_error=True)
+ toy_mod_glob = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '*')
self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
# check that no software was installed
@@ -385,6 +382,46 @@ def test_skip(self):
easybuild_dir = os.path.join(installdir, 'easybuild')
self.assertEqual(installdir_glob, [easybuild_dir])
+ def test_skipsteps(self):
+ """Test skipping of steps using skipsteps."""
+ # use toy-0.0.eb easyconfig file that comes with the tests
+ topdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+ # make sure that sanity check is *NOT* skipped
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ regex = re.compile(r"sanity_check_paths = \{(.|\n)*\}", re.M)
+ test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt)
+ write_file(test_ec, test_ec_txt)
+ args = [
+ test_ec,
+ '--rebuild',
+ ]
+ error_pattern = "Sanity check failed: no file found at 'bin/nosuchfile'"
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.mocked_main, args, do_build=True, raise_error=True)
+
+ # Verify a wrong step name is caught
+ test_ec_txt += "\nskipsteps = ['wrong-step-name']\n"
+ write_file(test_ec, test_ec_txt)
+ error_pattern = "Found one or more unknown step names in 'skipsteps' easyconfig parameter:\n"
+ error_pattern += r"\* wrong-step-name"
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ # 'source' step was renamed to 'extract' in EasyBuild 5.0,
+ # see https://github.com/easybuilders/easybuild-framework/pull/4629
+ test_ec_txt += "\nskipsteps = ['source']\n"
+ write_file(test_ec, test_ec_txt)
+ error_pattern = error_pattern.replace('wrong-step-name', r"source \(did you mean 'extract'\?\)")
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+
+ # check use of skipsteps to skip sanity check
+ test_ec_txt += "\nskipsteps = ['sanitycheck']\n"
+ write_file(test_ec, test_ec_txt)
+ self.mocked_main(args, do_build=True, raise_error=True)
+
+ toy_mod_glob = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '*')
+ self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
+
def test_skip_test_step(self):
"""Test skipping testing the build (--skip-test-step)."""
@@ -453,24 +490,28 @@ def test_skip_sanity_check(self):
args = [test_ec, '--rebuild']
err_msg = "Sanity check failed"
- self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
args.append('--skip-sanity-check')
- outtext = self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtext = self.eb_main(args, do_build=True, raise_error=True)
self.assertNotIn('sanity checking...', outtext)
# Passing skip and only options is disallowed
args.append('--sanity-check-only')
error_pattern = 'Found both skip-sanity-check and sanity-check-only enabled'
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
def test_job(self):
"""Test submitting build as a job."""
# use gzip-1.4.eb easyconfig file that comes with the tests
- eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ eb_file = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb')
- def check_args(job_args, passed_args=None):
+ def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb_file='gzip-1.4.eb'):
"""Check whether specified args yield expected result."""
if passed_args is None:
passed_args = job_args[:]
@@ -482,22 +523,50 @@ def check_args(job_args, passed_args=None):
eb_file,
'--job',
] + job_args
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
job_msg = r"INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(passed_args)
assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
self.assertTrue(re.search(job_msg, outtxt), assertmsg)
+ if msgstrs is None:
+ msgstrs = [(tweaked_eb_file, eb_file + try_opts)]
+
+ assertmsg = "Info log msg with creating job for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
+ for msgstr in msgstrs:
+ job_msg = r"INFO creating job for ec: %s using %s\n" % msgstr
+ self.assertTrue(re.search(job_msg, outtxt), assertmsg)
+
# options passed are reordered, so order here matters to make tests pass
check_args(['--debug'])
check_args(['--debug', '--stop=configure', '--try-software-name=foo'],
- passed_args=['--debug', "--stop='configure'"])
+ passed_args=['--debug', "--stop='configure'"],
+ try_opts=" --try-software-name='foo'",
+ tweaked_eb_file="foo-1.4.eb")
check_args(['--debug', '--robot-paths=/tmp/foo:/tmp/bar'],
passed_args=['--debug', "--robot-paths='/tmp/foo:/tmp/bar'"])
# --robot has preference over --robot-paths, --robot is not passed down
check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=%s' % self.test_prefix],
passed_args=['--debug', "--robot-paths='%s:/tmp/foo'" % self.test_prefix])
+ # check if libtoy dep uses --try-toolchain but gzip does not (easyconfig exists already)
+ eb_file = os.path.join(self.test_buildpath, 'toy-0.0-with-deps.eb')
+ copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), eb_file)
+ write_file(eb_file, "dependencies = [('libtoy', '0.0'), ('gzip', '1.4')]\n", append=True)
+ try_opts = " --try-toolchain='GCC,4.9.3-2.26'"
+ tweaked_eb_file = "toy-0.0-GCC-4.9.3-2.26.eb"
+ gzip_eb_file = 'gzip-1.4-GCC-4.9.3-2.26.eb'
+ check_args(['--debug', '--stop=configure', '--try-toolchain=GCC,4.9.3-2.26', '--robot'],
+ passed_args=['--debug', "--stop='configure'"],
+ msgstrs=[
+ (tweaked_eb_file, eb_file + try_opts),
+ ('libtoy-0.0-GCC-4.9.3-2.26.eb',
+ os.path.join(test_ecs, 'l', 'libtoy', 'libtoy-0.0.eb') + try_opts),
+ (gzip_eb_file, os.path.join(test_ecs, 'g', 'gzip', gzip_eb_file))],
+ try_opts=try_opts,
+ tweaked_eb_file=tweaked_eb_file)
+
# 'zzz' prefix in the test name is intentional to make this test run last,
# since it fiddles with the logging infrastructure which may break things
def test_zzz_logtostdout(self):
@@ -535,7 +604,6 @@ def test_zzz_logtostdout(self):
self.mock_stdout(False)
self.assertIn("Auto-enabling streaming output", stdout)
- self.assertIn("== (streaming) output for command 'gcc toy.c -o toy':", stdout)
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
@@ -564,6 +632,7 @@ def run_test(fmt=None):
r'^``ARCH``\s*``(aarch64|ppc64le|x86_64)``\s*CPU architecture .*',
r'^``EXTERNAL_MODULE``.*',
r'^``HOME``.*',
+ r'^``MODULE_LOAD_ENV_HEADERS``.*Environment variables .*',
r'``OS_NAME``.*',
r'``OS_PKG_IBVERBS_DEV``.*',
]
@@ -572,6 +641,7 @@ def run_test(fmt=None):
r'^\s*ARCH: (aarch64|ppc64le|x86_64) \(CPU architecture .*\)',
r'^\s*EXTERNAL_MODULE:.*',
r'^\s*HOME:.*',
+ r'^\s*MODULE_LOAD_ENV_HEADERS:.*\(Environment variables.*\)',
r'\s*OS_NAME: .*',
r'\s*OS_PKG_IBVERBS_DEV: .*',
]
@@ -664,7 +734,8 @@ def run_test(custom=None, extra_params=[], fmt=None):
if custom is not None:
args.extend(['-e', custom])
- self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
logtxt = read_file(self.logfile)
# check whether all parameter types are listed
@@ -730,8 +801,8 @@ def test_avail_hooks(self):
" post_fetch_hook",
" pre_ready_hook",
" post_ready_hook",
- " pre_source_hook",
- " post_source_hook",
+ " pre_extract_hook",
+ " post_extract_hook",
" pre_patch_hook",
" post_patch_hook",
" pre_prepare_hook",
@@ -768,6 +839,7 @@ def test_avail_hooks(self):
" post_build_and_install_loop_hook",
" end_hook",
" cancel_hook",
+ " crash_hook",
" fail_hook",
" pre_run_shell_cmd_hook",
" post_run_shell_cmd_hook",
@@ -787,7 +859,8 @@ def test__list_toolchains(self):
'--list-toolchains',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
regex = re.compile(r"INFO List of known toolchains \(toolchain name: module\[, module, \.\.\.\]\):")
logtxt = read_file(self.logfile)
@@ -869,7 +942,7 @@ def test_avail_lists(self):
os.close(fd)
name_items = {
- 'modules-tools': ['EnvironmentModulesC', 'Lmod'],
+ 'modules-tools': ['EnvironmentModules', 'Lmod'],
'module-naming-schemes': ['EasyBuildMNS', 'HierarchicalMNS', 'CategorizedHMNS'],
}
for (name, items) in name_items.items():
@@ -877,14 +950,15 @@ def test_avail_lists(self):
'--avail-%s' % name,
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
words = name.replace('-', ' ')
info_msg = r"INFO List of supported %s:" % words
self.assertTrue(re.search(info_msg, logtxt), "Info message with list of available %s" % words)
for item in items:
- res = re.findall(r"^\s*%s" % item, logtxt, re.M)
+ res = re.findall(r"^\s*%s\n" % item, logtxt, re.M)
self.assertTrue(res, "%s is included in list of available %s" % (item, words))
# every item should only be mentioned once
n = len(res)
@@ -913,7 +987,8 @@ def test_avail_cfgfile_constants(self):
'--avail-cfgfile-constants',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
cfgfile_constants = {
'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'),
@@ -945,41 +1020,51 @@ def test_000_list_easyblocks(self):
list_arg,
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
expected = '\n'.join([
- r'EasyBlock',
- r'\|-- bar',
- r'\|-- ConfigureMake',
- r'\| \|-- MakeCp',
- r'\|-- EB_EasyBuildMeta',
- r'\|-- EB_FFTW',
- r'\|-- EB_foo',
- r'\| \|-- EB_foofoo',
- r'\|-- EB_GCC',
- r'\|-- EB_HPL',
- r'\|-- EB_libtoy',
- r'\|-- EB_OpenBLAS',
- r'\|-- EB_OpenMPI',
- r'\|-- EB_ScaLAPACK',
- r'\|-- EB_toy_buggy',
- r'\|-- ExtensionEasyBlock',
- r'\| \|-- DummyExtension',
- r'\| \|-- EB_toy',
- r'\| \| \|-- EB_toy_eula',
- r'\| \| \|-- EB_toytoy',
- r'\| \|-- Toy_Extension',
- r'\|-- ModuleRC',
- r'\|-- PythonBundle',
- r'\|-- Toolchain',
- r'Extension',
- r'\|-- ExtensionEasyBlock',
- r'\| \|-- DummyExtension',
- r'\| \|-- EB_toy',
- r'\| \| \|-- EB_toy_eula',
- r'\| \| \|-- EB_toytoy',
- r'\| \|-- Toy_Extension',
+ "EasyBlock",
+ "|-- bar",
+ "|-- ConfigureMake",
+ "| |-- MakeCp",
+ "|-- EB_EasyBuildMeta",
+ "|-- EB_FFTW",
+ "|-- EB_foo",
+ "| |-- EB_foofoo",
+ "|-- EB_GCC",
+ "|-- EB_HPL",
+ "|-- EB_libtoy",
+ "|-- EB_OpenBLAS",
+ "|-- EB_OpenMPI",
+ "|-- EB_ScaLAPACK",
+ "|-- EB_toy_buggy",
+ "|-- ExtensionEasyBlock",
+ "| |-- DummyExtension",
+ "| | |-- CustomDummyExtension",
+ "| | | |-- ChildCustomDummyExtension",
+ "| | |-- DeprecatedDummyExtension",
+ "| | | |-- ChildDeprecatedDummyExtension",
+ "| |-- EB_toy",
+ "| | |-- EB_toy_eula",
+ "| | |-- EB_toytoy",
+ "| |-- Toy_Extension",
+ "|-- ModuleRC",
+ "|-- PythonBundle",
+ "|-- Toolchain",
+ "Extension",
+ "|-- ExtensionEasyBlock",
+ "| |-- DummyExtension",
+ "| | |-- CustomDummyExtension",
+ "| | | |-- ChildCustomDummyExtension",
+ "| | |-- DeprecatedDummyExtension",
+ "| | | |-- ChildDeprecatedDummyExtension",
+ "| |-- EB_toy",
+ "| | |-- EB_toy_eula",
+ "| | |-- EB_toytoy",
+ "| |-- Toy_Extension",
+ "",
])
regex = re.compile(expected, re.M)
self.assertTrue(regex.search(logtxt), "Pattern '%s' found in: %s" % (regex.pattern, logtxt))
@@ -992,7 +1077,8 @@ def test_000_list_easyblocks(self):
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
patterns = [
@@ -1128,7 +1214,8 @@ def test_search(self):
for opt in ['--search', '-S', '--search-short']:
for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']:
args = [opt, pattern, '--robot', test_easyconfigs_dir]
- self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True)
def test_ignore_index(self):
"""
@@ -1233,12 +1320,7 @@ def mocked_main(self, args, **kwargs):
if not kwargs:
kwargs = {'raise_error': True}
- self.mock_stderr(True)
- self.mock_stdout(True)
- self.eb_main(args, **kwargs)
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
+ stdout, stderr = self._run_mock_eb(args, **kwargs)
self.assertEqual(stderr, '')
return stdout.strip()
@@ -1327,7 +1409,8 @@ def check_copied_files():
self.assertTrue(os.path.isfile(target))
args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', target]
error_pattern = ".*/test.eb exists but is not a directory"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
# test use of --copy-ec with only one argument: copy to current working directory
test_working_dir = os.path.join(self.test_prefix, 'test_working_dir')
@@ -1345,7 +1428,8 @@ def check_copied_files():
# --copy-ec without arguments results in a proper error
args = ['--copy-ec']
error_pattern = "One or more files to copy should be specified!"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
def test_github_copy_ec_from_pr(self):
"""Test combination of --copy-ec with --from-pr."""
@@ -1359,60 +1443,59 @@ def test_github_copy_ec_from_pr(self):
# Make sure the test target directory doesn't exist
remove_dir(test_target_dir)
- all_files_pr8007 = [
- 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb',
- 'bat-0.3.3-fix-pyspark.patch',
- 'bat-0.3.3-intel-2017b-Python-3.6.3.eb',
+ all_files_pr22345 = [
+ 'QuantumESPRESSO-7.4-foss-2024a.eb',
+ 'QuantumESPRESSO-7.4-parallel-symmetrization.patch',
]
# test use of --copy-ec with --from-pr to the current working directory
cwd = change_dir(test_working_dir)
- args = ['--copy-ec', '--from-pr', '8007']
+ args = ['--copy-ec', '--from-pr', '22345']
stdout = self.mocked_main(args)
- regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_working_dir))
+ regex = re.compile(r"2 file\(s\) copied to .*/%s" % os.path.basename(test_working_dir))
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
# check that the files exist
- for pr_file in all_files_pr8007:
+ for pr_file in all_files_pr22345:
self.assertExists(os.path.join(test_working_dir, pr_file))
remove_file(os.path.join(test_working_dir, pr_file))
# copying all files touched by PR to a non-existing target directory (which is created automatically)
self.assertNotExists(test_target_dir)
- args = ['--copy-ec', '--from-pr', '8007', test_target_dir]
+ args = ['--copy-ec', '--from-pr', '22345', test_target_dir]
stdout = self.mocked_main(args)
- regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_target_dir))
+ regex = re.compile(r"2 file\(s\) copied to .*/%s" % os.path.basename(test_target_dir))
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
- for pr_file in all_files_pr8007:
+ for pr_file in all_files_pr22345:
self.assertExists(os.path.join(test_target_dir, pr_file))
remove_dir(test_target_dir)
# test where we select a single easyconfig file from a PR
mkdir(test_target_dir)
- ec_filename = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'
- args = ['--copy-ec', '--from-pr', '8007', ec_filename, test_target_dir]
+ ec_filename = 'QuantumESPRESSO-7.4-foss-2024a.eb'
+ args = ['--copy-ec', '--from-pr', '22345', ec_filename, test_target_dir]
stdout = self.mocked_main(args)
regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_target_dir)))
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
self.assertEqual(os.listdir(test_target_dir), [ec_filename])
- self.assertIn("name = 'bat'", read_file(os.path.join(test_target_dir, ec_filename)))
+ self.assertIn("name = 'QuantumESPRESSO'", read_file(os.path.join(test_target_dir, ec_filename)))
remove_dir(test_target_dir)
# test copying of a single easyconfig file from a PR to a non-existing path
- bat_ec = os.path.join(self.test_prefix, 'bat.eb')
- args[-1] = bat_ec
+ environ_ec = os.path.join(self.test_prefix, 'QuantumESPRESSO.eb')
+ args[-1] = environ_ec
stdout = self.mocked_main(args)
- regex = re.compile(r"%s copied to .*/bat.eb" % ec_filename)
+ regex = re.compile(r"%s copied to .*/QuantumESPRESSO.eb" % ec_filename)
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
- self.assertExists(bat_ec)
- self.assertIn("name = 'bat'", read_file(bat_ec))
+ self.assertExists(environ_ec)
+ self.assertIn("name = 'QuantumESPRESSO'", read_file(environ_ec))
change_dir(cwd)
remove_dir(test_working_dir)
@@ -1420,8 +1503,8 @@ def test_github_copy_ec_from_pr(self):
change_dir(test_working_dir)
# test copying of a patch file from a PR via --copy-ec to current directory
- patch_fn = 'bat-0.3.3-fix-pyspark.patch'
- args = ['--copy-ec', '--from-pr', '8007', patch_fn, '.']
+ patch_fn = 'QuantumESPRESSO-7.4-parallel-symmetrization.patch'
+ args = ['--copy-ec', '--from-pr', '22345', patch_fn, '.']
stdout = self.mocked_main(args)
self.assertEqual(os.listdir(test_working_dir), [patch_fn])
@@ -1432,18 +1515,18 @@ def test_github_copy_ec_from_pr(self):
# test the same thing but where we don't provide a target location
change_dir(test_working_dir)
- args = ['--copy-ec', '--from-pr', '8007', ec_filename]
+ args = ['--copy-ec', '--from-pr', '22345', ec_filename]
stdout = self.mocked_main(args)
regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_working_dir)))
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
self.assertEqual(os.listdir(test_working_dir), [ec_filename])
- self.assertIn("name = 'bat'", read_file(os.path.join(test_working_dir, ec_filename)))
+ self.assertIn("name = 'QuantumESPRESSO'", read_file(os.path.join(test_working_dir, ec_filename)))
# also test copying of patch file to current directory (without specifying target location)
change_dir(test_working_dir)
- args = ['--copy-ec', '--from-pr', '8007', patch_fn]
+ args = ['--copy-ec', '--from-pr', '22345', patch_fn]
stdout = self.mocked_main(args)
regex = re.compile(r"%s copied to .*/%s" % (patch_fn, os.path.basename(test_working_dir)))
@@ -1457,13 +1540,13 @@ def test_github_copy_ec_from_pr(self):
# test with only one ec in the PR (final argument is taken as a filename)
test_ec = os.path.join(self.test_prefix, 'test.eb')
- args = ['--copy-ec', '--from-pr', '11521', test_ec]
- ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb"
+ args = ['--copy-ec', '--from-pr', '22380', test_ec]
+ ec_pr22380 = "PySide2-5.14.2.3-GCCcore-10.2.0.eb"
stdout = self.mocked_main(args)
- regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec))
+ regex = re.compile(r'.*/%s copied to %s' % (ec_pr22380, test_ec))
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
self.assertExists(test_ec)
- self.assertIn("name = 'ExifTool'", read_file(test_ec))
+ self.assertIn("name = 'PySide2'", read_file(test_ec))
remove_file(test_ec)
def test_copy_ec_from_commit(self):
@@ -1532,15 +1615,19 @@ def test_copy_ec_from_commit(self):
def test_dry_run(self):
"""Test dry run (long format)."""
+
+ # first test with --robot
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
args = [
'gzip-1.4-GCC-4.6.3.eb',
- '--dry-run', # implies enabling dependency resolution
+ '--dry-run',
+ '--robot', # implies enabling dependency resolution
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
info_msg = r"Dry run: printing build status of easyconfigs and dependencies"
@@ -1553,6 +1640,70 @@ def test_dry_run(self):
regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M)
self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt))
+ # next test without --robot
+ fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+ os.close(fd)
+
+ args = [
+ 'gzip-1.4-GCC-4.6.3.eb',
+ '--dry-run',
+ '--unittest-file=%s' % self.logfile,
+ ]
+ self.eb_main(args, logfile=dummylogfn)
+ logtxt = read_file(self.logfile)
+
+ info_msg = r"Dry run: printing build status of easyconfigs"
+ self.assertTrue(re.search(info_msg, logtxt, re.M), "Info message dry running in '%s'" % logtxt)
+ ec, mod, mark = ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3", ' ')
+ regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M)
+ self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt))
+
+ def test_persistence_copying_restrictions(self):
+ """
+ Test that EasyBuild fails when instructed to move logs or artifacts inside the build directory
+
+ Moving log files or artifacts inside the build directory modifies the build artifacts, and in the case of
+ build artifacts it is also copying directories into themselves.
+ """
+ base_args = [
+ 'gzip-1.4-GCC-4.6.3.eb',
+ '--dry-run',
+ '--robot',
+ ]
+
+ def test_eb_with(option_flag, is_valid):
+ with tempfile.TemporaryDirectory() as root_dir:
+ build_dir = os.path.join(root_dir, 'build_dir')
+ if is_valid:
+ persist_path = os.path.join(root_dir, 'persist_dir')
+ else:
+ persist_path = os.path.join(root_dir, 'build_dir', 'persist_dir')
+
+ extra_args = [
+ f"--buildpath={build_dir}",
+ f"{option_flag}={persist_path}",
+ ]
+
+ pattern = rf"The {option_flag} \(.*\) cannot reside in a subdirectory of the --buildpath \(.*\)"
+
+ args = base_args
+ args.extend(extra_args)
+
+ if is_valid:
+ try:
+ self.eb_main(args, raise_error=True)
+ except EasyBuildError:
+ self.fail(
+ "Should not fail with --buildpath='{build_dir}' and {option_flag}='{persist_path}'."
+ )
+ else:
+ self.assertErrorRegex(EasyBuildError, pattern, self.eb_main, args, raise_error=True)
+
+ test_eb_with(option_flag='--failed-install-logs-path', is_valid=True)
+ test_eb_with(option_flag='--failed-install-logs-path', is_valid=False)
+ test_eb_with(option_flag='--failed-install-build-dirs-path', is_valid=True)
+ test_eb_with(option_flag='--failed-install-build-dirs-path', is_valid=False)
+
def test_missing(self):
"""Test use of --missing/-M."""
@@ -1631,7 +1782,8 @@ def test_dry_run_short(self):
'--robot=%s' % robot_decoy,
'--unittest-file=%s' % self.logfile,
]
- outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
info_msg = r"Dry run: printing build status of easyconfigs and dependencies"
self.assertTrue(re.search(info_msg, outtxt, re.M), "Info message dry running in '%s'" % outtxt)
@@ -1668,9 +1820,6 @@ def test_try_robot_force(self):
args = [
eb1,
eb2,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--force',
'--robot=%s' % test_ecs,
@@ -1678,7 +1827,8 @@ def test_try_robot_force(self):
'--dry-run',
'--unittest-file=%s' % self.logfile,
]
- outtxt = self.eb_main(args, logfile=dummylogfn)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn)
scalapack_ver = '2.0.2-gompi-2018b-OpenBLAS-0.2.20'
ecs_mods = [
@@ -1705,16 +1855,19 @@ def test_try_toolchain_mapping(self):
gzip_ec,
'--try-toolchain=iccifort,2016.1.150-GCC-4.9.3-2.25',
'--dry-run',
+ '--robot',
]
# by default, toolchain mapping is enabled
# if it fails, an error is printed
error_pattern = "Toolchain iccifort is not equivalent to toolchain foss in terms of capabilities."
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True, do_build=True)
# can continue anyway using --disable-map-toolchains
args.append('--disable-map-toolchains')
- outtxt = self.eb_main(args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True, do_build=True)
patterns = [
r"^ \* \[ \] .*/iccifort-2016.1.150-GCC-4.9.3-2.25.eb \(module: iccifort/.*\)$",
@@ -1759,12 +1912,15 @@ def test_try_update_deps(self):
'--try-toolchain-version=6.4.0-2.28',
'--try-update-deps',
'-D',
+ '--robot',
]
- self.assertErrorRegex(EasyBuildError, "Experimental functionality", self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Experimental functionality", self.eb_main, args, raise_error=True)
args.append('--experimental')
- outtxt = self.eb_main(args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True, do_build=True)
patterns = [
# toolchain got updated
@@ -1817,7 +1973,8 @@ def test_try_update_deps(self):
# Now verify that we can ignore versionsuffixes
args.append('--try-ignore-versionsuffixes')
- outtxt = self.eb_main(args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True, do_build=True)
patterns = [
# toolchain got updated
r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$",
@@ -1840,13 +1997,15 @@ def test_dry_run_hierarchical(self):
'gzip-1.5-foss-2018a.eb',
'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb',
'--dry-run',
+ '--robot',
'--unittest-file=%s' % self.logfile,
'--module-naming-scheme=HierarchicalMNS',
'--ignore-osdeps',
'--force',
'--debug',
]
- outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
ecs_mods = [
# easyconfig, module subdir, (short) module name
@@ -1880,13 +2039,15 @@ def test_dry_run_categorized(self):
'gzip-1.5-foss-2018a.eb',
'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb',
'--dry-run',
+ '--robot',
'--unittest-file=%s' % self.logfile,
'--module-naming-scheme=CategorizedHMNS',
'--ignore-osdeps',
'--force',
'--debug',
]
- outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True)
ecs_mods = [
# easyconfig, module subdir, (short) module name, mark
@@ -1922,8 +2083,8 @@ def test_github_from_pr(self):
tmpdir = tempfile.mkdtemp()
args = [
- # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files
- '--from-pr=6424',
+ # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
+ '--from-pr=22227',
'--dry-run',
# an argument must be specified to --robot, since easybuild-easyconfigs may not be installed
'--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'),
@@ -1932,17 +2093,10 @@ def test_github_from_pr(self):
'--tmpdir=%s' % tmpdir,
]
try:
- outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
modules = [
- (tmpdir, 'FFTW/3.3.8-gompi-2018b'),
- (tmpdir, 'foss/2018b'),
- ('.*', 'GCC/7.3.0-2.30'), # not included in PR
- (tmpdir, 'gompi/2018b'),
- (tmpdir, 'HPL/2.2-foss-2018b'),
- ('.*', 'hwloc/1.11.8-GCC-7.3.0-2.30'),
- ('.*', 'OpenBLAS/0.3.1-GCC-7.3.0-2.30'),
- ('.*', 'OpenMPI/3.1.1-GCC-7.3.0-2.30'),
- (tmpdir, 'ScaLAPACK/2.0.2-gompi-2018b-OpenBLAS-0.3.1'),
+ (tmpdir, 'XCrySDen/1.6.2-foss-2024a'),
]
for path_prefix, module in modules:
ec_fn = "%s.eb" % '-'.join(module.split('/'))
@@ -1950,11 +2104,7 @@ def test_github_from_pr(self):
regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M)
self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
- # make sure that *only* these modules are listed, no others
- regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
- self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
-
- pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424')
+ pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr22227')
regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
except URLError as err:
@@ -1964,8 +2114,8 @@ def test_github_from_pr(self):
# test with multiple prs
tmpdir = tempfile.mkdtemp()
args = [
- # PRs for ReFrame 3.4.1 and 3.5.0
- '--from-pr=12150,12366',
+ # PRs for various easyconfigs
+ '--from-pr=22227,19834',
'--dry-run',
# an argument must be specified to --robot, since easybuild-easyconfigs may not be installed
'--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'),
@@ -1974,10 +2124,11 @@ def test_github_from_pr(self):
'--tmpdir=%s' % tmpdir,
]
try:
- outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
modules = [
- (tmpdir, 'ReFrame/3.4.1'),
- (tmpdir, 'ReFrame/3.5.0'),
+ (tmpdir, 'quarto/1.5.57-x64'),
+ (tmpdir, 'Gblocks/0.91b'),
]
for path_prefix, module in modules:
ec_fn = "%s.eb" % '-'.join(module.split('/'))
@@ -1985,11 +2136,7 @@ def test_github_from_pr(self):
regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M)
self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
- # make sure that *only* these modules are listed, no others
- regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
- self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
-
- for pr in ('12150', '12366'):
+ for pr in ('22227', '19834'):
pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr)
regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M)
self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt))
@@ -2008,8 +2155,8 @@ def test_github_from_pr_token_log(self):
os.close(fd)
args = [
- # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files
- '--from-pr=6424',
+ # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
+ '--from-pr=22227',
'--dry-run',
'--debug',
# an argument must be specified to --robot, since easybuild-easyconfigs may not be installed
@@ -2052,10 +2199,10 @@ def test_github_from_pr_listed_ecs(self):
tmpdir = tempfile.mkdtemp()
args = [
'toy-0.0.eb',
- 'gompi-2018b.eb', # also pulls in GCC, OpenMPI (which pulls in hwloc)
+ 'XCrySDen-1.6.2-foss-2024a.eb',
'GCC-4.6.3.eb',
- # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files
- '--from-pr=6424',
+ # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
+ '--from-pr=22227',
'--dry-run',
# an argument must be specified to --robot, since easybuild-easyconfigs may not be installed
'--robot=%s' % test_ecs_path,
@@ -2064,13 +2211,12 @@ def test_github_from_pr_listed_ecs(self):
'--tmpdir=%s' % tmpdir,
]
try:
- outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
modules = [
(test_ecs_path, 'toy/0.0'), # not included in PR
- (test_ecs_path, 'GCC/7.3.0-2.30'), # not included in PR, available locally
- (test_ecs_path, 'hwloc/1.11.8-GCC-7.3.0-2.30'),
- (test_ecs_path, 'OpenMPI/3.1.1-GCC-7.3.0-2.30'),
- ('.*%s' % os.path.dirname(tmpdir), 'gompi/2018b'),
+ ('.*%s' % os.path.dirname(tmpdir), 'XCrySDen/1.6.2-foss-2024a'),
+ ('.*%s' % os.path.dirname(tmpdir), 'Togl/2.0-GCCcore-13.3.0'),
(test_ecs_path, 'GCC/4.6.3'), # not included in PR, available locally
]
for path_prefix, module in modules:
@@ -2096,19 +2242,15 @@ def test_github_from_pr_x(self):
os.close(fd)
args = [
- # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files
- '--from-pr=6424',
- 'FFTW-3.3.8-gompi-2018b.eb',
+ # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
+ '--from-pr=22227',
+ 'XCrySDen-1.6.2-foss-2024a.eb',
# an argument must be specified to --robot, since easybuild-easyconfigs may not be installed
'--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user
'--tmpdir=%s' % self.test_prefix,
'--extended-dry-run',
]
try:
- # PR #6424 includes easyconfigs that use 'dummy' toolchain,
- # so we need to allow triggering deprecated behaviour
- self.allow_deprecated_behaviour()
-
self.mock_stderr(True) # just to capture deprecation warning
self.mock_stdout(True)
self.mock_stderr(True)
@@ -2119,8 +2261,8 @@ def test_github_from_pr_x(self):
msg_regexs = [
re.compile(r"^== Build succeeded for 1 out of 1", re.M),
- re.compile(r"^\*\*\* DRY RUN using 'EB_FFTW' easyblock", re.M),
- re.compile(r"^== building and installing FFTW/3.3.8-gompi-2018b\.\.\.", re.M),
+ re.compile(r"^\*\*\* DRY RUN using 'EB_XCrySDen' easyblock", re.M),
+ re.compile(r"^== building and installing XCrySDen/1.6.2-foss-2024a\.\.\.", re.M),
re.compile(r"^building... \[DRY RUN\]", re.M),
re.compile(r"^== COMPLETED: Installation ended successfully \(took .* secs?\)", re.M),
]
@@ -2187,6 +2329,7 @@ def test_from_commit(self):
args = [
'--from-commit=%s' % test_commit,
'--dry-run',
+ '--robot',
'--tmpdir=%s' % tmpdir,
'--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'),
]
@@ -2269,7 +2412,8 @@ def test_no_such_software(self):
'--robot=.',
'--debug',
]
- outtxt = self.eb_main(args)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args)
# error message when template is not found
error_msg1 = "ERROR.* No easyconfig files found for software nosuchsoftware, and no templates available. "
@@ -2318,15 +2462,13 @@ def test_header_footer(self):
# check log message with --skip for existing module
args = [
eb_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--force',
'--modules-header=%s' % modules_header,
'--modules-footer=%s' % modules_footer,
]
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
@@ -2358,14 +2500,12 @@ def test_recursive_module_unload(self):
for lastarg in lastargs:
args = [
eb_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--force',
lastarg,
]
- self.eb_main(args, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, verbose=True)
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps')
@@ -2391,13 +2531,11 @@ def test_tmpdir(self):
# check log message with --skip for existing module
args = [
eb_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--tmpdir=%s' % tmpdir,
]
- outtxt = self.eb_main(args, do_build=True, reset_env=False)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, reset_env=False)
tmpdir_msg = r"Using %s\S+ as temporary directory" % os.path.join(tmpdir, 'eb-')
found = re.search(tmpdir_msg, outtxt, re.M)
@@ -2434,7 +2572,8 @@ def test_ignore_osdeps(self):
args = [
eb_file,
]
- outtxt = self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True)
regex = re.compile("Checking OS dependencies")
self.assertTrue(regex.search(outtxt), "OS dependencies are checked, outtxt: %s" % outtxt)
@@ -2449,7 +2588,8 @@ def test_ignore_osdeps(self):
'--ignore-osdeps',
'--dry-run',
]
- outtxt = self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True)
regex = re.compile("Not checking OS dependencies", re.M)
self.assertTrue(regex.search(outtxt), "OS dependencies are ignored with --ignore-osdeps, outtxt: %s" % outtxt)
@@ -2460,7 +2600,8 @@ def test_ignore_osdeps(self):
eb_file,
'--dry-run', # no explicit --ignore-osdeps, but implied by --dry-run
]
- outtxt = self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True)
regex = re.compile("stop provided 'notavalidstop' is not valid", re.M)
self.assertTrue(regex.search(outtxt), "Validations are performed with --ignore-osdeps, outtxt: %s" % outtxt)
@@ -2572,7 +2713,8 @@ def test_allow_modules_tool_mismatch(self):
'--modules-tool=MockModulesTool',
'--module-syntax=Tcl', # Lua would require Lmod
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
error_regex = re.compile("ERROR .*pattern .* not found in defined 'module' function")
self.assertTrue(error_regex.search(outtxt), "Found error w.r.t. module function mismatch: %s" % outtxt[-600:])
@@ -2585,7 +2727,8 @@ def test_allow_modules_tool_mismatch(self):
'--module-syntax=Tcl', # Lua would require Lmod
'--allow-modules-tool-mismatch',
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(outtxt), "Found warning w.r.t. module function mismatch: %s" % outtxt[-600:])
@@ -2598,7 +2741,8 @@ def test_allow_modules_tool_mismatch(self):
'--module-syntax=Tcl', # Lua would require Lmod
'--debug',
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
found_regex = re.compile("DEBUG Found pattern .* in defined 'module' function")
self.assertTrue(found_regex.search(outtxt), "Found debug message w.r.t. module function: %s" % outtxt[-600:])
@@ -2618,9 +2762,6 @@ def test_try(self):
args = [
tweaked_toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--dry-run',
'--robot=%s' % ecs_path,
]
@@ -2657,18 +2798,21 @@ def test_try(self):
]
for extra_args, mod in test_cases:
- outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True)
mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M)
self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt))
for extra_arg in ['--try-software=foo', '--try-toolchain=gompi', '--try-toolchain=gomp,2018a,-a-suffix']:
allargs = args + [extra_arg]
- self.assertErrorRegex(EasyBuildError, "problems validating the options",
- self.eb_main, allargs, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "problems validating the options",
+ self.eb_main, allargs, raise_error=True)
# no --try used, so no tweaked easyconfig files are generated
allargs = args + ['--software-version=1.2.3', '--toolchain=gompi,2018a']
- self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True)
# Try changing only name or version of toolchain
args.pop(0) # Remove EC filename
@@ -2695,9 +2839,6 @@ def test_try_with_copy(self):
args = [
tweaked_toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--dry-run',
'--robot=%s' % ecs_path,
'--copy-ec',
@@ -2740,13 +2881,11 @@ def test_software_version_ordering(self):
args = [
'--software=GCC,4.10.1',
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--dry-run',
'--robot=%s:%s' % (ecs_path, self.test_prefix),
]
- out = self.eb_main(['--software=GCC,4.10.1'] + args[1:], raise_error=True)
+ with self.mocked_stdout_stderr():
+ out = self.eb_main(['--software=GCC,4.10.1'] + args[1:], raise_error=True)
regex = re.compile(r"GCC-4.10.1.eb \(module: GCC/4.10.1\)$", re.M)
self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out))
@@ -2762,8 +2901,6 @@ def test_recursive_try(self):
args = [
tweaked_toy_ec,
'--sourcepath=%s' % sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--try-toolchain=gompi,2018a',
'--robot=%s' % ecs_path,
'--ignore-osdeps',
@@ -2771,7 +2908,8 @@ def test_recursive_try(self):
]
for extra_args in [[], ['--module-naming-scheme=HierarchicalMNS']]:
- outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True)
# toolchain GCC/4.7.2 (subtoolchain of gompi/2018a) should be listed (and present)
tc_regex = re.compile(r"^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M)
@@ -2792,7 +2930,9 @@ def test_recursive_try(self):
# recursive try also when --(try-)software(-X) is involved
for extra_args in [[],
['--module-naming-scheme=HierarchicalMNS']]:
- outtxt = self.eb_main(args + extra_args + ['--try-software-version=1.2.3'], verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + extra_args + ['--try-software-version=1.2.3'], verbose=True,
+ raise_error=True)
# toolchain GCC/6.4.0-2.28 (subtoolchain of gompi/2018a) should be listed (and present)
tc_regex = re.compile(r"^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M)
@@ -2814,7 +2954,8 @@ def test_recursive_try(self):
# no recursive try if --disable-map-toolchains is involved
for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]:
- outtxt = self.eb_main(args + ['--disable-map-toolchains'] + extra_args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + ['--disable-map-toolchains'] + extra_args, raise_error=True)
for mod in ['toy/1.2.3-gompi-2018a', 'gompi/2018a', 'GCC/6.4.0-2.28']:
mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M)
self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt))
@@ -2831,13 +2972,15 @@ def test_cleanup_builddir(self):
toy_ec,
'--force',
]
- self.eb_main(args, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, verbose=True)
# make sure build directory is properly cleaned up after a successful build (default behavior)
self.assertFalse(os.path.exists(toy_buildpath), "Build dir %s removed after successful build" % toy_buildpath)
# make sure --disable-cleanup-builddir works
args.append('--disable-cleanup-builddir')
- self.eb_main(args, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, verbose=True)
self.assertExists(toy_buildpath, "Build dir %s is retained when requested" % toy_buildpath)
shutil.rmtree(toy_buildpath)
@@ -2858,12 +3001,11 @@ def test_filter_deps(self):
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
args = [
ec_file,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--robot=%s' % os.path.join(test_dir, 'easyconfigs'),
'--dry-run',
]
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
# note: using loose regex pattern when we expect no match, strict pattern when we do expect a match
self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt))
@@ -2875,7 +3017,8 @@ def test_filter_deps(self):
# filter deps (including a non-existing dep, i.e. zlib)
args.extend(['--filter-deps', 'FFTW,ScaLAPACK,zlib'])
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW/3.3.7-gompi', outtxt))
self.assertFalse(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2884,7 +3027,8 @@ def test_filter_deps(self):
# filter specific version of deps
args[-1] = 'FFTW=3.2.3,zlib,ScaLAPACK=2.0.2'
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt))
self.assertFalse(re.search('module: ScaLAPACK', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2892,7 +3036,8 @@ def test_filter_deps(self):
write_file(self.logfile, '')
args[-1] = 'zlib,FFTW=3.3.7,ScaLAPACK=2.0.1'
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW', outtxt))
self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2901,7 +3046,8 @@ def test_filter_deps(self):
# filter deps with version range: only filter FFTW 3.x, ScaLAPACK 1.x
args[-1] = 'zlib,ScaLAPACK=]1.0:2.0[,FFTW=[3.0:4.0['
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW', outtxt))
self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2910,7 +3056,8 @@ def test_filter_deps(self):
# also test open ended ranges
args[-1] = 'zlib,ScaLAPACK=[1.0:,FFTW=:4.0['
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW', outtxt))
self.assertFalse(re.search('module: ScaLAPACK', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2918,14 +3065,16 @@ def test_filter_deps(self):
write_file(self.logfile, '')
args[-1] = 'zlib,ScaLAPACK=[2.1:,FFTW=:3.0['
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt))
self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
# test corner cases where version to filter in equal to low/high range limit
args[-1] = 'FFTW=[3.3.7:4.0],zlib,ScaLAPACK=[1.0:2.0.2]'
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW', outtxt))
self.assertFalse(re.search('module: ScaLAPACK', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2934,7 +3083,8 @@ def test_filter_deps(self):
# FFTW & ScaLAPACK versions are not included in range, so no filtering
args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=[1.0:2.0.2['
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt))
self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2943,14 +3093,16 @@ def test_filter_deps(self):
# also test mix of ranges & specific versions
args[-1] = 'FFTW=3.3.7,zlib,ScaLAPACK=[1.0:2.0.2['
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertFalse(re.search('module: FFTW', outtxt))
self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
write_file(self.logfile, '')
args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=2.0.2'
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt))
self.assertFalse(re.search('module: ScaLAPACK', outtxt))
self.assertFalse(re.search('module: zlib', outtxt))
@@ -2961,7 +3113,8 @@ def test_filter_deps(self):
ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb')
args[0] = ec_file
args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib'
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
# dictionaries can be printed in any order
regexp = "filtered out dependency.*('name': 'CMake'.*'version': '2.8.10'|'version': '2.8.10'.*'name': 'CMake')"
self.assertTrue(re.search(regexp, outtxt))
@@ -2971,7 +3124,8 @@ def test_filter_deps(self):
ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb')
args[0] = ec_file
args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib'
- outtxt = self.eb_main(args + ['--minimal-toolchains'], do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + ['--minimal-toolchains'], do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search(regexp, outtxt))
def test_hide_deps(self):
@@ -2981,12 +3135,11 @@ def test_hide_deps(self):
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
args = [
ec_file,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--robot=%s' % os.path.join(test_dir, 'easyconfigs'),
'--dry-run',
]
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: GCC/6.4.0-2.28', outtxt))
self.assertTrue(re.search('module: OpenMPI/2.1.2-GCC-6.4.0-2.28', outtxt))
self.assertTrue(re.search('module: OpenBLAS/0.2.20-GCC-6.4.0-2.28', outtxt))
@@ -3000,7 +3153,8 @@ def test_hide_deps(self):
# hide deps (including a non-existing dep, i.e. zlib)
args.append('--hide-deps=FFTW,ScaLAPACK,zlib')
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.assertTrue(re.search('module: GCC/6.4.0-2.28', outtxt))
self.assertTrue(re.search('module: OpenMPI/2.1.2-GCC-6.4.0-2.28', outtxt))
self.assertTrue(re.search('module: OpenBLAS/0.2.20-GCC-6.4.0-2.28', outtxt))
@@ -3018,9 +3172,11 @@ def test_hide_toolchains(self):
args = [
ec_file,
'--dry-run',
+ '--robot',
'--hide-toolchains=GCC',
]
- outtxt = self.eb_main(args)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args)
self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt))
self.assertTrue(re.search(r'module: gzip/1\.6-GCC-4\.9\.2', outtxt))
@@ -3221,16 +3377,13 @@ def toy(extra_args=None):
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
args = [
eb_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--force',
'--debug',
]
if extra_args is not None:
args.extend(extra_args)
- self.eb_main(args, do_build=True, raise_error=True, verbose=True)
-
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True, verbose=True)
software_path = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-0.0*test_report.md')
test_report_txt = read_file(glob.glob(test_report_path_pattern)[0])
@@ -3278,7 +3431,8 @@ def test_robot(self):
'--robot-paths=%s' % test_ecs_path,
]
error_regex = r"Missing modules for dependencies .*: toy/\.0.0-deps"
- self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True)
# enable robot, but without passing path required to resolve toy dependency => FAIL
# note that --dry-run is now robust against missing easyconfig, so shouldn't use it here
@@ -3286,11 +3440,13 @@ def test_robot(self):
eb_file,
'--robot',
]
- self.assertErrorRegex(EasyBuildError, 'Missing dependencies', self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, 'Missing dependencies', self.eb_main, args, raise_error=True)
# add path to test easyconfigs to robot paths, so dependencies can be resolved
args.append('--dry-run')
- self.eb_main(args + ['--robot-paths=%s' % test_ecs_path], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + ['--robot-paths=%s' % test_ecs_path], raise_error=True)
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
@@ -3302,7 +3458,8 @@ def test_robot(self):
del os.environ['EASYBUILD_ROBOT_PATHS']
orig_sys_path = sys.path[:]
sys.path.insert(0, tmpdir)
- self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, raise_error=True)
shutil.rmtree(tmpdir)
sys.path[:] = orig_sys_path
@@ -3314,7 +3471,8 @@ def test_robot(self):
'--robot-paths=%s' % os.path.join(tmpdir, 'easybuild', 'easyconfigs'),
'--dry-run',
]
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
ecfiles = [
'g/GCC/GCC-4.6.3.eb',
@@ -3334,34 +3492,40 @@ def test_robot_path_check(self):
error_pattern = "Argument passed to --robot is not an existing directory"
for robot in ['--robot=foo', '--robot=%s' % empty_file]:
args = ['toy-0.0.eb', '--dry-run', robot]
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
toy_regex = re.compile('module: toy/0.0')
# works fine is directory exists
args = ['toy-0.0.eb', '-r', self.test_prefix, '--dry-run']
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(toy_regex.search(outtxt), "Pattern '%s' not found in: %s" % (toy_regex.pattern, outtxt))
# no error when name of an easyconfig file is specified to --robot (even if it doesn't exist)
args = ['--dry-run', '--robot', 'toy-0.0.eb']
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(toy_regex.search(outtxt), "Pattern '%s' not found in: %s" % (toy_regex.pattern, outtxt))
# different error when a non-existing easyconfig file is specified to --robot
args = ['--dry-run', '--robot', 'no_such_easyconfig_file_in_robot_search_path.eb']
error_pattern = "One or more files not found: no_such_easyconfig_file_in_robot_search_path.eb"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True)
for robot in ['-r%s' % self.test_prefix, '--robot=%s' % self.test_prefix]:
args = ['toy-0.0.eb', '--dry-run', robot]
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(toy_regex.search(outtxt), "Pattern '%s' not found in: %s" % (toy_regex.pattern, outtxt))
# no problem with using combos of single-letter options with -r included, no matter the order
for arg in ['-Dr', '-rD', '-frkD', '-rfDk']:
args = ['toy-0.0.eb', arg]
- outtxt = self.eb_main(args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(toy_regex.search(outtxt), "Pattern '%s' not found in: %s" % (toy_regex.pattern, outtxt))
# unknown options are still recognized, even when used in single-letter combo arguments
@@ -3377,7 +3541,8 @@ def test_missing_cfgfile(self):
"""Test behaviour when non-existing config file is specified."""
args = ['--configfiles=/no/such/cfgfile.foo']
error_regex = "parseconfigfiles: configfile .* not found"
- self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True)
def test_show_default_moduleclasses(self):
"""Test --show-default-moduleclasses."""
@@ -3389,7 +3554,8 @@ def test_show_default_moduleclasses(self):
'--show-default-moduleclasses',
]
write_file(self.logfile, '')
- self.eb_main(args, logfile=dummylogfn, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, verbose=True)
logtxt = read_file(self.logfile)
lst = ["\t%s:[ ]*%s" % (c, d.replace('(', '\\(').replace(')', '\\)')) for (c, d) in DEFAULT_MODULECLASSES]
@@ -3424,12 +3590,13 @@ def test_show_default_configfiles(self):
'',
"* user-level: ${XDG_CONFIG_HOME:-$HOME/.config}/easybuild/config.cfg",
" -> %s",
- "* system-level: ${XDG_CONFIG_DIRS:-/etc}/easybuild.d/*.cfg",
+ "* system-level: ${XDG_CONFIG_DIRS:-/etc/xdg}/easybuild.d/*.cfg",
" -> %s/easybuild.d/*.cfg => ",
])
write_file(self.logfile, '')
- self.eb_main(args, logfile=dummylogfn, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, verbose=True)
logtxt = read_file(self.logfile)
homecfgfile = os.path.join(os.environ['HOME'], '.config', 'easybuild', 'config.cfg')
@@ -3438,18 +3605,19 @@ def test_show_default_configfiles(self):
homecfgfile_str += " => found"
else:
homecfgfile_str += " => not found"
- expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}')
+ expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc/xdg}')
self.assertIn(expected, logtxt)
# to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS
os.environ['HOME'] = self.test_prefix
- xdg_config_dirs = os.path.join(self.test_prefix, 'etc')
+ xdg_config_dirs = os.path.join(self.test_prefix, 'etc', 'xdg')
os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs
expected_tmpl += '\n'.join([
"%s",
'',
- "Default list of existing configuration files (%d): %s",
+ "Default list of existing configuration files (%d, most important last):",
+ "%s",
])
# put dummy cfgfile in place in $HOME (to predict last line of output which only lists *existing* files)
@@ -3459,7 +3627,8 @@ def test_show_default_configfiles(self):
reload(easybuild.tools.options)
write_file(self.logfile, '')
- self.eb_main(args, logfile=dummylogfn, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, verbose=True)
logtxt = read_file(self.logfile)
expected = expected_tmpl % ('(not set)', xdg_config_dirs, "%s => found" % homecfgfile, '{%s}' % xdg_config_dirs,
'(no matches)', 1, homecfgfile)
@@ -3467,12 +3636,12 @@ def test_show_default_configfiles(self):
xdg_config_home = os.path.join(self.test_prefix, 'home')
os.environ['XDG_CONFIG_HOME'] = xdg_config_home
- xdg_config_dirs = [os.path.join(self.test_prefix, 'etc'), os.path.join(self.test_prefix, 'moaretc')]
+ xdg_config_dirs = [os.path.join(self.test_prefix, 'moaretc'), os.path.join(self.test_prefix, 'etc', 'xdg')]
os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join(xdg_config_dirs)
# put various dummy cfgfiles in place
cfgfiles = [
- os.path.join(self.test_prefix, 'etc', 'easybuild.d', 'config.cfg'),
+ os.path.join(self.test_prefix, 'etc', 'xdg', 'easybuild.d', 'config.cfg'),
os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'bar.cfg'),
os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'foo.cfg'),
os.path.join(xdg_config_home, 'easybuild', 'config.cfg'),
@@ -3483,12 +3652,13 @@ def test_show_default_configfiles(self):
reload(easybuild.tools.options)
write_file(self.logfile, '')
- self.eb_main(args, logfile=dummylogfn, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, verbose=True)
logtxt = read_file(self.logfile)
expected = expected_tmpl % (xdg_config_home, os.pathsep.join(xdg_config_dirs),
"%s => found" % os.path.join(xdg_config_home, 'easybuild', 'config.cfg'),
'{' + ', '.join(xdg_config_dirs) + '}',
- ', '.join(cfgfiles[:-1]), 4, ', '.join(cfgfiles))
+ ', '.join(cfgfiles[1:3]+[cfgfiles[0]]), 4, ', '.join(cfgfiles))
self.assertIn(expected, logtxt)
del os.environ['XDG_CONFIG_DIRS']
@@ -3556,7 +3726,8 @@ def test_xxx_include_easyblocks(self):
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
test_easyblocks = os.path.dirname(os.path.abspath(__file__))
@@ -3613,7 +3784,8 @@ def __init__(self, *args, **kwargs):
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks',
@@ -3672,7 +3844,8 @@ def test_xxx_include_generic_easyblocks(self):
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
]
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks',
@@ -3717,7 +3890,8 @@ def test_xxx_include_generic_easyblocks(self):
write_file(os.path.join(self.test_prefix, 'generictest.py'), txt)
args[0] = '--include-easyblocks=%s/*.py' % self.test_prefix
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
mod_pattern = 'easybuild.easyblocks.generic.generictest'
@@ -3759,7 +3933,7 @@ def test_github_xxx_include_easyblocks_from_pr(self):
args = [
'--include-easyblocks=%s/*.py' % self.test_prefix, # this shouldn't interfere
- '--include-easyblocks-from-pr=1915', # a PR for CMakeMake easyblock
+ '--include-easyblocks-from-pr=3399', # a PR for CMakeMake easyblock
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
'--github-user=%s' % GITHUB_TEST_ACCOUNT,
@@ -3773,7 +3947,7 @@ def test_github_xxx_include_easyblocks_from_pr(self):
logtxt = read_file(self.logfile)
self.assertFalse(stderr)
- self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #1915\n")
+ self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #3399\n")
# easyblock included from pr is found
path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks')
@@ -3803,7 +3977,7 @@ def test_github_xxx_include_easyblocks_from_pr(self):
# including the same easyblock twice should work and give priority to the one from the PR
args = [
'--include-easyblocks=%s/*.py' % self.test_prefix,
- '--include-easyblocks-from-pr=1915',
+ '--include-easyblocks-from-pr=3399',
'--list-easyblocks=detailed',
'--unittest-file=%s' % self.logfile,
'--github-user=%s' % GITHUB_TEST_ACCOUNT,
@@ -3817,9 +3991,9 @@ def test_github_xxx_include_easyblocks_from_pr(self):
logtxt = read_file(self.logfile)
expected = "WARNING: One or more easyblocks included from multiple locations: "
- expected += "cmakemake.py (the one(s) from PR #1915 will be used)"
+ expected += "cmakemake.py (the one(s) from PR #3399 will be used)"
self.assertEqual(stderr.strip(), expected)
- self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #1915\n")
+ self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #3399\n")
# easyblock included from pr is found
path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks')
@@ -3853,8 +4027,8 @@ def test_github_xxx_include_easyblocks_from_pr(self):
write_file(self.logfile, '')
args = [
- '--from-pr=10487', # PR for CMake easyconfig
- '--include-easyblocks-from-pr=1936,2204', # PRs for EB_CMake and Siesta easyblock
+ '--from-pr=22227', # PR for Togl easyconfig
+ '--include-easyblocks-from-pr=3563,3634', # PRs for ConfigureMake and GROMACS easyblock
'--unittest-file=%s' % self.logfile,
'--github-user=%s' % GITHUB_TEST_ACCOUNT,
'--extended-dry-run',
@@ -3868,25 +4042,32 @@ def test_github_xxx_include_easyblocks_from_pr(self):
logtxt = read_file(self.logfile)
self.assertFalse(stderr)
- self.assertEqual(stdout, "== easyblock cmake.py included from PR #1936\n" +
- "== easyblock siesta.py included from PR #2204\n")
+ self.assertEqual(stdout, "== easyblock configuremake.py included from PR #3563\n" +
+ "== easyblock gromacs.py included from PR #3634\n")
# easyconfig from pr is found
- ec_pattern = os.path.join(self.test_prefix, '.*', 'files_pr10487', 'c', 'CMake',
- 'CMake-3.16.4-GCCcore-9.3.0.eb')
+ ec_pattern = os.path.join(self.test_prefix, '.*', 'files_pr22227', 't', 'Togl',
+ 'Togl-2.0-GCCcore-13.3.0.eb')
ec_regex = re.compile(r"Parsing easyconfig file %s" % ec_pattern, re.M)
self.assertTrue(ec_regex.search(logtxt), "Pattern '%s' found in: %s" % (ec_regex.pattern, logtxt))
# easyblock included from pr is found
- eb_regex = re.compile(r"Successfully obtained EB_CMake class instance from easybuild.easyblocks.cmake", re.M)
+ eb_regex = re.compile(
+ r"Derived full easyblock module path for ConfigureMake: easybuild.easyblocks.generic.configuremake", re.M)
self.assertTrue(eb_regex.search(logtxt), "Pattern '%s' found in: %s" % (eb_regex.pattern, logtxt))
# easyblock is found via get_easyblock_class
- klass = get_easyblock_class('EB_CMake')
+ klass = get_easyblock_class('ConfigureMake')
self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass)
# 'undo' import of easyblocks
- del sys.modules['easybuild.easyblocks.cmake']
+ del sys.modules['easybuild.easyblocks.gromacs']
+ del sys.modules['easybuild.easyblocks.generic.configuremake']
+ sys.path[:] = orig_local_sys_path
+ import easybuild.easyblocks
+ reload(easybuild.easyblocks)
+ import easybuild.easyblocks.generic
+ reload(easybuild.easyblocks.generic)
def mk_eb_test_cmd(self, args):
"""Construct test command for 'eb' with given options."""
@@ -3917,8 +4098,9 @@ def test_include_module_naming_schemes(self):
# try and make sure top-level directory is in $PYTHONPATH if it isn't yet
pythonpath = self.env_pythonpath
- _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False)
- if ec > 0:
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False)
+ if res.exit_code != 0:
pythonpath = '%s:%s' % (topdir, pythonpath)
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
@@ -3932,8 +4114,10 @@ def test_include_module_naming_schemes(self):
# TestIncludedMNS module naming scheme is not available by default
args = ['--avail-module-naming-schemes']
test_cmd = self.mk_eb_test_cmd(args)
- logtxt, _ = run_cmd(test_cmd, simple=False)
- self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd)
+ self.assertFalse(mns_regex.search(res.output),
+ f"Unexpected pattern '{mns_regex.pattern}' found in: {res.output}")
# include extra test MNS
mns_txt = '\n'.join([
@@ -3948,8 +4132,10 @@ def test_include_module_naming_schemes(self):
args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix)
test_cmd = self.mk_eb_test_cmd(args)
- logtxt, _ = run_cmd(test_cmd, simple=False)
- self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd)
+ self.assertTrue(mns_regex.search(res.output),
+ f"Pattern '{mns_regex.pattern}' *not* found in: {res.output}")
def test_use_included_module_naming_scheme(self):
"""Test using an included module naming scheme."""
@@ -3978,11 +4164,13 @@ def test_use_included_module_naming_scheme(self):
# selecting a module naming scheme that doesn't exist leads to 'invalid choice'
error_regex = "Selected module naming scheme \'AnotherTestIncludedMNS\' is unknown"
- self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, logfile=dummylogfn,
- raise_error=True, raise_systemexit=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, logfile=dummylogfn,
+ raise_error=True, raise_systemexit=True)
args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix)
- self.eb_main(args, logfile=dummylogfn, do_build=True, raise_error=True, raise_systemexit=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, do_build=True, raise_error=True, raise_systemexit=True, verbose=True)
toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
toy_mod += '.lua'
@@ -4002,8 +4190,9 @@ def test_include_toolchains(self):
# try and make sure top-level directory is in $PYTHONPATH if it isn't yet
pythonpath = self.env_pythonpath
- _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False)
- if ec > 0:
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False)
+ if res.exit_code != 0:
pythonpath = '%s:%s' % (topdir, pythonpath)
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
@@ -4020,8 +4209,10 @@ def test_include_toolchains(self):
# TestIncludedCompiler is not available by default
args = ['--list-toolchains']
test_cmd = self.mk_eb_test_cmd(args)
- logtxt, _ = run_cmd(test_cmd, simple=False)
- self.assertFalse(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd)
+ self.assertFalse(tc_regex.search(res.output),
+ f"Pattern '{tc_regex.pattern}' *not* found in: {res.output}")
# include extra test toolchain
comp_txt = '\n'.join([
@@ -4041,8 +4232,10 @@ def test_include_toolchains(self):
args.append('--include-toolchains=%s/*.py,%s/*/*.py' % (self.test_prefix, self.test_prefix))
test_cmd = self.mk_eb_test_cmd(args)
- logtxt, _ = run_cmd(test_cmd, simple=False)
- self.assertTrue(tc_regex.search(logtxt), "Pattern '%s' found in: %s" % (tc_regex.pattern, logtxt))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd)
+ self.assertTrue(tc_regex.search(res.output),
+ f"Pattern '{tc_regex.pattern}' found in: {res.output}")
def test_cleanup_tmpdir(self):
"""Test --cleanup-tmpdir."""
@@ -4106,17 +4299,17 @@ def test_github_review_pr(self):
self.mock_stdout(True)
self.mock_stderr(True)
- # PR for gzip 1.10 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/9921
+ # PR for bwidget 1.10.1 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
args = [
'--color=never',
'--github-user=%s' % GITHUB_TEST_ACCOUNT,
- '--review-pr=9921',
+ '--review-pr=22227',
]
self.eb_main(args, raise_error=True)
txt = self.get_stdout()
self.mock_stdout(False)
self.mock_stderr(False)
- regex = re.compile(r"^Comparing gzip-1.10-\S* with gzip-1.10-")
+ regex = re.compile(r"^Comparing bwidget-1.10.1-\S* with bwidget-")
self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt))
self.mock_stdout(True)
@@ -4248,6 +4441,7 @@ def test_minimal_toolchains(self):
'--minimal-toolchains',
'--module-naming-scheme=HierarchicalMNS',
'--dry-run',
+ '--robot',
]
self.mock_stdout(True)
self.eb_main(args, do_build=True, raise_error=True, testing=False)
@@ -4263,17 +4457,9 @@ def test_extended_dry_run(self):
ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
args = [
ec_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
+ '--disable-rpath',
]
- # *no* output in testing mode (honor 'silent')
- self.mock_stdout(True)
- self.eb_main(args + ['--extended-dry-run'], do_build=True, raise_error=True, testing=True)
- stdout = self.get_stdout()
- self.mock_stdout(False)
- self.assertEqual(len(stdout), 0)
msg_regexs = [
re.compile(r"the actual build \& install procedure that will be performed may diverge", re.M),
@@ -4322,7 +4508,8 @@ def test_last_log(self):
# run something that fails first, we need a log file to find
last_log_path = os.path.join(tempfile.gettempdir(), 'eb-tmpdir0', 'easybuild-last.log')
mkdir(os.path.dirname(last_log_path))
- self.eb_main(['thisisaneasyconfigthatdoesnotexist.eb'], logfile=last_log_path, raise_error=False)
+ with self.mocked_stdout_stderr():
+ self.eb_main(['thisisaneasyconfigthatdoesnotexist.eb'], logfile=last_log_path, raise_error=False)
# $TMPDIR determines path to build log, we need to get it right to make the test check what we want it to
os.environ['TMPDIR'] = tmpdir
@@ -4373,14 +4560,14 @@ def _assert_regexs(self, regexs, txt, assert_true=True):
self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt))
def _run_mock_eb(self, args, strip=False, **kwargs):
- """Helper function to mock easybuild runs"""
- self.mock_stdout(True)
- self.mock_stderr(True)
- self.eb_main(args, **kwargs)
- stdout_txt = self.get_stdout()
- stderr_txt = self.get_stderr()
- self.mock_stdout(False)
- self.mock_stderr(False)
+ """Helper function to mock easybuild runs
+
+ Return (stdout, stderr) optionally stripped of whitespace at start/end
+ """
+ with self.mocked_stdout_stderr() as (stdout, stderr):
+ self.eb_main(args, **kwargs)
+ stdout_txt = stdout.getvalue()
+ stderr_txt = stderr.getvalue()
if strip:
stdout_txt = stdout_txt.strip()
stderr_txt = stderr_txt.strip()
@@ -4465,7 +4652,7 @@ def test_new_branch_github(self):
def test_github_new_pr_from_branch(self):
"""Test --new-pr-from-branch."""
if self.github_token is None:
- print("Skipping test_new_pr_from_branch, no GitHub token available?")
+ print("Skipping test_github_new_pr_from_branch, no GitHub token available?")
return
# see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE
@@ -4497,8 +4684,8 @@ def test_github_new_pr_from_branch(self):
r"^\* from: boegel/easybuild-easyconfigs:test_new_pr_from_branch_DO_NOT_REMOVE$",
r'^\* title: "\{tools\}\[system/system\] toy v0\.0"$',
r'^"an easyconfig for toy"$',
- r"^ 1 file changed, 32 insertions\(\+\)$",
- r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | 32",
+ r"^ 1 file changed, [0-9]+ insertions\(\+\)$",
+ r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | [0-9]+",
]
self._assert_regexs(regexs, txt)
@@ -4526,7 +4713,7 @@ def test_update_branch_github(self):
r"^== fetching branch 'develop' from https://github.com/%s.git\.\.\." % full_repo,
r"^== copying files to .*/git-working-dir.*/easybuild-easyconfigs...",
r"^== pushing branch 'develop' to remote '.*' \(git@github.com:%s.git\) \[DRY RUN\]" % full_repo,
- r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| 32",
+ r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| [0-9]+",
r"== pushed updated branch 'develop' to boegel/easybuild-easyconfigs \[DRY RUN\]",
]
self._assert_regexs(regexs, txt)
@@ -4599,7 +4786,7 @@ def test_github_new_update_pr(self):
'--git-working-dirs-path=%s' % git_working_dir,
':bzip2-1.0.6.eb',
])
- error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+ error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\nDeleted: bzip2-1.0.6.eb"
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True, testing=False)
@@ -4678,7 +4865,8 @@ def test_github_new_update_pr(self):
gcc_ec,
'-D',
]
- error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+ error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\n"
+ error_msg += "Modified: " + os.path.basename(gcc_ec)
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
self.mock_stdout(False)
@@ -5059,25 +5247,25 @@ def test_github_merge_pr(self):
# note: we frequently need to change to a more recent PR here,
# to avoid that this test starts failing because commit status is set to None for old commits
del args[-1]
- # easyconfig PR for EasyBuild v4.8.2
- args[1] = '19105'
+ # easyconfig PR for EasyBuild v5.0.0
+ args[1] = '22405'
stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
expected_stdout = '\n'.join([
- "Checking eligibility of easybuilders/easybuild-easyconfigs PR #19105 for merging...",
+ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #22405 for merging...",
"* targets develop branch: OK",
"* test suite passes: OK",
"* last test report is successful: OK",
"* no pending change requests: OK",
- "* approved review: OK (by SebastianAchilles)",
- "* milestone is set: OK (4.9.0)",
+ "* approved review: OK (by verdurin)",
+ "* milestone is set: OK (5.0.0)",
"* mergeable state is clean: PR is already merged",
'',
"Review OK, merging pull request!",
'',
- "[DRY RUN] Adding comment to easybuild-easyconfigs issue #19105: 'Going in, thanks @boegel!'",
- "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #19105",
+ "[DRY RUN] Adding comment to easybuild-easyconfigs issue #22405: 'Going in, thanks @PetrKralCZ!'",
+ "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #22405",
])
expected_stderr = ''
self.assertEqual(stderr.strip(), expected_stderr)
@@ -5086,7 +5274,7 @@ def test_github_merge_pr(self):
# --merge-pr also works on easyblocks (& framework) PRs
args = [
'--merge-pr',
- '2995',
+ '3582',
'--pr-target-repo=easybuild-easyblocks',
'-D',
'--github-user=%s' % GITHUB_TEST_ACCOUNT,
@@ -5094,12 +5282,12 @@ def test_github_merge_pr(self):
stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
self.assertEqual(stderr.strip(), '')
expected_stdout = '\n'.join([
- "Checking eligibility of easybuilders/easybuild-easyblocks PR #2995 for merging...",
+ "Checking eligibility of easybuilders/easybuild-easyblocks PR #3582 for merging...",
"* targets develop branch: OK",
"* test suite passes: OK",
"* no pending change requests: OK",
- "* approved review: OK (by boegel)",
- "* milestone is set: OK (4.8.1)",
+ "* approved review: OK (by hajgato)",
+ "* milestone is set: OK (5.0.0)",
"* mergeable state is clean: PR is already merged",
'',
"Review OK, merging pull request!",
@@ -5116,7 +5304,8 @@ def test_github_empty_pr(self):
full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO,
'develop/easybuild/easyconfigs/z/zlib/zlib-1.2.11-GCCcore-6.4.0.eb'])
ec_fn = os.path.basename(full_url)
- ec = download_file(ec_fn, full_url, path=os.path.join(self.test_prefix, ec_fn))
+ with self.mocked_stdout_stderr():
+ ec = download_file(ec_fn, full_url, path=os.path.join(self.test_prefix, ec_fn))
# try to open new pr with unchanged file
args = [
@@ -5173,6 +5362,7 @@ def test_show_config(self):
r"installpath\s* \(E\) = " + os.path.join(self.test_prefix, 'tmp.*'),
r"repositorypath\s* \(D\) = " + os.path.join(default_prefix, 'ebfiles_repo'),
r"robot-paths\s* \(E\) = " + os.path.join(test_dir, 'easyconfigs', 'test_ecs'),
+ r"rpath\s* \(D\) = " + ('False' if get_os_type() == DARWIN else 'True'),
r"sourcepath\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'sources'),
r"subdir-modules\s* \(F\) = mods",
]
@@ -5225,19 +5415,19 @@ def test_show_config_cfg_levels(self):
# configuring --modules-tool and --module-syntax on different levels should NOT cause problems
# cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564
- os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC'
+ os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules'
args = [
'--module-syntax=Tcl',
'--show-config',
]
# set init_config to False to avoid that eb_main (called by _run_mock_eb) re-initialises configuration
- # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModulesC conflicts with default module syntax (Lua)
+ # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModules conflicts with default module syntax (Lua)
stdout, _ = self._run_mock_eb(args, raise_error=True, redo_init_config=False)
patterns = [
r"^# Current EasyBuild configuration",
r"^module-syntax\s*\(C\) = Tcl",
- r"^modules-tool\s*\(E\) = EnvironmentModulesC",
+ r"^modules-tool\s*\(E\) = EnvironmentModules",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -5249,8 +5439,8 @@ def test_modules_tool_vs_syntax_check(self):
# make sure default module syntax is used
os.environ.pop('EASYBUILD_MODULE_SYNTAX', None)
- # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem
- os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC'
+ # using EnvironmentModules modules tool with default module syntax (Lua) is a problem
+ os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules'
args = ['--show-full-config']
error_pattern = "Generating Lua module files requires Lmod as modules tool"
self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True)
@@ -5258,10 +5448,10 @@ def test_modules_tool_vs_syntax_check(self):
patterns = [
r"^# Current EasyBuild configuration",
r"^module-syntax\s*\(C\) = Tcl",
- r"^modules-tool\s*\(E\) = EnvironmentModulesC",
+ r"^modules-tool\s*\(E\) = EnvironmentModules",
]
- # EnvironmentModulesC modules tool + Tcl module syntax is fine
+ # EnvironmentModules modules tool + Tcl module syntax is fine
args.append('--module-syntax=Tcl')
stdout, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, redo_init_config=False)
for pattern in patterns:
@@ -5282,7 +5472,14 @@ def test_prefix_option(self):
regex = re.compile(r"(?P\S*).*%s.*" % self.test_prefix, re.M)
- expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath']
+ expected = [
+ 'buildpath',
+ 'containerpath',
+ 'installpath',
+ 'packagepath',
+ 'prefix',
+ 'repositorypath',
+ ]
self.assertEqual(sorted(regex.findall(txt)), expected)
def test_dump_env_script(self):
@@ -5309,7 +5506,8 @@ def test_dump_env_script(self):
os.chdir(self.test_prefix)
args = ['%s.eb' % openmpi, '--dump-env-script']
error_msg = r"Script\(s\) already exists, not overwriting them \(unless --force is used\): %s.env" % openmpi
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, do_build=True, raise_error=True)
os.chdir(self.test_prefix)
args.append('--force')
@@ -5329,13 +5527,14 @@ def test_dump_env_script(self):
regex = re.compile("^%s$" % pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
- out, ec = run_cmd("function module { echo $@; } && source %s && echo FC: $FC" % env_script, simple=False)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"function module {{ echo $@; }} && source {env_script} && echo FC: $FC")
expected_out = '\n'.join([
"load GCC/4.6.4",
"load hwloc/1.11.8-GCC-4.6.4",
"FC: gfortran",
])
- self.assertEqual(out.strip(), expected_out)
+ self.assertEqual(res.output.strip(), expected_out)
def test_dump_env_script_existing_module(self):
toy_ec = 'toy-0.0.eb'
@@ -5384,7 +5583,14 @@ def test_stop(self):
regex = re.compile(r"COMPLETED: Installation STOPPED successfully \(took .* secs?\)", re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+ # 'source' step was renamed to 'extract' in EasyBuild 5.0,
+ # see https://github.com/easybuilders/easybuild-framework/pull/4629
+ args = ['toy-0.0.eb', '--force', '--stop=source']
+ _, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
+ self.assertIn("option --stop: invalid choice: 'source' (choose from", stderr)
+
def test_fetch(self):
+ """Test use of --fetch"""
options = EasyBuildOptions(go_args=['--fetch'])
self.assertTrue(options.options.fetch)
@@ -5405,10 +5611,10 @@ def test_fetch(self):
# which might trip up the dependency resolution (see #4298)
for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'):
args = [ec, '--fetch']
- stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
+ stdout, _ = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
patterns = [
- r"^== fetching files\.\.\.$",
+ r"^== fetching files and verifying checksums\.\.\.$",
r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$",
]
for pattern in patterns:
@@ -5418,6 +5624,17 @@ def test_fetch(self):
regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$")
self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ # --fetch should also verify the checksums
+ tmpdir = tempfile.mkdtemp(prefix='easybuild-sources')
+ write_file(os.path.join(tmpdir, 'toy-0.0.tar.gz'), 'Make checksum check fail')
+ args = ['--sourcepath=%s:%s' % (tmpdir, self.test_sourcepath), '--fetch', 'toy-0.0.eb']
+ with self.mocked_stdout_stderr():
+ pattern = 'Checksum verification for .*/toy-0.0.tar.gz .*failed'
+ self.assertErrorRegex(EasyBuildError, pattern, self.eb_main, args, do_build=True, raise_error=True)
+ # We can avoid that failure by ignoring the checksums
+ args.append('--ignore-checksums')
+ self.eb_main(args, do_build=True, raise_error=True)
+
def test_parse_external_modules_metadata(self):
"""Test parse_external_modules_metadata function."""
# by default, provided external module metadata cfg files are picked up
@@ -5530,7 +5747,8 @@ def test_zip_logs(self):
args = ['toy-0.0.eb', '--force', '--debug']
if zip_logs:
args.append(zip_logs)
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
logs = glob.glob(os.path.join(toy_eb_install_dir, 'easybuild-toy-0.0*log*'))
self.assertEqual(len(logs), 1, "Found exactly 1 log file in %s: %s" % (toy_eb_install_dir, logs))
@@ -5570,19 +5788,23 @@ def test_list_prs(self):
"""Test --list-prs."""
args = ['--list-prs', 'foo']
error_msg = r"must be one of \['open', 'closed', 'all'\]"
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
args = ['--list-prs', 'open,foo']
error_msg = r"must be one of \['created', 'updated', 'popularity', 'long-running'\]"
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
args = ['--list-prs', 'open,created,foo']
error_msg = r"must be one of \['asc', 'desc'\]"
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
args = ['--list-prs', 'open,created,asc,foo']
error_msg = r"must be in the format 'state\[,order\[,direction\]\]"
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
args = ['--list-prs', 'closed,updated,asc']
txt, _ = self._run_mock_eb(args, testing=False)
@@ -5761,14 +5983,9 @@ def test_parse_optarch(self):
def test_check_contrib_style(self):
"""Test style checks performed by --check-contrib + dedicated --check-style option."""
- try:
- import pycodestyle # noqa
- except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- print("Skipping test_check_contrib_style, since pycodestyle or pep8 is not available")
- return
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_contrib_style pycodestyle is not available")
+ return
regex = re.compile(r"Running style check on 2 easyconfig\(s\)(.|\n)*>> All style checks PASSed!", re.M)
args = [
@@ -5821,8 +6038,8 @@ def test_check_contrib_style(self):
def test_check_contrib_non_style(self):
"""Test non-style checks performed by --check-contrib."""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_check_contrib_non_style (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_contrib_non_style pycodestyle is not available")
return
args = [
@@ -5880,7 +6097,8 @@ def test_allow_use_as_root(self):
# running as root is disallowed by default
error_msg = "You seem to be running EasyBuild with root privileges which is not wise, so let's end this here"
- self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, ['toy-0.0.eb'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, ['toy-0.0.eb'], raise_error=True)
# running as root is allowed under --allow-use-as-root, but does result in a warning being printed to stderr
args = ['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences']
@@ -5908,7 +6126,8 @@ def test_verify_easyconfig_filenames(self):
]
# filename of provided easyconfig doesn't matter by default
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
self.assertIn('module: toy/0.0', logtxt)
@@ -5919,12 +6138,15 @@ def test_verify_easyconfig_filenames(self):
error_pattern = r"Easyconfig filename 'test.eb' does not match with expected filename 'toy-0.0.eb' \(specs: "
error_pattern += r"name: 'toy'; version: '0.0'; versionsuffix: ''; "
error_pattern += r"toolchain name, version: 'system', 'system'\)"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, logfile=dummylogfn,
+ raise_error=True)
write_file(self.logfile, '')
args[0] = toy_ec
- self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=dummylogfn, raise_error=True)
logtxt = read_file(self.logfile)
self.assertIn('module: toy/0.0', logtxt)
@@ -5933,7 +6155,8 @@ def test_set_default_module(self):
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')
- self.eb_main([toy_ec, '--set-default-module'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([toy_ec, '--set-default-module'], do_build=True, raise_error=True)
toy_mod_dir = os.path.join(self.test_installpath, 'modules', 'all', 'toy')
toy_mod = os.path.join(toy_mod_dir, '0.0-deps')
@@ -6008,7 +6231,8 @@ def test_set_default_module_robot(self):
'--robot',
self.test_prefix,
]
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
# default module is set for specified easyconfig, but *not* for its dependency
modfiles_dir = os.path.join(self.test_installpath, 'modules', 'all')
@@ -6135,8 +6359,10 @@ def test_inject_checksums(self):
{'toy-0.0_fix-silly-typo-in-printf-statement.patch': toy_patch_sha256}])
self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']})
self.assertEqual(ec['exts_list'][0], 'ulimit')
+ expected_buildopts = " && gcc bar.c -o anotherbar && "
+ expected_buildopts += 'echo "TOY_EXAMPLES=$TOY_EXAMPLES" > %(installdir)s/toy_libs_path.txt'
self.assertEqual(ec['exts_list'][1], ('bar', '0.0', {
- 'buildopts': " && gcc bar.c -o anotherbar",
+ 'buildopts': expected_buildopts,
'checksums': [
{'bar-0.0.tar.gz': bar_tar_gz_sha256},
{'bar-0.0_fix-silly-typo-in-printf-statement.patch': bar_patch_sha256},
@@ -6146,7 +6372,7 @@ def test_inject_checksums(self):
'patches': [bar_patch, bar_patch_bis],
'toy_ext_param': "mv anotherbar bar_bis",
'unknowneasyconfigparameterthatshouldbeignored': 'foo',
- 'keepsymlinks': True,
+ 'keepsymlinks': False,
}))
self.assertEqual(ec['exts_list'][2], ('barbar', '1.2', {
'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838'],
@@ -6191,18 +6417,19 @@ def test_inject_checksums(self):
self.assertNotIn('checksums = ', toy_ec_txt)
write_file(test_ec, toy_ec_txt)
- args = [test_ec, '--inject-checksums=md5']
+ args = [test_ec, '--inject-checksums=sha256']
stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True)
patterns = [
- r"^== injecting md5 checksums in .*/test\.eb$",
+ r"^== injecting sha256 checksums in .*/test\.eb$",
r"^== fetching sources & patches for test\.eb\.\.\.$",
r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$",
- r"^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$",
- r"^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$",
- r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$",
- r"^== \* toy-extra\.txt: 3b0787b3bf36603ae1398c4a49097893$",
+ r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$",
+ r"^== \* toy-0.0\.tar\.gz: 44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc$",
+ r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: " # no comma, continues on next line
+ r"81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487$",
+ r"^== \* toy-extra\.txt: 4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458$",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -6218,9 +6445,10 @@ def test_inject_checksums(self):
# no parse errors for updated easyconfig file...
ec = EasyConfigParser(test_ec).get_config_dict()
checksums = [
- {'toy-0.0.tar.gz': 'be662daa971a640e40be5c804d9d7d10'},
- {'toy-0.0_fix-silly-typo-in-printf-statement.patch': 'a99f2a72cee1689a2f7e3ace0356efb1'},
- {'toy-extra.txt': '3b0787b3bf36603ae1398c4a49097893'},
+ {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'},
+ {'toy-0.0_fix-silly-typo-in-printf-statement.patch':
+ '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'},
+ {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'},
]
self.assertEqual(ec['checksums'], checksums)
@@ -6349,7 +6577,6 @@ def test_force_download(self):
'--sourcepath=%s' % self.test_prefix,
]
stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True)
- self.assertEqual(stdout, '')
regex = re.compile(r"^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$")
self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr))
@@ -6369,14 +6596,15 @@ def test_enforce_checksums(self):
args = [
test_ec,
- '--stop=source',
+ '--stop=fetch',
'--enforce-checksums',
]
# checksum is missing for patch of 'bar' extension, so --enforce-checksums should result in an error
copy_file(toy_ec, test_ec)
error_pattern = r"Missing checksum for bar-0.0[^ ]*\.patch"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
# get rid of checksums for extensions, should result in different error message
# because of missing checksum for source of 'bar' extension
@@ -6385,7 +6613,8 @@ def test_enforce_checksums(self):
self.assertNotIn("'checksums':", test_ec_txt)
write_file(test_ec, test_ec_txt)
error_pattern = r"Missing checksum for bar-0\.0\.tar\.gz"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
# wipe both exts_list and checksums, so we can check whether missing checksum for main source is caught
test_ec_txt = read_file(test_ec)
@@ -6396,7 +6625,8 @@ def test_enforce_checksums(self):
write_file(test_ec, test_ec_txt)
error_pattern = "Missing checksum for toy-0.0.tar.gz"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
def test_show_system_info(self):
"""Test for --show-system-info."""
@@ -6474,14 +6704,12 @@ def test_tmp_logdir(self):
# check log message with --skip for existing module
args = [
toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--force',
'--debug',
'--tmp-logdir=%s' % tmp_logdir,
]
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
tmp_logs = os.listdir(tmp_logdir)
self.assertEqual(len(tmp_logs), 1)
@@ -6509,19 +6737,21 @@ def test_sanity_check_only(self):
write_file(test_ec, test_ec_txt)
# sanity check fails if software was not installed yet
- outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True)
self.assertIn("Sanity check failed", str(error_thrown))
# actually install, then try --sanity-check-only again;
# need to use --force to install toy because module already exists (but installation doesn't)
- self.eb_main([test_ec, '--force'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([test_ec, '--force'], do_build=True, raise_error=True)
args = [test_ec, '--sanity-check-only']
stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False)
skipped = [
- "fetching files",
+ "fetching files and verifying checksums",
"creating build dir, resetting environment",
"unpacking",
"patching",
@@ -6561,7 +6791,8 @@ def test_sanity_check_only(self):
libbarbar = os.path.join(ebroottoy, 'lib', 'libbarbar.a')
move_file(libbarbar, libbarbar + '.moved')
- outtxt, error_thrown = self.eb_main(args + ['--debug'], do_build=True, return_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt, error_thrown = self.eb_main(args + ['--debug'], do_build=True, return_error=True)
error_msg = str(error_thrown)
error_patterns = [
r"Sanity check failed",
@@ -6572,7 +6803,8 @@ def test_sanity_check_only(self):
self.assertTrue(regex.search(error_msg), "Pattern '%s' should be found in: %s" % (regex.pattern, error_msg))
# failing sanity check for extension can be bypassed via --skip-extensions
- outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True)
self.assertIn("Sanity check for toy successful", outtxt)
# restore fail, we want a passing sanity check for the next check
@@ -6597,7 +6829,8 @@ def test_sanity_check_only(self):
"]",
])
write_file(test_ec, test_ec_txt)
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
# also check when using easyblock that enables build_in_installdir in its constructor
test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks')
@@ -6618,7 +6851,8 @@ def test_sanity_check_only(self):
orig_local_sys_path = sys.path[:]
args.append('--include-easyblocks=%s' % toy_eb)
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
# undo import of the toy easyblock, to avoid problems with other tests
del sys.modules['easybuild.easyblocks.toy']
@@ -6650,7 +6884,8 @@ def test_skip_extensions(self):
write_file(test_ec, test_ec_txt)
args = [test_ec, '--force', '--skip-extensions']
- self.eb_main(args, do_build=True, return_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, return_error=True)
toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
@@ -6711,50 +6946,6 @@ def test_installdir(self):
eb = EasyBlock(EasyConfig(toy_ec))
self.assertTrue(eb.installdir.endswith('/software/Core/toy/0.0'))
- def test_sort_looseversions(self):
- """Test sort_looseversions function."""
- # Test twice: With the standard distutils LooseVersion (when available) and with our class
- # Note that our class directly allows sorting but should also work with sort_loosversions
- for use_distutils in (True, False):
- if use_distutils:
- try:
- from distutils.version import LooseVersion as version_class
- except ImportError:
- continue
- else:
- version_class = LooseVersion
-
- with warnings.catch_warnings():
- if use_distutils:
- warnings.simplefilter("ignore", category=DeprecationWarning)
- ver1 = version_class('1.2.3')
- ver2 = version_class('4.5.6')
- ver3 = version_class('1.2.3dev')
- ver4 = version_class('system')
- ver5 = version_class('rc3')
- ver6 = version_class('v1802')
-
- # some versions are included multiple times on purpose,
- # to also test comparison between equal LooseVersion instances
- input = [ver3, ver5, ver1, ver2, ver4, ver6, ver3, ver4, ver1]
- expected = [ver1, ver1, ver3, ver3, ver2, ver5, ver4, ver4, ver6]
- self.assertEqual(sort_looseversions(input), expected)
- if not use_distutils:
- self.assertEqual(sorted(input), expected)
-
- # also test on list of tuples consisting of a LooseVersion instance + a string
- # (as in the list_software_* functions)
- suff1 = ''
- suff2 = '-foo'
- suff3 = '-bar'
- input = [(ver3, suff1), (ver5, suff3), (ver1, suff2), (ver2, suff3), (ver4, suff1),
- (ver6, suff2), (ver3, suff3), (ver4, suff3), (ver1, suff1)]
- expected = [(ver1, suff1), (ver1, suff2), (ver3, suff1), (ver3, suff3), (ver2, suff3),
- (ver5, suff3), (ver4, suff1), (ver4, suff3), (ver6, suff2)]
- self.assertEqual(sort_looseversions(input), expected)
- if not use_distutils:
- self.assertEqual(sorted(input), expected)
-
def test_cuda_compute_capabilities(self):
"""Test --cuda-compute-capabilities configuration option."""
args = ['--cuda-compute-capabilities=3.5,6.2,7.0', '--show-config']
@@ -6872,14 +7063,16 @@ def test_accept_eula_for(self):
# by default, no EULAs are accepted at all
args = [test_ec, '--force']
error_pattern = r"The End User License Agreement \(EULA\) for toy is currently not accepted!"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
toy_modfile += '.lua'
# installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for
for val in ('foo,toy,bar', '.*', 't.y'):
- self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True)
self.assertExists(toy_modfile)
@@ -6888,7 +7081,8 @@ def test_accept_eula_for(self):
# also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software
os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = val
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_modfile)
remove_dir(self.test_installpath)
@@ -6896,36 +7090,10 @@ def test_accept_eula_for(self):
del os.environ['EASYBUILD_ACCEPT_EULA_FOR']
- # also check deprecated --accept-eula configuration option
- self.allow_deprecated_behaviour()
-
- self.mock_stderr(True)
- self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True)
- stderr = self.get_stderr()
- self.mock_stderr(False)
- self.assertIn("Use accept-eula-for configuration setting rather than accept-eula", stderr)
-
- remove_dir(self.test_installpath)
- self.assertNotExists(toy_modfile)
-
- # also via $EASYBUILD_ACCEPT_EULA
- self.mock_stderr(True)
- os.environ['EASYBUILD_ACCEPT_EULA'] = 'toy'
- self.eb_main(args, do_build=True, raise_error=True)
- stderr = self.get_stderr()
- self.mock_stderr(False)
-
- self.assertExists(toy_modfile)
- self.assertIn("Use accept-eula-for configuration setting rather than accept-eula", stderr)
-
- remove_dir(self.test_installpath)
- self.assertNotExists(toy_modfile)
-
# also check accepting EULA via 'accept_eula = True' in easyconfig file
- self.disallow_deprecated_behaviour()
- del os.environ['EASYBUILD_ACCEPT_EULA']
write_file(test_ec, test_ec_txt + '\naccept_eula = True')
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_modfile)
def test_config_abs_path(self):
@@ -7016,7 +7184,8 @@ def test_easystack_wrong_read(self):
toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml')
args = ['--easystack', toy_easystack, '--experimental']
expected_err = "No such file or directory: '%s'" % toy_easystack
- self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True)
# testing basics - end-to-end
# expecting successful build
@@ -7026,7 +7195,8 @@ def test_easystack_basic(self):
toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml')
args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run']
- stdout = self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ stdout = self.eb_main(args, do_build=True, raise_error=True)
patterns = [
r"INFO Building from easystack:",
r"DEBUG Parsed easystack:\n"
@@ -7077,7 +7247,8 @@ def test_easystack_opts(self):
'--easystack', test_es_path,
'--installpath', self.test_installpath,
]
- self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False)
mod_ext = '.lua' if get_module_syntax() == 'Lua' else ''
@@ -7135,7 +7306,8 @@ def test_easystack_easyconfigs_cache(self):
'--dry-run',
'--robot=%s' % self.test_prefix,
]
- stdout = self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False)
+ with self.mocked_stdout_stderr():
+ stdout = self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False)
# check whether libtoy-0.0.eb comes from 2nd
regex = re.compile(r"^ \* \[ \] %s" % libtoy_ec, re.M)
@@ -7252,6 +7424,15 @@ def test_opts_dict_to_eb_opts(self):
]
self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+ # multi-call options
+ opts_dict = {'try-amend': ['a=1', 'b=2', 'c=3']}
+ expected = ['--try-amend=a=1', '--try-amend=b=2', '--try-amend=c=3']
+ self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+
+ opts_dict = {'amend': ['a=1', 'b=2', 'c=3']}
+ expected = ['--amend=a=1', '--amend=b=2', '--amend=c=3']
+ self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/output.py b/test/framework/output.py
index 4429fe29ee..d8b43c8c55 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,7 +35,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_output_style, update_build_option
from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, PROGRESS_BAR_TYPES
-from easybuild.tools.output import DummyRich, colorize, get_progress_bar, show_progress_bars
+from easybuild.tools.output import DummyRich, colorize, get_progress_bar, print_error, show_progress_bars
from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich
try:
@@ -139,6 +139,26 @@ def test_colorize(self):
self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor')
+ def test_print_error(self):
+ """
+ Test print_error function
+ """
+ msg = "This is yellow: " + colorize("a banana", color='yellow')
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ print_error(msg)
+ stderr = self.get_stderr()
+ stdout = self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertEqual(stdout, '')
+ if HAVE_RICH:
+ # when using Rich, message printed to stderr won't have funny terminal escape characters for the color
+ expected = '\n\nThis is yellow: a banana\n\n'
+ else:
+ expected = '\nThis is yellow: \x1b[1;33ma banana\x1b[0m\n\n'
+ self.assertEqual(stderr, expected)
+
def test_get_progress_bar(self):
"""
Test get_progress_bar.
diff --git a/test/framework/package.py b/test/framework/package.py
index b839fb32bb..d230ae265e 100644
--- a/test/framework/package.py
+++ b/test/framework/package.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -44,7 +44,7 @@
FPM_OUTPUT_FILE = 'fpm_mocked.out'
-# purposely using non-bash script, to detect issues with shebang line being ignored (run_cmd with shell=False)
+# purposely using non-bash script, to detect issues with shebang line being ignored (run_shell_cmd with use_bash=False)
MOCKED_FPM = """#!/usr/bin/env python
import os, sys
@@ -199,6 +199,7 @@ def test_active_pns(self):
def test_package(self):
"""Test package function."""
+ self.mock_stdout(True)
build_options = {
'package_tool_options': '--foo bar',
'silent': True,
@@ -265,6 +266,7 @@ def test_package(self):
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
regex_pkg = re.compile(r"""DESCRIPTION:.*\nand newlines""", re.MULTILINE)
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
+ self.mock_stdout(False)
def suite():
diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py
index 4e75672815..90c4d0df1c 100644
--- a/test/framework/parallelbuild.py
+++ b/test/framework/parallelbuild.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -112,6 +112,7 @@ class ParallelBuildTest(EnhancedTestCase):
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
+ self.mock_stdout(True)
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
@@ -212,6 +213,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self):
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
+ self.mock_stdout(False)
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
@@ -221,6 +223,8 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
print("GC3Pie not available, skipping test")
return
+ self.allow_deprecated_behaviour()
+
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
@@ -260,7 +264,9 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
- build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
+
+ with self.mocked_stdout_stderr():
+ build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
@@ -278,12 +284,13 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
ecs = resolve_dependencies(process_easyconfig(test_ecfile), self.modtool)
error = "1 jobs failed: toy-1.2.3"
- self.assertErrorRegex(EasyBuildError, error, build_easyconfigs_in_parallel, cmd, ecs, prepare_first=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error, build_easyconfigs_in_parallel, cmd, ecs, prepare_first=False)
def test_submit_jobs(self):
"""Test submit_jobs"""
test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
- toy_ec = os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')
+ toy_ec = process_easyconfig(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb'))
args = [
'--debug',
@@ -296,7 +303,7 @@ def test_submit_jobs(self):
'--job-cores=3',
]
eb_go = parse_options(args=args)
- cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
+ cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True)
# these patterns must be found
regexs = [
@@ -324,7 +331,7 @@ def test_submit_jobs(self):
# test again with custom EasyBuild command to use in jobs
update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug")
- cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
+ cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True)
regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ")
self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd))
diff --git a/test/framework/repository.py b/test/framework/repository.py
index feb96a13cf..aeb4f97155 100644
--- a/test/framework/repository.py
+++ b/test/framework/repository.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,7 +35,6 @@
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
from unittest import TextTestRunner
-import easybuild.tools.build_log
from easybuild.framework.easyconfig.parser import EasyConfigParser
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file
@@ -43,7 +42,7 @@
from easybuild.tools.repository.gitrepo import GitRepository
from easybuild.tools.repository.svnrepo import SvnRepository
from easybuild.tools.repository.repository import init_repository
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.version import VERSION
@@ -94,10 +93,11 @@ def test_gitrepo(self):
# filepath
tmpdir = tempfile.mkdtemp()
cmd = "cd %s && git clone --bare %s" % (tmpdir, test_repo_url)
- _, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
# skip remainder of test if creating bare git repo didn't work
- if ec == 0:
+ if res.exit_code == 0:
repo = GitRepository(os.path.join(tmpdir, 'testrepository.git'))
repo.init()
toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
@@ -163,23 +163,6 @@ def check_ec(path, expected_buildstats):
path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}])
check_ec(path, [{'time': 0.9, 'size': 2}, {'time': 1.23, 'size': 123}])
- orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
- easybuild.tools.build_log.EXPERIMENTAL = True
-
- if 'yaml' in sys.modules:
- toy_yeb_file = os.path.join(test_easyconfigs, 'yeb', 'toy-0.0.yeb')
- path = repo.add_easyconfig(toy_yeb_file, 'test', '1.0', {'time': 1.23}, None)
- check_ec(path, [{'time': 1.23}])
-
- stats1 = {'time': 1.23, 'size': 123}
- stats2 = [{'time': 0.9, 'size': 2}]
- path = repo.add_easyconfig(toy_yeb_file, 'test', '1.0', stats1, stats2)
- check_ec(path, stats2 + [stats1])
-
- easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
- else:
- print("Skipping .yeb part of test_add_easyconfig (no PyYAML available)")
-
def tearDown(self):
"""Clean up after test."""
super(RepositoryTest, self).tearDown()
diff --git a/test/framework/robot.py b/test/framework/robot.py
index bbfe8e2f03..746d71639e 100644
--- a/test/framework/robot.py
+++ b/test/framework/robot.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -783,9 +783,9 @@ def test_github_det_easyconfig_paths_from_pr(self):
args = [
os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'),
test_ec, # relative path, should be resolved via robot search path
- # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files
- '--from-pr=6424',
- 'FFTW-3.3.8-gompi-2018b.eb',
+ # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227
+ '--from-pr=22227',
+ 'XCrySDen-1.6.2-foss-2024a.eb',
'gompi-2018b-test.eb', # relative path, available in robot search path
'--dry-run',
'--robot',
@@ -796,8 +796,10 @@ def test_github_det_easyconfig_paths_from_pr(self):
]
self.mock_stderr(True)
+ self.mock_stdout(True)
outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
self.mock_stderr(False)
+ self.mock_stdout(False)
# full path doesn't matter (helps to avoid failing tests due to resolved symlinks)
test_ecs_path = os.path.join('.*', 'test', 'framework', 'easyconfigs', 'test_ecs')
@@ -807,9 +809,9 @@ def test_github_det_easyconfig_paths_from_pr(self):
(self.test_prefix, 'intel/2018a'), # dependency, found in robot search path
(self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path
(self.test_prefix, 'gompi/2018b-test'), # specified easyconfig, found in robot search path
- ('.*/files_pr6424', 'FFTW/3.3.8-gompi-2018b'), # specified easyconfig
- (test_ecs_path, 'gompi/2018b'), # part of PR easyconfigs, found in robot search path
- (test_ecs_path, 'GCC/7.3.0-2.30'), # dependency for PR easyconfigs, found in robot search path
+ ('.*/files_pr22227', 'XCrySDen/1.6.2-foss-2024a'), # specified easyconfig
+ ('.*/files_pr22227', 'Togl/2.0-GCCcore-13.3.0'), # part of PR easyconfigs, found in robot search path
+ ('.*/files_pr22227', 'GCC/13.3.0'), # dependency for PR easyconfigs, found in robot search path
]
for path_prefix, module in modules:
ec_fn = "%s.eb" % '-'.join(module.split('/'))
@@ -1137,7 +1139,8 @@ def test_tweak_robotpath(self):
# Tweak the toolchain version of the easyconfig
tweak_specs = {'toolchain_version': '6.4.0-2.28'}
- easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths)
+ easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths,
+ return_map=True)
# Check that all expected tweaked easyconfigs exists
tweaked_openmpi = os.path.join(tweaked_ecs_paths[0], 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb')
@@ -1153,6 +1156,10 @@ def test_tweak_robotpath(self):
# Check it picks up the untweaked dependency of the tweaked OpenMPI
untweaked_hwloc = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')
self.assertIn(untweaked_hwloc, specs)
+ # Check correctness of tweak_map (maps back to the original untweaked file, even for hwloc, where the
+ # tweaked version is generated but not used)
+ self.assertEqual(tweak_map, {tweaked_openmpi: untweaked_openmpi,
+ tweaked_hwloc: untweaked_hwloc.replace("6.4.0-2.28", "4.6.4")})
def test_robot_find_subtoolchain_for_dep(self):
"""Test robot_find_subtoolchain_for_dep."""
diff --git a/test/framework/run.py b/test/framework/run.py
index bb2e5c0558..1c5cf3a422 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -1,6 +1,6 @@
# #
# -*- coding: utf-8 -*-
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -24,7 +24,7 @@
# along with EasyBuild. If not, see .
# #
"""
-Unit tests for filetools.py
+Unit tests for run.py
@author: Toon Willems (Ghent University)
@author: Kenneth Hoste (Ghent University)
@@ -35,25 +35,28 @@
import os
import re
import signal
+import string
import stat
import subprocess
import sys
import tempfile
import textwrap
import time
+from concurrent.futures import ThreadPoolExecutor
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
-from unittest import TextTestRunner
+from unittest import TextTestRunner, mock
from easybuild.base.fancylogger import setLogLevelDebug
import easybuild.tools.asyncprocess as asyncprocess
import easybuild.tools.utilities
from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging
from easybuild.tools.config import update_build_option
-from easybuild.tools.filetools import adjust_permissions, read_file, write_file
-from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process
-from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa
+from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, remove_dir, write_file
+from easybuild.tools.modules import EnvironmentModules, Lmod
+from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors
+from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error
+from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate
from easybuild.tools.config import ERROR, IGNORE, WARN
-from easybuild.tools.py2vs3 import subprocess_terminate
class RunTest(EnhancedTestCase):
@@ -74,6 +77,9 @@ def tearDown(self):
def test_get_output_from_process(self):
"""Test for get_output_from_process utility function."""
+ # use of get_output_from_process is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
@contextlib.contextmanager
def get_proc(cmd, asynchronous=False):
if asynchronous:
@@ -90,57 +96,67 @@ def get_proc(cmd, asynchronous=False):
subprocess_terminate(proc, timeout=1)
# get all output at once
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc)
- self.assertEqual(out, 'hello\n')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc)
+ self.assertEqual(out, 'hello\n')
# first get 100 bytes, then get the rest all at once
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc, read_size=100)
- self.assertEqual(out, 'hello\n')
- out = get_output_from_process(proc)
- self.assertEqual(out, '')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc, read_size=100)
+ self.assertEqual(out, 'hello\n')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '')
# get output in small bits, keep trying to get output (which shouldn't fail)
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc, read_size=1)
- self.assertEqual(out, 'h')
- out = get_output_from_process(proc, read_size=3)
- self.assertEqual(out, 'ell')
- out = get_output_from_process(proc, read_size=2)
- self.assertEqual(out, 'o\n')
- out = get_output_from_process(proc, read_size=1)
- self.assertEqual(out, '')
- out = get_output_from_process(proc, read_size=10)
- self.assertEqual(out, '')
- out = get_output_from_process(proc)
- self.assertEqual(out, '')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc, read_size=1)
+ self.assertEqual(out, 'h')
+ out = get_output_from_process(proc, read_size=3)
+ self.assertEqual(out, 'ell')
+ out = get_output_from_process(proc, read_size=2)
+ self.assertEqual(out, 'o\n')
+ out = get_output_from_process(proc, read_size=1)
+ self.assertEqual(out, '')
+ out = get_output_from_process(proc, read_size=10)
+ self.assertEqual(out, '')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '')
# can also get output asynchronously (read_size is *ignored* in that case)
async_cmd = "echo hello; read reply; echo $reply"
- with get_proc(async_cmd, asynchronous=True) as proc:
- out = get_output_from_process(proc, asynchronous=True)
- self.assertEqual(out, 'hello\n')
- asyncprocess.send_all(proc, 'test123\n')
- out = get_output_from_process(proc)
- self.assertEqual(out, 'test123\n')
+ with self.mocked_stdout_stderr():
+ with get_proc(async_cmd, asynchronous=True) as proc:
+ out = get_output_from_process(proc, asynchronous=True)
+ self.assertEqual(out, 'hello\n')
+ asyncprocess.send_all(proc, 'test123\n')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, 'test123\n')
- with get_proc(async_cmd, asynchronous=True) as proc:
- out = get_output_from_process(proc, asynchronous=True, read_size=1)
- # read_size is ignored when getting output asynchronously, we're getting more than 1 byte!
- self.assertEqual(out, 'hello\n')
- asyncprocess.send_all(proc, 'test123\n')
- out = get_output_from_process(proc, read_size=3)
- self.assertEqual(out, 'tes')
- out = get_output_from_process(proc, read_size=2)
- self.assertEqual(out, 't1')
- out = get_output_from_process(proc)
- self.assertEqual(out, '23\n')
+ with self.mocked_stdout_stderr():
+ with get_proc(async_cmd, asynchronous=True) as proc:
+ out = get_output_from_process(proc, asynchronous=True, read_size=1)
+ # read_size is ignored when getting output asynchronously, we're getting more than 1 byte!
+ self.assertEqual(out, 'hello\n')
+ asyncprocess.send_all(proc, 'test123\n')
+ out = get_output_from_process(proc, read_size=3)
+ self.assertEqual(out, 'tes')
+ out = get_output_from_process(proc, read_size=2)
+ self.assertEqual(out, 't1')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '23\n')
def test_run_cmd(self):
"""Basic test for run_cmd function."""
- (out, ec) = run_cmd("echo hello")
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello")
self.assertEqual(out, "hello\n")
# no reason echo hello could fail
self.assertEqual(ec, 0)
@@ -150,18 +166,234 @@ def test_run_cmd(self):
# this is constructed to reproduce errors like:
# UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
# UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018'
- for text in [b"foo \xe2 bar", b"foo \u2018 bar"]:
+ for text in [b"foo \xe2 bar", "foo \u2018 bar"]:
test_file = os.path.join(self.test_prefix, 'foo.txt')
write_file(test_file, text)
cmd = "cat %s" % test_file
- (out, ec) = run_cmd(cmd)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd(cmd)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith('foo ') and out.endswith(' bar'))
self.assertEqual(type(out), str)
+ def test_run_shell_cmd_basic(self):
+ """Basic test for run_shell_cmd function."""
+
+ os.environ['FOOBAR'] = 'foobar'
+
+ cwd = change_dir(self.test_prefix)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("echo hello")
+ self.assertEqual(res.output, "hello\n")
+ # no reason echo hello could fail
+ self.assertEqual(res.cmd, "echo hello")
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(isinstance(res.output, str))
+ self.assertEqual(res.stderr, None)
+ self.assertTrue(res.work_dir and isinstance(res.work_dir, str))
+
+ change_dir(cwd)
+ del os.environ['FOOBAR']
+
+ # check on helper scripts that were generated for this command
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+
+ # check on env.sh script that can be used to set up environment in which command was run
+ env_script = os.path.join(cmd_tmpdir, 'env.sh')
+ self.assertExists(env_script)
+ env_script_txt = read_file(env_script)
+ self.assertIn("export FOOBAR=foobar", env_script_txt)
+ self.assertIn("history -s 'echo hello'", env_script_txt)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"source {env_script}; echo $USER; echo $FOOBAR; history")
+ self.assertEqual(res.exit_code, 0)
+ user = os.getenv('USER')
+ self.assertTrue(res.output.startswith(f'{user}\nfoobar\n'))
+ self.assertTrue(res.output.endswith("echo hello\n"))
+
+ # check on cmd.sh script that can be used to create interactive shell environment for command
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ cmd = f"{cmd_script} -c 'echo pwd: $PWD; echo $FOOBAR; echo $EB_CMD_OUT_FILE; cat $EB_CMD_OUT_FILE'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ regex = re.compile("pwd: .*\nfoobar\n.*/echo-.*/out.txt\nhello$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
+ # check whether working directory is what's expected
+ regex = re.compile('^pwd: .*', re.M)
+ res = regex.findall(res.output)
+ self.assertEqual(len(res), 1)
+ pwd = res[0].strip()[5:]
+ self.assertTrue(os.path.samefile(pwd, self.test_prefix))
+
+ cmd = f"{cmd_script} -c 'module --version'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+
+ if isinstance(self.modtool, Lmod):
+ regex = re.compile("^Modules based on Lua: Version [0-9]", re.M)
+ elif isinstance(self.modtool, EnvironmentModules):
+ regex = re.compile("^Modules Release [0-9]", re.M)
+ else:
+ self.fail("Unknown modules tool used!")
+
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
+ # test running command that emits non-UTF-8 characters
+ # this is constructed to reproduce errors like:
+ # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
+ # UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018'
+ # (such errors are ignored by the 'run' implementation)
+ for text in [b"foo \xe2 bar", "foo \u2018 bar"]:
+ test_file = os.path.join(self.test_prefix, 'foo.txt')
+ write_file(test_file, text)
+ cmd = "cat %s" % test_file
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.startswith('foo ') and res.output.endswith(' bar'))
+ self.assertTrue(isinstance(res.output, str))
+ self.assertTrue(res.work_dir and isinstance(res.work_dir, str))
+
+ def test_run_shell_cmd_perl(self):
+ """
+ Test running of Perl script via run_shell_cmd that detects type of shell
+ """
+ perl_script = os.path.join(self.test_prefix, 'test.pl')
+ perl_script_txt = """#!/usr/bin/perl
+
+ # wait for input, see what happens (should not hang)
+ print STDOUT "foo:\n";
+ STDOUT->autoflush(1);
+ my $stdin = ;
+ print "stdin: $stdin\n";
+
+ # conditional print statements below should *not* be triggered
+ print "stdin+stdout are terminals\n" if -t STDIN && -t STDOUT;
+ print "stdin is terminal\n" if -t STDIN;
+ print "stdout is terminal\n" if -t STDOUT;
+ my $ISA_TTY = -t STDIN && (-t STDOUT || !(-f STDOUT || -c STDOUT)) ;
+ print "ISA_TTY" if $ISA_TTY;
+
+ print "PS1 is set\n" if $ENV{PS1};
+
+ print "tty -s returns 0\n" if system("tty -s") == 0;
+
+ # check if parent process is a shell
+ my $ppid = getppid();
+ my $parent_cmd = `ps -p $ppid -o comm=`;
+ print "parent process is bash\n" if ($parent_cmd =~ '/bash$');
+ """
+ write_file(perl_script, perl_script_txt)
+ adjust_permissions(perl_script, stat.S_IXUSR)
+
+ def handler(signum, _):
+ raise RuntimeError(f"Test for running Perl script via run_shell_cmd took too long, signal {signum}")
+
+ orig_sigalrm_handler = signal.getsignal(signal.SIGALRM)
+
+ try:
+ # set the signal handler and a 3-second alarm
+ signal.signal(signal.SIGALRM, handler)
+ signal.alarm(3)
+
+ res = run_shell_cmd(perl_script, hidden=True)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'foo:\nstdin: \n')
+
+ res = run_shell_cmd(perl_script, hidden=True, stdin="test")
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'foo:\nstdin: test\n')
+
+ res = run_shell_cmd(perl_script, hidden=True, qa_patterns=[('foo:', 'bar')], qa_timeout=1)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'foo:\nstdin: bar\n\n')
+
+ error_pattern = "No matching questions found for current command output"
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, perl_script,
+ hidden=True, qa_patterns=[('bleh', 'blah')], qa_timeout=1)
+ finally:
+ # cleanup: disable the alarm + reset signal handler for SIGALRM
+ signal.signal(signal.SIGALRM, orig_sigalrm_handler)
+ signal.alarm(0)
+
+ def test_run_shell_cmd_env(self):
+ """Test env option in run_shell_cmd."""
+
+ # use 'env' to define environment in which command should be run;
+ # with a few exceptions (like $_, $PWD) no other environment variables will be defined,
+ # so $HOME and $USER will not be set
+ cmd = "env | sort"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, env={'FOOBAR': 'foobar', 'PATH': os.getenv('PATH')})
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertIn("FOOBAR=foobar\n", res.output)
+ self.assertTrue(re.search("^_=.*/env$", res.output, re.M))
+ for var in ('HOME', 'USER'):
+ self.assertFalse(re.search('^' + var + '=.*', res.output, re.M))
+
+ # check on helper scripts that were generated for this command
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'env-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+
+ # set environment variable in current environment,
+ # this should not be set in shell environment produced by scripts
+ os.environ['TEST123'] = 'test123'
+
+ env_script = os.path.join(cmd_tmpdir, 'env.sh')
+ self.assertExists(env_script)
+ env_script_txt = read_file(env_script)
+ self.assertIn('unset "$var"', env_script_txt)
+ self.assertIn('unset -f "$func"', env_script_txt)
+ self.assertIn('\nexport FOOBAR=foobar\nexport PATH', env_script_txt)
+
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{cmd_script} -c 'echo $FOOBAR; echo TEST123:$TEST123'", fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.endswith('\nfoobar\nTEST123:\n'))
+
+ def test_fileprefix_from_cmd(self):
+ """test simplifications from fileprefix_from_cmd."""
+ cmds = {
+ 'abd123': 'abd123',
+ 'ab"a': 'aba',
+ 'a{:$:S@"a': 'aSa',
+ 'cmd-with-dash': 'cmd-with-dash',
+ 'cmd_with_underscore': 'cmd_with_underscore',
+ }
+ for cmd, expected_simplification in cmds.items():
+ self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification)
+
+ cmds = {
+ 'abd123': 'abd',
+ 'ab"a': 'aba',
+ '0a{:$:2@"a': 'aa',
+ }
+ for cmd, expected_simplification in cmds.items():
+ self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification)
+
def test_run_cmd_log(self):
"""Test logging of executed commands."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
@@ -169,13 +401,15 @@ def test_run_cmd_log(self):
# command output is not logged by default without debug logging
init_logging(logfile, silent=True)
- self.assertTrue(run_cmd("echo hello"))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(run_cmd("echo hello"))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 0)
write_file(logfile, '')
init_logging(logfile, silent=True)
- self.assertTrue(run_cmd("echo hello", log_all=True))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(run_cmd("echo hello", log_all=True))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
@@ -184,20 +418,22 @@ def test_run_cmd_log(self):
setLogLevelDebug()
init_logging(logfile, silent=True)
- self.assertTrue(run_cmd("echo hello"))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(run_cmd("echo hello"))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
init_logging(logfile, silent=True)
- self.assertTrue(run_cmd("echo hello", log_all=True))
+ with self.mocked_stdout_stderr():
+ self.assertTrue(run_cmd("echo hello", log_all=True))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
# Test that we can set the directory for the logfile
log_path = os.path.join(self.test_prefix, 'chicken')
- os.mkdir(log_path)
+ mkdir(log_path)
logfile = None
init_logging(logfile, silent=True, tmp_logdir=log_path)
logfiles = os.listdir(log_path)
@@ -205,8 +441,46 @@ def test_run_cmd_log(self):
self.assertTrue(logfiles[0].startswith("easybuild"))
self.assertTrue(logfiles[0].endswith("log"))
+ def test_run_shell_cmd_log(self):
+ """Test logging of executed commands with run_shell_cmd function."""
+
+ fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
+ os.close(fd)
+
+ regex_start_cmd = re.compile("Running shell command 'echo hello' in /")
+ regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello")
+
+ # command output is always logged
+ init_logging(logfile, silent=True)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("echo hello")
+ stop_logging(logfile)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'hello\n')
+ logtxt = read_file(logfile)
+ self.assertEqual(len(regex_start_cmd.findall(logtxt)), 1)
+ self.assertEqual(len(regex_cmd_exit.findall(logtxt)), 1)
+ write_file(logfile, '')
+
+ # with debugging enabled, exit code and output of command should only get logged once
+ setLogLevelDebug()
+
+ init_logging(logfile, silent=True)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("echo hello")
+ stop_logging(logfile)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(len(regex_start_cmd.findall(read_file(logfile))), 1)
+ self.assertEqual(len(regex_cmd_exit.findall(read_file(logfile))), 1)
+ write_file(logfile, '')
+
def test_run_cmd_negative_exit_code(self):
"""Test run_cmd function with command that has negative exit code."""
+
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# define signal handler to call in case run_cmd takes too long
def handler(signum, _):
raise RuntimeError("Signal handler called with signal %s" % signum)
@@ -218,14 +492,16 @@ def handler(signum, _):
signal.signal(signal.SIGALRM, handler)
signal.alarm(3)
- (_, ec) = run_cmd("kill -9 $$", log_ok=False)
+ with self.mocked_stdout_stderr():
+ (_, ec) = run_cmd("kill -9 $$", log_ok=False)
self.assertEqual(ec, -9)
# reset the alarm
signal.alarm(0)
signal.alarm(3)
- (_, ec) = run_cmd_qa("kill -9 $$", {}, log_ok=False)
+ with self.mocked_stdout_stderr():
+ (_, ec) = run_cmd_qa("kill -9 $$", {}, log_ok=False)
self.assertEqual(ec, -9)
finally:
@@ -233,17 +509,205 @@ def handler(signum, _):
signal.signal(signal.SIGALRM, orig_sigalrm_handler)
signal.alarm(0)
+ def test_run_shell_cmd_fail(self):
+ """Test run_shell_cmd function with command that has negative exit code."""
+ # define signal handler to call in case run takes too long
+ def handler(signum, _):
+ raise RuntimeError("Signal handler called with signal %s" % signum)
+
+ # disable trace output for this test (so stdout remains empty)
+ update_build_option('trace', False)
+
+ orig_sigalrm_handler = signal.getsignal(signal.SIGALRM)
+
+ try:
+ # set the signal handler and a 3-second alarm
+ signal.signal(signal.SIGALRM, handler)
+ signal.alarm(3)
+
+ # command to kill parent shell
+ cmd = "kill -9 $$"
+
+ work_dir = os.path.realpath(self.test_prefix)
+ change_dir(work_dir)
+
+ try:
+ run_shell_cmd(cmd)
+ self.assertFalse("This should never be reached, RunShellCmdError should occur!")
+ except RunShellCmdError as err:
+ self.assertEqual(str(err), "Shell command 'kill' failed!")
+ self.assertEqual(err.cmd, "kill -9 $$")
+ self.assertEqual(err.cmd_name, 'kill')
+ self.assertEqual(err.exit_code, -9)
+ self.assertEqual(err.work_dir, work_dir)
+ self.assertEqual(err.output, '')
+ self.assertEqual(err.stderr, None)
+ self.assertTrue(isinstance(err.caller_info, tuple))
+ self.assertEqual(len(err.caller_info), 3)
+ self.assertEqual(err.caller_info[0], __file__)
+ self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site
+ self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail')
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ err.print()
+
+ # check error reporting output
+ stderr = stderr.getvalue()
+ patterns = [
+ r"ERROR: Shell command failed!",
+ r"\s+full command\s* -> kill -9 \$\$",
+ r"\s+exit code\s* -> -9",
+ r"\s+working directory\s* -> " + work_dir,
+ r"\s+called from\s* -> 'test_run_shell_cmd_fail' function in "
+ r"(.|\n)*/test/(.|\n)*/run.py \(line [0-9]+\)",
+ r"\s+output \(stdout \+ stderr\)\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/out.txt",
+ r"\s+interactive shell script\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/cmd.sh",
+ ]
+ for pattern in patterns:
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr))
+
+ # check error reporting output when stdout/stderr are collected separately
+ try:
+ run_shell_cmd(cmd, split_stderr=True)
+ self.assertFalse("This should never be reached, RunShellCmdError should occur!")
+ except RunShellCmdError as err:
+ self.assertEqual(str(err), "Shell command 'kill' failed!")
+ self.assertEqual(err.cmd, "kill -9 $$")
+ self.assertEqual(err.cmd_name, 'kill')
+ self.assertEqual(err.exit_code, -9)
+ self.assertEqual(err.work_dir, work_dir)
+ self.assertEqual(err.output, '')
+ self.assertEqual(err.stderr, '')
+ self.assertTrue(isinstance(err.caller_info, tuple))
+ self.assertEqual(len(err.caller_info), 3)
+ self.assertEqual(err.caller_info[0], __file__)
+ self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site
+ self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail')
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ err.print()
+
+ # check error reporting output
+ stderr = stderr.getvalue()
+ patterns = [
+ r"ERROR: Shell command failed!",
+ r"\s+full command\s+ -> kill -9 \$\$",
+ r"\s+exit code\s+ -> -9",
+ r"\s+working directory\s+ -> " + work_dir,
+ r"\s+called from\s+ -> 'test_run_shell_cmd_fail' function in "
+ r"(.|\n)*/test/(.|\n)*/run.py \(line [0-9]+\)",
+ r"\s+output \(stdout\)\s+ -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/out.txt",
+ r"\s+error/warnings \(stderr\)\s+ -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/err.txt",
+ r"\s+interactive shell script\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/cmd.sh",
+ ]
+ for pattern in patterns:
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr))
+
+ # no error reporting when fail_on_error is disabled
+ with self.mocked_stdout_stderr() as (_, stderr):
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, -9)
+ self.assertEqual(stderr.getvalue(), '')
+
+ finally:
+ # cleanup: disable the alarm + reset signal handler for SIGALRM
+ signal.signal(signal.SIGALRM, orig_sigalrm_handler)
+ signal.alarm(0)
+
def test_run_cmd_bis(self):
"""More 'complex' test for run_cmd function."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# a more 'complex' command to run, make sure all required output is there
- (out, ec) = run_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done")
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done")
self.assertTrue(out.startswith('hello\nhello\n'))
self.assertEqual(len(out), len("hello\n" * 300))
self.assertEqual(ec, 0)
+ def test_run_shell_cmd_bis(self):
+ """More 'complex' test for run_shell_cmd function."""
+ # a more 'complex' command to run, make sure all required output is there
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done")
+ self.assertTrue(res.output.startswith('hello\nhello\n'))
+ self.assertEqual(len(res.output), len("hello\n" * 300))
+ self.assertEqual(res.exit_code, 0)
+
+ def test_run_cmd_work_dir(self):
+ """
+ Test running command in specific directory with run_cmd function.
+ """
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ orig_wd = os.getcwd()
+ self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
+
+ test_dir = os.path.join(self.test_prefix, 'test')
+ for fn in ('foo.txt', 'bar.txt'):
+ write_file(os.path.join(test_dir, fn), 'test')
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("ls | sort", path=test_dir)
+
+ self.assertEqual(ec, 0)
+ self.assertEqual(out, 'bar.txt\nfoo.txt\n')
+
+ self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
+
+ def test_run_shell_cmd_work_dir(self):
+ """
+ Test running shell command in specific directory with run_shell_cmd function.
+ """
+ test_dir = os.path.join(self.test_prefix, 'test')
+ test_workdir = os.path.join(self.test_prefix, 'test', 'workdir')
+ for fn in ('foo.txt', 'bar.txt'):
+ write_file(os.path.join(test_workdir, fn), 'test')
+
+ os.chdir(test_dir)
+ orig_wd = os.getcwd()
+ self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
+
+ cmd = "ls | sort"
+
+ # working directory is not explicitly defined
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'workdir\n')
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, orig_wd)
+
+ self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
+
+ # working directory is explicitly defined
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, work_dir=test_workdir)
+
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'bar.txt\nfoo.txt\n')
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, test_workdir)
+
+ self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
+
def test_run_cmd_log_output(self):
"""Test run_cmd with log_output enabled"""
- (out, ec) = run_cmd("seq 1 100", log_output=True)
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("seq 1 100", log_output=True)
self.assertEqual(ec, 0)
self.assertEqual(type(out), str)
self.assertTrue(out.startswith("1\n2\n"))
@@ -266,17 +730,63 @@ def test_run_cmd_log_output(self):
write_file(test_file, text)
cmd = "cat %s" % test_file
- (out, ec) = run_cmd(cmd, log_output=True)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd(cmd, log_output=True)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith('foo ') and out.endswith(' bar'))
self.assertEqual(type(out), str)
+ def test_run_shell_cmd_split_stderr(self):
+ """Test getting split stdout/stderr output from run_shell_cmd function."""
+ cmd = ';'.join([
+ "echo ok",
+ "echo warning >&2",
+ ])
+
+ # by default, output contains both stdout + stderr
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
+ output_lines = res.output.split('\n')
+ self.assertTrue("ok" in output_lines)
+ self.assertTrue("warning" in output_lines)
+ self.assertEqual(res.stderr, None)
+
+ # cleanup of artifacts in between calls to run_shell_cmd
+ remove_dir(self.test_prefix)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, split_stderr=True)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.stderr, "warning\n")
+ self.assertEqual(res.output, "ok\n")
+
+ # check whether environment variables that point to stdout/stderr output files
+ # are set in environment defined by cmd.sh script
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ cmd_cmd = '; '.join([
+ "echo $EB_CMD_OUT_FILE",
+ "cat $EB_CMD_OUT_FILE",
+ "echo $EB_CMD_ERR_FILE",
+ "cat $EB_CMD_ERR_FILE",
+ ])
+ cmd = f"{cmd_script} -c '{cmd_cmd}'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+
+ regex = re.compile(".*/echo-.*/out.txt\nok\n.*/echo-.*/err.txt\nwarning$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
def test_run_cmd_trace(self):
- """Test run_cmd under --trace"""
- # replace log.experimental with log.warning to allow experimental code
- easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning
+ """Test run_cmd in trace mode, and with tracing disabled."""
- init_config(build_options={'trace': True})
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
pattern = [
r"^ >> running command:",
@@ -287,6 +797,7 @@ def test_run_cmd_trace(self):
r" >> command completed: exit 0, ran in .*",
]
+ # trace output is enabled by default (since EasyBuild v5.0)
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd("echo hello")
@@ -294,11 +805,28 @@ def test_run_cmd_trace(self):
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
+ self.assertEqual(out, 'hello\n')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
regex = re.compile('\n'.join(pattern))
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ init_config(build_options={'trace': False})
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ (out, ec) = run_cmd("echo hello")
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(out, 'hello\n')
+ self.assertEqual(ec, 0)
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+ self.assertEqual(stdout, '')
+
+ init_config(build_options={'trace': True})
+
# also test with command that is fed input via stdin
self.mock_stdout(True)
self.mock_stderr(True)
@@ -307,30 +835,168 @@ def test_run_cmd_trace(self):
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
+ self.assertEqual(out, 'hello')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
pattern.insert(3, r"\t\[input: hello\]")
pattern[-2] = "\tcat"
regex = re.compile('\n'.join(pattern))
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ init_config(build_options={'trace': False})
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ (out, ec) = run_cmd('cat', inp='hello')
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(out, 'hello')
+ self.assertEqual(ec, 0)
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+ self.assertEqual(stdout, '')
+
# trace output can be disabled on a per-command basis
+ for trace in (True, False):
+ init_config(build_options={'trace': trace})
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ (out, ec) = run_cmd("echo hello", trace=False)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(out, 'hello\n')
+ self.assertEqual(ec, 0)
+ self.assertEqual(stdout, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+
+ def test_run_shell_cmd_trace(self):
+ """Test run_shell_cmd function in trace mode, and with tracing disabled."""
+
+ pattern = [
+ r"^ >> running shell command:",
+ r"\techo hello",
+ r"\t\[started at: .*\]",
+ r"\t\[working dir: .*\]",
+ r"\t\[output and state saved to .*\]",
+ r" >> command completed: exit 0, ran in .*",
+ ]
+
+ # trace output is enabled by default (since EasyBuild v5.0)
self.mock_stdout(True)
self.mock_stderr(True)
- (out, ec) = run_cmd("echo hello", trace=False)
+ res = run_shell_cmd("echo hello")
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(stderr, '')
+ regex = re.compile('\n'.join(pattern))
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ init_config(build_options={'trace': False})
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ res = run_shell_cmd("echo hello")
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(stderr, '')
+ self.assertEqual(stdout, '')
+
+ init_config(build_options={'trace': True})
+
+ # trace output can be disabled on a per-command basis via 'hidden' option
+ for trace in (True, False):
+ init_config(build_options={'trace': trace})
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ res = run_shell_cmd("echo hello", hidden=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(stdout, '')
+ self.assertEqual(stderr, '')
+
+ def test_run_shell_cmd_trace_stdin(self):
+ """Test run_shell_cmd function under --trace + passing stdin input."""
+
+ init_config(build_options={'trace': True})
+
+ pattern = [
+ r"^ >> running shell command:",
+ r"\techo hello",
+ r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]",
+ r"\t\[working dir: .*\]",
+ r"\t\[output and state saved to .*\]",
+ r" >> command completed: exit 0, ran in .*",
+ ]
+
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ res = run_shell_cmd("echo hello")
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(stderr, '')
+ regex = re.compile('\n'.join(pattern))
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ # also test with command that is fed input via stdin
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ res = run_shell_cmd('cat', stdin='hello')
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(stderr, '')
+ pattern.insert(4, r"\t\[input: hello\]")
+ pattern[1] = "\tcat"
+ regex = re.compile('\n'.join(pattern))
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ # trace output can be disabled on a per-command basis by enabling 'hidden'
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ res = run_shell_cmd("echo hello", hidden=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(res.output, 'hello\n')
+ self.assertEqual(res.exit_code, 0)
self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
def test_run_cmd_qa(self):
"""Basic test for run_cmd_qa function."""
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = "echo question; read x; echo $x"
qa = {'question': 'answer'}
- (out, ec) = run_cmd_qa(cmd, qa)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, qa)
self.assertEqual(out, "question\nanswer\n")
# no reason echo hello could fail
self.assertEqual(ec, 0)
@@ -342,19 +1008,183 @@ def test_run_cmd_qa(self):
write_file(test_file, b"foo \xe2 bar")
cmd += "; cat %s" % test_file
- (out, ec) = run_cmd_qa(cmd, qa)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, qa)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith("question\nanswer\nfoo "))
self.assertTrue(out.endswith('bar'))
+ # test handling of output that is not actually a question
+ cmd = ';'.join([
+ "echo not-a-question-but-a-statement",
+ "sleep 3",
+ "echo question",
+ "read x",
+ "echo $x",
+ ])
+ qa = {'question': 'answer'}
+
+ # fails because non-question is encountered
+ error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False)
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False)
+ self.assertEqual(out, "not-a-question-but-a-statement\nquestion\nanswer\n")
+ self.assertEqual(ec, 0)
+
+ def test_run_shell_cmd_qa(self):
+ """Basic test for Q&A support in run_shell_cmd function."""
+
+ cmd = '; '.join([
+ "echo question1",
+ "read x",
+ "echo $x",
+ "echo question2",
+ "read y",
+ "echo $y",
+ ])
+ qa = [
+ ('question1', 'answer1'),
+ ('question2', 'answer2'),
+ ]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question1\nanswer1\nquestion2\nanswer2\n")
+ # no reason echo hello could fail
+ self.assertEqual(res.exit_code, 0)
+
+ # test running command that emits non-UTF8 characters
+ # this is constructed to reproduce errors like:
+ # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
+ test_file = os.path.join(self.test_prefix, 'foo.txt')
+ write_file(test_file, b"foo \xe2 bar")
+ cmd += "; cat %s" % test_file
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo "))
+ self.assertTrue(res.output.endswith('bar'))
+
+ # check type check on qa_patterns
+ error_pattern = "qa_patterns passed to run_shell_cmd should be a list of 2-tuples!"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns={'foo': 'bar'})
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=('foo', 'bar'))
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=(('foo', 'bar'),))
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns='foo:bar')
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=['foo:bar'])
+
+ # validate use of qa_timeout to give up if there's no matching question for too long
+ cmd = "sleep 3; echo 'question'; read a; echo $a"
+ error_pattern = "No matching questions found for current command output, giving up after 1 seconds!"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1)
+
+ # check using answer that is completed via pattern extracted from question
+ cmd = ';'.join([
+ "echo 'and the magic number is: 42'",
+ "read magic_number",
+ "echo $magic_number",
+ ])
+ qa = [("and the magic number is: (?P[0-9]+)", "%(nr)s")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "and the magic number is: 42\n42\n")
+
+ # test handling of output that is not actually a question
+ cmd = ';'.join([
+ "echo not-a-question-but-a-statement",
+ "sleep 3",
+ "echo question",
+ "read x",
+ "echo $x",
+ ])
+ qa = [('question', 'answer')]
+
+ # fails because non-question is encountered
+ error_pattern = "No matching questions found for current command output, giving up after 1 seconds!"
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1,
+ hidden=True)
+
+ qa_wait_patterns = ["not-a-question-but-a-statement"]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "not-a-question-but-a-statement\nquestion\nanswer\n")
+
+ # test multi-line question
+ cmd = ';'.join([
+ "echo please",
+ "echo answer",
+ "read x",
+ "echo $x",
+ ])
+ qa = [("please answer", "42")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "please\nanswer\n42\n")
+
+ # also test multi-line wait pattern
+ cmd = "echo just; echo wait; sleep 3; " + cmd
+ qa_wait_patterns = ["just wait"]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "just\nwait\nplease\nanswer\n42\n")
+
+ # test multi-line question pattern with hard space
+ cmd = ';'.join([
+ "echo please",
+ "echo answer",
+ "read x",
+ "echo $x",
+ ])
+ # question pattern uses hard space, should get replaced internally by more liberal whitespace regex pattern
+ qa = [(r"please\ answer", "42")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_timeout=3)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "please\nanswer\n42\n")
+
+ # test interactive command that takes a while before producing more output that includes second question
+ cmd = ';'.join([
+ "echo question1",
+ "read answer1",
+ "sleep 2",
+ "echo question2",
+ "read answer2",
+ # note: delaying additional output (except the actual questions) is important
+ # to verify that this is working as intended
+ "echo $answer1",
+ "echo $answer2",
+ ])
+ qa = [
+ (r'question1', 'answer1'),
+ (r'question2', 'answer2'),
+ ]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "question1\nquestion2\nanswer1\nanswer2\n")
+
def test_run_cmd_qa_buffering(self):
"""Test whether run_cmd_qa uses unbuffered output."""
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# command that generates a lot of output before waiting for input
# note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000!
cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; '
cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"'
- (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5)
self.assertEqual(ec, 0)
regex = re.compile("Picked number: 42$")
@@ -372,14 +1202,52 @@ def test_run_cmd_qa_buffering(self):
write_file(script, script_txt)
adjust_permissions(script, stat.S_IXUSR)
- out, ec = run_cmd_qa(script, {}, log_ok=False)
+ with self.mocked_stdout_stderr():
+ out, ec = run_cmd_qa(script, {}, log_ok=False)
self.assertEqual(ec, 1)
self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n")
+ def test_run_shell_cmd_qa_buffering(self):
+ """Test whether run_shell_cmd uses unbuffered output when running interactive commands."""
+
+ # command that generates a lot of output before waiting for input
+ # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000!
+ cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; '
+ cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"'
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10)
+
+ self.assertEqual(res.exit_code, 0)
+ regex = re.compile("Picked number: 42$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' found in: {res.output}")
+
+ # also test with script run as interactive command that quickly exits with non-zero exit code;
+ # see https://github.com/easybuilders/easybuild-framework/issues/3593
+ script_txt = '\n'.join([
+ "#/bin/bash",
+ "echo 'Hello, I am about to exit'",
+ "echo 'ERROR: I failed' >&2",
+ "exit 1",
+ ])
+ script = os.path.join(self.test_prefix, 'test.sh')
+ write_file(script, script_txt)
+ adjust_permissions(script, stat.S_IXUSR)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(script, qa_patterns=[], fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 1)
+ self.assertEqual(res.output, "Hello, I am about to exit\nERROR: I failed\n")
+
def test_run_cmd_qa_log_all(self):
"""Test run_cmd_qa with log_output enabled"""
- (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True)
+
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True)
self.assertEqual(ec, 0)
self.assertEqual(out, "n: \n1\n2\n3\n4\n5\n")
@@ -389,13 +1257,25 @@ def test_run_cmd_qa_log_all(self):
extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n"
self.assertEqual(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n")
+ def test_run_shell_cmd_qa_log(self):
+ """Test temporary log file for run_shell_cmd with qa_patterns"""
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n:', '5')])
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "n: \n1\n2\n3\n4\n5\n")
+
+ run_cmd_logs = glob.glob(os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output', 'echo-*', 'out.txt'))
+ self.assertEqual(len(run_cmd_logs), 1)
+ run_cmd_log_txt = read_file(run_cmd_logs[0])
+ self.assertEqual(run_cmd_log_txt, "n: \n1\n2\n3\n4\n5\n")
+
def test_run_cmd_qa_trace(self):
"""Test run_cmd under --trace"""
- # replace log.experimental with log.warning to allow experimental code
- easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning
- init_config(build_options={'trace': True})
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+ # --trace is enabled by default
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'})
@@ -403,7 +1283,7 @@ def test_run_cmd_qa_trace(self):
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
pattern = r"^ >> running interactive command:\n"
pattern += r"\t\[started at: .*\]\n"
pattern += r"\t\[working dir: .*\]\n"
@@ -421,128 +1301,316 @@ def test_run_cmd_qa_trace(self):
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+
+ def test_run_shell_cmd_qa_trace(self):
+ """Test run_shell_cmd with qa_patterns under --trace"""
+
+ # --trace is enabled by default
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')])
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(stderr, '')
+ pattern = r"^ >> running interactive shell command:\n"
+ pattern += r"\techo \'n: \'; read n; seq 1 \$n\n"
+ pattern += r"\t\[started at: .*\]\n"
+ pattern += r"\t\[working dir: .*\]\n"
+ pattern += r"\t\[output and state saved to .*\]\n"
+ pattern += r' >> command completed: exit 0, ran in .*'
+ self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout))
+
+ # trace output can be disabled on a per-command basis
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')], hidden=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
def test_run_cmd_qa_answers(self):
"""Test providing list of answers in run_cmd_qa."""
+
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = "echo question; read x; echo $x; " * 2
qa = {"question": ["answer1", "answer2"]}
- (out, ec) = run_cmd_qa(cmd, qa)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, qa)
self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n")
self.assertEqual(ec, 0)
- (out, ec) = run_cmd_qa(cmd, {}, std_qa=qa)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, {}, std_qa=qa)
self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n")
self.assertEqual(ec, 0)
- self.assertErrorRegex(EasyBuildError, "Invalid type for answer", run_cmd_qa, cmd, {'q': 1})
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Invalid type for answer", run_cmd_qa, cmd, {'q': 1})
# test cycling of answers
cmd = cmd * 2
- (out, ec) = run_cmd_qa(cmd, {}, std_qa=qa)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, {}, std_qa=qa)
self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n" * 2)
self.assertEqual(ec, 0)
+ def test_run_shell_cmd_qa_answers(self):
+ """Test providing list of answers for a question in run_shell_cmd."""
+
+ cmd = "echo question; read x; echo $x; " * 2
+ qa = [("question", ["answer1", "answer2"])]
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n")
+ self.assertEqual(res.exit_code, 0)
+
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd,
+ qa_patterns=[('question', 1)])
+
+ # test cycling of answers
+ cmd = cmd * 2
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n" * 2)
+ self.assertEqual(res.exit_code, 0)
+
def test_run_cmd_simple(self):
"""Test return value for run_cmd in 'simple' mode."""
- self.assertEqual(True, run_cmd("echo hello", simple=True))
- self.assertEqual(False, run_cmd("exit 1", simple=True, log_all=False, log_ok=False))
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ self.assertEqual(True, run_cmd("echo hello", simple=True))
+ self.assertEqual(False, run_cmd("exit 1", simple=True, log_all=False, log_ok=False))
def test_run_cmd_cache(self):
"""Test caching for run_cmd"""
- (first_out, ec) = run_cmd("ulimit -u")
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ (first_out, ec) = run_cmd("ulimit -u")
self.assertEqual(ec, 0)
- (cached_out, ec) = run_cmd("ulimit -u")
+ with self.mocked_stdout_stderr():
+ (cached_out, ec) = run_cmd("ulimit -u")
self.assertEqual(ec, 0)
self.assertEqual(first_out, cached_out)
# inject value into cache to check whether executing command again really returns cached value
- run_cmd.update_cache({("ulimit -u", None): ("123456", 123)})
- (cached_out, ec) = run_cmd("ulimit -u")
+ with self.mocked_stdout_stderr():
+ run_cmd.update_cache({("ulimit -u", None): ("123456", 123)})
+ (cached_out, ec) = run_cmd("ulimit -u")
self.assertEqual(ec, 123)
self.assertEqual(cached_out, "123456")
# also test with command that uses stdin
- (out, ec) = run_cmd("cat", inp='foo')
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("cat", inp='foo')
self.assertEqual(ec, 0)
self.assertEqual(out, 'foo')
# inject different output for cat with 'foo' as stdin to check whether cached value is used
- run_cmd.update_cache({('cat', 'foo'): ('bar', 123)})
- (cached_out, ec) = run_cmd("cat", inp='foo')
+ with self.mocked_stdout_stderr():
+ run_cmd.update_cache({('cat', 'foo'): ('bar', 123)})
+ (cached_out, ec) = run_cmd("cat", inp='foo')
self.assertEqual(ec, 123)
self.assertEqual(cached_out, 'bar')
run_cmd.clear_cache()
+ def test_run_shell_cmd_cache(self):
+ """Test caching for run_shell_cmd function"""
+
+ cmd = "ulimit -u"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ first_out = res.output
+ self.assertEqual(res.exit_code, 0)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ cached_out = res.output
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(first_out, cached_out)
+
+ # inject value into cache to check whether executing command again really returns cached value
+ with self.mocked_stdout_stderr():
+ cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None,
+ work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None,
+ cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None)
+ run_shell_cmd.update_cache({(cmd, None): cached_res})
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 123)
+ self.assertEqual(res.output, "123456")
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, '/test_ulimit')
+
+ # also test with command that uses stdin
+ cmd = "cat"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, stdin='foo')
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'foo')
+
+ # inject different output for cat with 'foo' as stdin to check whether cached value is used
+ with self.mocked_stdout_stderr():
+ cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None,
+ work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None,
+ cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None)
+ run_shell_cmd.update_cache({(cmd, 'foo'): cached_res})
+ res = run_shell_cmd(cmd, stdin='foo')
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 123)
+ self.assertEqual(res.output, 'bar')
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, '/test_cat')
+
+ run_shell_cmd.clear_cache()
+
def test_parse_log_error(self):
"""Test basic parse_log_for_error functionality."""
- errors = parse_log_for_error("error failed", True)
+
+ # use of parse_log_for_error is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ errors = parse_log_for_error("error failed", True)
self.assertEqual(len(errors), 1)
- def test_dry_run(self):
- """Test use of functions under (extended) dry run."""
+ def test_run_cmd_dry_run(self):
+ """Test use of run_cmd function under (extended) dry run."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
- self.mock_stdout(True)
- run_cmd("somecommand foo 123 bar")
- txt = self.get_stdout()
- self.mock_stdout(False)
+ cmd = "somecommand foo 123 bar"
- expected_regex = re.compile('\n'.join([
- r" running command \"somecommand foo 123 bar\"",
- r" \(in .*\)",
- ]))
- self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt))
+ with self.mocked_stdout_stderr():
+ run_cmd(cmd)
+ stdout = self.get_stdout()
+
+ expected = """ running command "somecommand foo 123 bar"\n"""
+ self.assertIn(expected, stdout)
# check disabling 'verbose'
- self.mock_stdout(True)
- run_cmd("somecommand foo 123 bar", verbose=False)
- txt = self.get_stdout()
- self.mock_stdout(False)
- self.assertEqual(txt, '')
+ with self.mocked_stdout_stderr():
+ run_cmd("somecommand foo 123 bar", verbose=False)
+ stdout = self.get_stdout()
+ self.assertNotIn(expected, stdout)
- # check forced run
+ # check forced run_cmd
outfile = os.path.join(self.test_prefix, 'cmd.out')
self.assertNotExists(outfile)
- self.mock_stdout(True)
- run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True)
- txt = self.get_stdout()
- self.mock_stdout(False)
- # nothing printed to stdout, but command was run
- self.assertEqual(txt, '')
+ with self.mocked_stdout_stderr():
+ run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True)
self.assertExists(outfile)
self.assertEqual(read_file(outfile), "This is always echoed\n")
# Q&A commands
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')])
+ stdout = self.get_stdout()
+
+ expected = """ running interactive shell command "some_qa_cmd"\n"""
+ self.assertIn(expected, stdout)
+
+ with self.mocked_stdout_stderr():
+ run_cmd_qa("some_qa_cmd", {'question1': 'answer1'})
+ stdout = self.get_stdout()
+
+ expected = """ running interactive command "some_qa_cmd"\n"""
+ self.assertIn(expected, stdout)
+
+ def test_run_shell_cmd_dry_run(self):
+ """Test use of run_shell_cmd function under (extended) dry run."""
+ build_options = {
+ 'extended_dry_run': True,
+ 'silent': False,
+ }
+ init_config(build_options=build_options)
+
+ cmd = "somecommand foo 123 bar"
+
self.mock_stdout(True)
- run_cmd_qa("some_qa_cmd", {'question1': 'answer1'})
- txt = self.get_stdout()
+ res = run_shell_cmd(cmd)
+ stdout = self.get_stdout()
self.mock_stdout(False)
-
- expected_regex = re.compile('\n'.join([
- r" running interactive command \"some_qa_cmd\"",
- r" \(in .*\)",
- ]))
- self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt))
+ # fake output/exit code is returned for commands not actually run in dry run mode
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, '')
+ self.assertEqual(res.stderr, None)
+ # check dry run output
+ expected = """ running shell command "somecommand foo 123 bar"\n"""
+ self.assertIn(expected, stdout)
+
+ # check enabling 'hidden'
+ self.mock_stdout(True)
+ res = run_shell_cmd(cmd, hidden=True)
+ stdout = self.get_stdout()
+ self.mock_stdout(False)
+ # fake output/exit code is returned for commands not actually run in dry run mode
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, '')
+ self.assertEqual(res.stderr, None)
+ # dry run output should be missing
+ self.assertNotIn(expected, stdout)
+
+ # check forced run_cmd
+ outfile = os.path.join(self.test_prefix, 'cmd.out')
+ self.assertNotExists(outfile)
+ self.mock_stdout(True)
+ res = run_shell_cmd("echo 'This is always echoed' > %s; echo done; false" % outfile,
+ fail_on_error=False, in_dry_run=True)
+ stdout = self.get_stdout()
+ self.mock_stdout(False)
+ self.assertNotIn('running shell command "', stdout)
+ self.assertNotEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'done\n')
+ self.assertEqual(res.stderr, None)
+ self.assertExists(outfile)
+ self.assertEqual(read_file(outfile), "This is always echoed\n")
def test_run_cmd_list(self):
"""Test run_cmd with command specified as a list rather than a string"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = ['/bin/sh', '-c', "echo hello"]
- self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!",
- run_cmd, cmd)
- (out, ec) = run_cmd(cmd, shell=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!",
+ run_cmd, cmd)
+ (out, ec) = run_cmd(cmd, shell=False)
self.assertEqual(out, "hello\n")
# no reason echo hello could fail
self.assertEqual(ec, 0)
def test_run_cmd_script(self):
"""Testing use of run_cmd with shell=False to call external scripts"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
py_test_script = os.path.join(self.test_prefix, 'test.py')
write_file(py_test_script, '\n'.join([
'#!%s' % sys.executable,
@@ -550,16 +1618,41 @@ def test_run_cmd_script(self):
]))
adjust_permissions(py_test_script, stat.S_IXUSR)
- (out, ec) = run_cmd(py_test_script)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd(py_test_script)
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
- (out, ec) = run_cmd([py_test_script], shell=False)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd([py_test_script], shell=False)
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
+ def test_run_shell_cmd_no_bash(self):
+ """Testing use of run_shell_cmd with use_bash=False to call external scripts"""
+ py_test_script = os.path.join(self.test_prefix, 'test.py')
+ write_file(py_test_script, '\n'.join([
+ '#!%s' % sys.executable,
+ 'print("hello")',
+ ]))
+ adjust_permissions(py_test_script, stat.S_IXUSR)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(py_test_script)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "hello\n")
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd([py_test_script], use_bash=False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "hello\n")
+
def test_run_cmd_stream(self):
"""Test use of run_cmd with streaming output."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd("echo hello", stream_output=True)
@@ -571,21 +1664,79 @@ def test_run_cmd_stream(self):
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
- self.assertEqual(stderr, '')
- expected = '\n'.join([
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+ expected = [
"== (streaming) output for command 'echo hello':",
"hello",
'',
+ ]
+ for line in expected:
+ self.assertIn(line, stdout)
+
+ def test_run_shell_cmd_stream(self):
+ """Test use of run_shell_cmd with streaming output."""
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ cmd = '; '.join([
+ "echo hello there",
+ "sleep 1",
+ "echo testing command that produces a fair amount of output",
+ "sleep 1",
+ "echo more than 128 bytes which means a whole bunch of characters...",
+ "sleep 1",
+ "echo more than 128 characters in fact, which is quite a bit when you think of it",
+ ])
+ res = run_shell_cmd(cmd, stream_output=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+
+ expected_output = '\n'.join([
+ "hello there",
+ "testing command that produces a fair amount of output",
+ "more than 128 bytes which means a whole bunch of characters...",
+ "more than 128 characters in fact, which is quite a bit when you think of it",
+ '',
])
- self.assertEqual(stdout, expected)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, expected_output)
+
+ self.assertEqual(stderr, '')
+ expected = ("running shell command:\n\techo hello" + '\n' + expected_output).split('\n')
+ for line in expected:
+ self.assertIn(line, stdout)
+
+ def test_run_shell_cmd_eof_stdin(self):
+ """Test use of run_shell_cmd with streaming output and blocking stdin read."""
+ cmd = 'timeout 1 cat -'
+
+ inp = 'hello\nworld\n'
+ # test with streaming output
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, stream_output=True, stdin=inp, fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 0, "Streaming output: Command timed out")
+ self.assertEqual(res.output, inp)
+
+ # test with non-streaming output (proc.communicate() is used)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, stdin=inp, fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 0, "Non-streaming output: Command timed out")
+ self.assertEqual(res.output, inp)
def test_run_cmd_async(self):
"""Test asynchronously running of a shell command via run_cmd + complete_cmd."""
+ # use of run_cmd/check_async_cmd/get_output_from_process is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
os.environ['TEST'] = 'test123'
test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST"
- cmd_info = run_cmd(test_cmd, asynchronous=True)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(test_cmd, asynchronous=True)
proc = cmd_info[0]
# change value of $TEST to check that command is completed with correct environment
@@ -600,37 +1751,45 @@ def test_run_cmd_async(self):
time.sleep(1)
ec = proc.poll()
- out, ec = complete_cmd(*cmd_info, simple=False)
+ with self.mocked_stdout_stderr():
+ out, ec = complete_cmd(*cmd_info, simple=False)
self.assertEqual(ec, 0)
self.assertEqual(out, 'sleeping...\ntest123\n')
# also test use of check_async_cmd function
os.environ['TEST'] = 'test123'
- cmd_info = run_cmd(test_cmd, asynchronous=True)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(test_cmd, asynchronous=True)
# first check, only read first 12 output characters
# (otherwise we'll be waiting until command is completed)
- res = check_async_cmd(*cmd_info, output_read_size=12)
+ with self.mocked_stdout_stderr():
+ res = check_async_cmd(*cmd_info, output_read_size=12)
self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'})
# 2nd check with default output size (1024) gets full output
# (keep checking until command is fully done)
- while not res['done']:
- res = check_async_cmd(*cmd_info, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
# check asynchronous running of failing command
error_test_cmd = "echo 'FAIL!' >&2; exit 123"
- cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(error_test_cmd, asynchronous=True)
time.sleep(1)
error_pattern = 'cmd ".*" exited with exit code 123'
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
- cmd_info = run_cmd(error_test_cmd, asynchronous=True)
- res = check_async_cmd(*cmd_info, fail_on_error=False)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+ res = check_async_cmd(*cmd_info, fail_on_error=False)
# keep checking until command is fully done
- while not res['done']:
- res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output'])
self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
# also test with a command that produces a lot of output,
@@ -640,62 +1799,136 @@ def test_run_cmd_async(self):
"for i in $(seq 1 50)",
"do sleep 0.1",
"for j in $(seq 1000)",
- "do echo foo",
+ "do echo foo${i}${j}",
"done",
"done",
"echo done",
])
- cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
proc = cmd_info[0]
output = ''
ec = proc.poll()
self.assertEqual(ec, None)
- while ec is None:
- time.sleep(1)
- output += get_output_from_process(proc)
- ec = proc.poll()
+ with self.mocked_stdout_stderr():
+ while ec is None:
+ time.sleep(1)
+ output += get_output_from_process(proc)
+ ec = proc.poll()
- out, ec = complete_cmd(*cmd_info, simple=False, output=output)
+ with self.mocked_stdout_stderr():
+ out, ec = complete_cmd(*cmd_info, simple=False, output=output)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith('start\n'))
self.assertTrue(out.endswith('\ndone\n'))
# also test use of check_async_cmd on verbose test command
- cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
+ with self.mocked_stdout_stderr():
+ cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)"
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
# with output_read_size set to 0, no output is read yet, only status of command is checked
- res = check_async_cmd(*cmd_info, output_read_size=0)
+ with self.mocked_stdout_stderr():
+ res = check_async_cmd(*cmd_info, output_read_size=0)
self.assertEqual(res['done'], False)
self.assertEqual(res['exit_code'], None)
self.assertEqual(res['output'], '')
- res = check_async_cmd(*cmd_info)
+ with self.mocked_stdout_stderr():
+ res = check_async_cmd(*cmd_info)
self.assertEqual(res['done'], False)
self.assertEqual(res['exit_code'], None)
self.assertTrue(res['output'].startswith('start\n'))
self.assertFalse(res['output'].endswith('\ndone\n'))
# keep checking until command is complete
- while not res['done']:
- res = check_async_cmd(*cmd_info, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res['done'], True)
self.assertEqual(res['exit_code'], 0)
- self.assertTrue(res['output'].startswith('start\n'))
- self.assertTrue(res['output'].endswith('\ndone\n'))
+ self.assertEqual(len(res['output']), 435661)
+ self.assertTrue(res['output'].startswith('start\nfoo11\nfoo12\n'))
+ self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res['output'])
+ self.assertTrue(res['output'].endswith('\nfoo501000\ndone\n'))
+
+ def test_run_shell_cmd_async(self):
+ """Test asynchronously running of a shell command via run_shell_cmd """
+
+ thread_pool = ThreadPoolExecutor()
+
+ os.environ['TEST'] = 'test123'
+ env = os.environ.copy()
+
+ test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST"
+ task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env)
+
+ # change value of $TEST to check that command is completed with correct environment
+ os.environ['TEST'] = 'some_other_value'
+
+ # initial poll should result in None, since it takes a while for the command to complete
+ self.assertEqual(task.done(), False)
+
+ # wait until command is done
+ while not task.done():
+ time.sleep(1)
+ res = task.result()
+
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'sleeping...\ntest123\n')
+
+ # check asynchronous running of failing command
+ error_test_cmd = "echo 'FAIL!' >&2; exit 123"
+ task = thread_pool.submit(run_shell_cmd, error_test_cmd, hidden=True, fail_on_error=False, asynchronous=True)
+ time.sleep(1)
+ res = task.result()
+ self.assertEqual(res.exit_code, 123)
+ self.assertEqual(res.output, "FAIL!\n")
+ self.assertTrue(res.thread_id)
+
+ # also test with a command that produces a lot of output,
+ # since that tends to lock up things unless we frequently grab some output...
+ verbose_test_cmd = ';'.join([
+ "echo start",
+ "for i in $(seq 1 50)",
+ "do sleep 0.1",
+ "for j in $(seq 1000)",
+ "do echo foo${i}${j}",
+ "done",
+ "done",
+ "echo done",
+ ])
+ task = thread_pool.submit(run_shell_cmd, verbose_test_cmd, hidden=True, asynchronous=True)
+
+ while not task.done():
+ time.sleep(1)
+ res = task.result()
+
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(len(res.output), 435661)
+ self.assertTrue(res.output.startswith('start\nfoo11\nfoo12\n'))
+ self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res.output)
+ self.assertTrue(res.output.endswith('\nfoo501000\ndone\n'))
def test_check_log_for_errors(self):
+ """Test for check_log_for_errors"""
+
+ # use of check_log_for_errors is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")])
input_text = "\n".join([
"OK",
@@ -710,20 +1943,24 @@ def test_check_log_for_errors(self):
r"\tthe process crashed with 0"
# String promoted to list
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- r"\b(error|crashed)\b")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ r"\b(error|crashed)\b")
# List of string(s)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- [r"\b(error|crashed)\b"])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ [r"\b(error|crashed)\b"])
# List of tuple(s)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- [(r"\b(error|crashed)\b", ERROR)])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ [(r"\b(error|crashed)\b", ERROR)])
expected_msg = "Found 2 potential error(s) in command output:\n"\
"\terror found\n"\
"\tthe process crashed with 0"
init_logging(logfile, silent=True)
- check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)])
+ with self.mocked_stdout_stderr():
+ check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)])
stop_logging(logfile)
self.assertIn(expected_msg, read_file(logfile))
@@ -732,12 +1969,13 @@ def test_check_log_for_errors(self):
r"\ttest failed"
write_file(logfile, '')
init_logging(logfile, silent=True)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [
- r"\berror\b",
- (r"\ballowed-test failed\b", IGNORE),
- (r"(?i)\bCRASHED\b", WARN),
- "fail"
- ])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [
+ r"\berror\b",
+ (r"\ballowed-test failed\b", IGNORE),
+ (r"(?i)\bCRASHED\b", WARN),
+ "fail"
+ ])
stop_logging(logfile)
expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0"
self.assertIn(expected_msg, read_file(logfile))
@@ -746,6 +1984,10 @@ def test_run_cmd_with_hooks(self):
"""
Test running command with run_cmd with pre/post run_shell_cmd hooks in place.
"""
+
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cwd = os.getcwd()
hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
@@ -774,6 +2016,9 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
write_file(hooks_file, hooks_file_txt)
update_build_option('hooks', hooks_file)
+ # disable trace output to make checking of generated output produced by hooks easier
+ update_build_option('trace', False)
+
with self.mocked_stdout_stderr():
run_cmd("make")
stdout = self.get_stdout()
@@ -785,6 +2030,17 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
])
self.assertEqual(stdout, expected_stdout)
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("sleep 2; make", qa_patterns=[('q', 'a')])
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook interactive 'sleep 2; make' in %s" % cwd,
+ "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')",
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
with self.mocked_stdout_stderr():
run_cmd_qa("sleep 2; make", qa={})
stdout = self.get_stdout()
@@ -796,9 +2052,154 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
])
self.assertEqual(stdout, expected_stdout)
+ def test_run_shell_cmd_with_hooks(self):
+ """
+ Test running shell command with run_shell_cmd function with pre/post run_shell_cmd hooks in place.
+ """
+ cwd = os.getcwd()
+
+ hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
+ hooks_file_txt = textwrap.dedent("""
+ def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
+ work_dir = kwargs['work_dir']
+ if kwargs.get('interactive'):
+ print("pre-run hook interactive '||%s||' in %s" % (cmd, work_dir))
+ else:
+ print("pre-run hook '%s' in %s" % (cmd, work_dir))
+ import sys
+ sys.stderr.write('pre-run hook done\\n')
+ if not cmd.startswith('echo'):
+ cmds = cmd.split(';')
+ return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()])
+
+ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
+ exit_code = kwargs.get('exit_code')
+ output = kwargs.get('output')
+ work_dir = kwargs['work_dir']
+ if kwargs.get('interactive'):
+ msg = "post-run hook interactive '%s'" % cmd
+ else:
+ msg = "post-run hook '%s'" % cmd
+ msg += " (exit code: %s, output: '%s')" % (exit_code, output)
+ print(msg)
+ """)
+ write_file(hooks_file, hooks_file_txt)
+ update_build_option('hooks', hooks_file)
+
+ # disable trace output to make checking of generated output produced by hooks easier
+ update_build_option('trace', False)
+
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("make")
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ f"pre-run hook 'make' in {cwd}",
+ "post-run hook 'echo make' (exit code: 0, output: 'make\n')",
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
+ # also check in dry run mode, to verify that pre-run_shell_cmd hook is triggered sufficiently early
+ update_build_option('extended_dry_run', True)
+
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("make")
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook 'make' in %s" % cwd,
+ ' running shell command "echo make"',
+ ' (in %s)' % cwd,
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
+ # also check with trace output enabled
+ update_build_option('extended_dry_run', False)
+ update_build_option('trace', True)
+
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("make")
+ stdout = self.get_stdout()
+
+ regex = re.compile('>> running shell command:\n\techo make', re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ def test_run_shell_cmd_delete_cwd(self):
+ """
+ Test commands that destroy directories inside initial working directory
+ """
+ workdir = os.path.join(self.test_prefix, 'workdir')
+ sub_workdir = os.path.join(workdir, 'subworkdir')
+
+ # 1. test destruction of CWD which is a subdirectory inside original working directory
+ cmd_subworkdir_rm = (
+ "echo 'Command that jumps to subdir and removes it' && "
+ f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && "
+ "echo 'Working sub-directory removed.'"
+ )
+
+ # 1.a. in a robust system
+ expected_output = (
+ "Command that jumps to subdir and removes it\n"
+ f"{sub_workdir}\n"
+ "Working sub-directory removed.\n"
+ )
+
+ mkdir(sub_workdir, parents=True)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir)
+
+ self.assertEqual(res.cmd, cmd_subworkdir_rm)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, expected_output)
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, workdir)
+
+ # 1.b. in a flaky system that ends up in an unknown CWD after execution
+ mkdir(sub_workdir, parents=True)
+ fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
+ os.close(fd)
+
+ with self.mocked_stdout_stderr():
+ with mock.patch('os.getcwd') as mock_getcwd:
+ mock_getcwd.side_effect = [
+ workdir,
+ FileNotFoundError(),
+ ]
+ init_logging(logfile, silent=True)
+ res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir)
+ stop_logging(logfile)
+
+ self.assertEqual(res.cmd, cmd_subworkdir_rm)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, expected_output)
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, workdir)
+
+ expected_warning = f"Changing back to initial working directory: {workdir}\n"
+ logtxt = read_file(logfile)
+ self.assertTrue(logtxt.endswith(expected_warning))
+
+ # 2. test destruction of CWD which is main working directory passed to run_shell_cmd
+ cmd_workdir_rm = (
+ "echo 'Command that removes working directory' && pwd && "
+ f"rm -rf {workdir} && echo 'Working directory removed.'"
+ )
+
+ error_pattern = rf"Failed to return to .*/{os.path.basename(self.test_prefix)}/workdir after executing command"
+
+ mkdir(workdir, parents=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir)
+
def test_run_cmd_sysroot(self):
"""Test with_sysroot option of run_cmd function."""
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# put fake /bin/bash in place that will be picked up when using run_cmd with with_sysroot=True
bin_bash = os.path.join(self.test_prefix, 'bin', 'bash')
bin_bash_txt = '\n'.join([
@@ -811,13 +2212,15 @@ def test_run_cmd_sysroot(self):
update_build_option('sysroot', self.test_prefix)
- (out, ec) = run_cmd("echo hello")
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello")
self.assertEqual(ec, 0)
self.assertTrue(out.startswith("Hi there I am a fake /bin/bash in"))
self.assertTrue(out.endswith("\nhello\n"))
# picking up on alternate sysroot is enabled by default, but can be disabled via with_sysroot=False
- (out, ec) = run_cmd("echo hello", with_sysroot=False)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello", with_sysroot=False)
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
diff --git a/test/framework/sandbox/easybuild/easyblocks/b/binutils.py b/test/framework/sandbox/easybuild/easyblocks/b/binutils.py
new file mode 100644
index 0000000000..abae626602
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/b/binutils.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for binutils
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_binutils(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/b/bzip2.py b/test/framework/sandbox/easybuild/easyblocks/b/bzip2.py
new file mode 100644
index 0000000000..246da8f9d9
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/b/bzip2.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for bzip2
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_bzip2(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/c/cmake.py b/test/framework/sandbox/easybuild/easyblocks/c/cmake.py
new file mode 100644
index 0000000000..3bf181b3ac
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/c/cmake.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for CMake
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_CMake(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
index 7d2ef2d6da..3a15f26b70 100644
--- a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
+++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
index 0871ce62e1..67e23d27b9 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftwmpi.py b/test/framework/sandbox/easybuild/easyblocks/f/fftwmpi.py
new file mode 100644
index 0000000000..d7c468a8a9
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/f/fftwmpi.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for FFTW.MPI
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_FFTW_period_MPI(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/flex.py b/test/framework/sandbox/easybuild/easyblocks/f/flex.py
new file mode 100644
index 0000000000..d06874c3d4
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/f/flex.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for flex
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_flex(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py
index b5a91d9503..903a2fbc61 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
index 6415d6e6e1..86d2a5a934 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,6 +32,14 @@
from easybuild.framework.easyconfig import CUSTOM, MANDATORY
+class dummy1:
+ """Only to verify that unrelated classes in software specific easyblocks are ignored"""
+
+
+class dummy2(dummy1):
+ """Same but with inheritance"""
+
+
class EB_foofoo(EB_foo):
"""Support for building/installing foofoo."""
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/freetype.py b/test/framework/sandbox/easybuild/easyblocks/f/freetype.py
new file mode 100644
index 0000000000..9149ab8e42
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/f/freetype.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for freetype
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_freetype(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
index 5ee5aae1c6..203815b124 100644
--- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
+++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
index 8ba475f700..d1bd4700d5 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,6 +32,18 @@
from easybuild.framework.easyconfig import CUSTOM, MANDATORY
+class dummy1:
+ """Only to verify that unrelated classes in software specific easyblocks are ignored"""
+
+
+class dummy2(dummy1):
+ """Same but with inheritance"""
+
+
+class dummy3:
+ """Class without inheritance before the real easyblock to verify the regex not being too greedy"""
+
+
class bar(EasyBlock):
"""Generic support for building/installing bar."""
diff --git a/easybuild/toolchains/dummy.py b/test/framework/sandbox/easybuild/easyblocks/generic/bundle.py
similarity index 78%
rename from easybuild/toolchains/dummy.py
rename to test/framework/sandbox/easybuild/easyblocks/generic/bundle.py
index f29f6736ae..11cece6a3f 100644
--- a/easybuild/toolchains/dummy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/bundle.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -23,16 +23,12 @@
# along with EasyBuild. If not, see .
##
"""
-EasyBuild support for dummy compiler toolchain.
-
-Authors:
-
-* Kenneth Hoste (Ghent University)
+Dummy support for building and installing a bundle of applications.
"""
+from easybuild.framework.easyblock import EasyBlock
-from easybuild.toolchains.compiler.dummycompiler import DummyCompiler
+class Bundle(EasyBlock):
+ """Dummy support for building and installing a bundle of applications."""
-class Dummy(DummyCompiler):
- """Dummy toolchain."""
- NAME = 'dummy'
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py
new file mode 100644
index 0000000000..e5f80791b0
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py
@@ -0,0 +1,35 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with customized methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.customdummyextension import CustomDummyExtension
+
+
+class ChildCustomDummyExtension(CustomDummyExtension):
+ """Extension EasyBlock inheriting customized install step"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py
new file mode 100644
index 0000000000..e72b97940a
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py
@@ -0,0 +1,35 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with deprecated methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.deprecateddummyextension import DeprecatedDummyExtension
+
+
+class ChildDeprecatedDummyExtension(DeprecatedDummyExtension):
+ """Extension EasyBlock inheriting deprecated install step"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/cmakemake.py b/test/framework/sandbox/easybuild/easyblocks/generic/cmakemake.py
new file mode 100644
index 0000000000..d0728fabbe
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/cmakemake.py
@@ -0,0 +1,34 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy support for building and installing applications with CMake.
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class CMakeMake(EasyBlock):
+ """Dummy support for building and installing applications with CMake."""
+
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/cmdcp.py b/test/framework/sandbox/easybuild/easyblocks/generic/cmdcp.py
new file mode 100644
index 0000000000..f0850dac39
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/cmdcp.py
@@ -0,0 +1,34 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy support for building and installing applications via a command and copy.
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class CmdCp(EasyBlock):
+ """Dummy support for building and installing applications via a command and copy."""
+
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
index b82668434c..c616c97ddd 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py
new file mode 100644
index 0000000000..17a4434fe0
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py
@@ -0,0 +1,47 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with customized methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.dummyextension import DummyExtension
+
+
+class CustomDummyExtension(DummyExtension):
+ """Extension EasyBlock with customized install step"""
+
+ def pre_install_extension(self):
+
+ return "Extension installed with custom pre_install_extension()"
+
+ def install_extension(self):
+
+ return "Extension installed with custom install_extension()"
+
+ def post_install_extension(self):
+
+ return "Extension installed with custom post_install_extension()"
diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py
similarity index 62%
rename from easybuild/toolchains/compiler/dummycompiler.py
rename to test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py
index d74c2e873e..6e071417de 100644
--- a/easybuild/toolchains/compiler/dummycompiler.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -23,28 +23,25 @@
# along with EasyBuild. If not, see .
##
"""
-Support for dummy compiler.
+Test EasyBlocks building and installing dummy extensions with deprecated methods
-Authors:
-
-* Stijn De Weirdt (Ghent University)
-* Kenneth Hoste (Ghent University)
+@author: Alex Domingo (Vrije Universiteit Brussel)
"""
-from easybuild.tools.toolchain.compiler import Compiler
+from easybuild.easyblocks.generic.dummyextension import DummyExtension
+
+
+class DeprecatedDummyExtension(DummyExtension):
+ """Extension EasyBlock with deprecated install step"""
+ def prerun(self):
-TC_CONSTANT_DUMMY = "DUMMY"
+ return "Extension installed with custom prerun()"
+ def run(self):
-class DummyCompiler(Compiler):
- """Dummy compiler : try not to even use system gcc"""
- COMPILER_MODULE_NAME = []
- COMPILER_FAMILY = TC_CONSTANT_DUMMY
+ return "Extension installed with custom run()"
- COMPILER_CC = '%sCC' % TC_CONSTANT_DUMMY
- COMPILER_CXX = '%sCXX' % TC_CONSTANT_DUMMY
+ def postrun(self):
- COMPILER_F77 = '%sF77' % TC_CONSTANT_DUMMY
- COMPILER_F90 = '%sF90' % TC_CONSTANT_DUMMY
- COMPILER_FC = '%sFC' % TC_CONSTANT_DUMMY
+ return "Extension installed with custom postrun()"
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
index 3466ce28f2..e0429a450b 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
index dd89704638..dbe1790951 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/mesonninja.py b/test/framework/sandbox/easybuild/easyblocks/generic/mesonninja.py
new file mode 100644
index 0000000000..1251af1d0c
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/mesonninja.py
@@ -0,0 +1,34 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy support for building and installing applications with Meson and Ninja.
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class MesonNinja(EasyBlock):
+ """Dummy support for building and installing applications with Meson and Ninja."""
+
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
index 72252a46f4..aea939fda0 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/perlbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/perlbundle.py
new file mode 100644
index 0000000000..0644157561
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/perlbundle.py
@@ -0,0 +1,33 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for bundle of Perl modules.
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class PerlBundle(EasyBlock):
+ """Dummy support for bundle of Perl modules."""
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
index 9c01951adf..f93b5190fd 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonpackage.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonpackage.py
new file mode 100644
index 0000000000..35bc926829
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonpackage.py
@@ -0,0 +1,44 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy support for building and installing a Python package.
+"""
+from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig import CUSTOM
+
+
+class PythonPackage(EasyBlock):
+ """Dummy support for building and installing a Python package."""
+
+ @staticmethod
+ def extra_options(extra_vars=None):
+ """Extra easyconfig parameters specific to ConfigureMake."""
+ extra_vars = EasyBlock.extra_options(extra=extra_vars)
+ extra_vars.update({
+ 'install_target': ['', "Just a test", CUSTOM],
+ 'options': ['', "Another test", CUSTOM],
+ 'use_pip': ['', "Test 1, 2, 3", CUSTOM],
+ })
+ return extra_vars
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/tarball.py b/test/framework/sandbox/easybuild/easyblocks/generic/tarball.py
new file mode 100644
index 0000000000..012c05b67b
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/tarball.py
@@ -0,0 +1,34 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy support for building and installing applications from a Tarball.
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class Tarball(EasyBlock):
+ """Dummy support for building and installing applications from a Tarball."""
+
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
index 5151221a13..39dfde65b4 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
index 9335611f58..12a1e5830b 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -27,12 +27,13 @@
@author: Kenneth Hoste (Ghent University)
"""
+import os
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
class Toy_Extension(ExtensionEasyBlock):
@@ -59,7 +60,17 @@ def required_deps(self):
else:
raise EasyBuildError("Dependencies for %s are unknown!", self.name)
- def run(self, *args, **kwargs):
+ def pre_install_extension(self):
+ """
+ Prepare installation of toy extension.
+ """
+ super(Toy_Extension, self).pre_install_extension()
+
+ if self.src:
+ super(Toy_Extension, self).install_extension(unpack_src=True)
+ EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg)
+
+ def install_extension(self, *args, **kwargs):
"""
Install toy extension.
"""
@@ -67,35 +78,28 @@ def run(self, *args, **kwargs):
EB_toy.build_step(self.master, name=self.name, cfg=self.cfg)
if self.cfg['toy_ext_param']:
- run_cmd(self.cfg['toy_ext_param'])
+ run_shell_cmd(self.cfg['toy_ext_param'])
return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name)
- def prerun(self):
- """
- Prepare installation of toy extension.
- """
- super(Toy_Extension, self).prerun()
-
- if self.src:
- super(Toy_Extension, self).run(unpack_src=True)
- EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg)
-
- def run_async(self):
+ def install_extension_async(self, thread_pool):
"""
Install toy extension asynchronously.
"""
+ task_id = f'ext_{self.name}_{self.version}'
if self.src:
cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
- self.async_cmd_start(cmd)
else:
- self.async_cmd_info = False
+ cmd = f"echo 'no sources for {self.name}'"
+
+ return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
+ fail_on_error=False, task_id=task_id, work_dir=os.getcwd())
- def postrun(self):
+ def post_install_extension(self):
"""
Wrap up installation of toy extension.
"""
- super(Toy_Extension, self).postrun()
+ super(Toy_Extension, self).post_install_extension()
EB_toy.install_step(self.master, name=self.name)
diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
index db929d4d31..ffc2263839 100644
--- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
+++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
index d3e55ed05d..51b089e424 100644
--- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2024 Ghent University
+# Copyright 2021-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -30,7 +30,7 @@
import os
from easybuild.framework.easyblock import EasyBlock
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
SHLIB_EXT = get_shared_lib_ext()
@@ -40,7 +40,7 @@ class EB_libtoy(EasyBlock):
"""Support for building/installing libtoy."""
def banned_linked_shared_libs(self):
- default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT
+ default = f'/thiswillnotbethere,libtoytoytoy.{SHLIB_EXT},toytoytoy'
return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',')
def required_linked_shared_libs(self):
@@ -53,8 +53,8 @@ def configure_step(self, name=None):
def build_step(self, name=None, buildopts=None):
"""Build libtoy."""
- run_cmd('make')
+ run_shell_cmd('make')
def install_step(self, name=None):
"""Install libtoy."""
- run_cmd('make install PREFIX="%s"' % self.installdir)
+ run_shell_cmd(f'make install PREFIX="{self.installdir}"')
diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libxml2.py b/test/framework/sandbox/easybuild/easyblocks/l/libxml2.py
new file mode 100644
index 0000000000..d0153d43a3
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/l/libxml2.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for libxml2
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_libxml2(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/l/llvm.py b/test/framework/sandbox/easybuild/easyblocks/l/llvm.py
new file mode 100644
index 0000000000..b59bae4acb
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/l/llvm.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for LLVM
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_LLVM(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/m/mesa.py b/test/framework/sandbox/easybuild/easyblocks/m/mesa.py
new file mode 100644
index 0000000000..3086985eb1
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/m/mesa.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for Mesa
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_Mesa(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
index e5dc093347..600998cc1c 100644
--- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
+++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
index b110b4f921..79b6ef1406 100644
--- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
+++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openssl_wrapper.py b/test/framework/sandbox/easybuild/easyblocks/o/openssl_wrapper.py
new file mode 100644
index 0000000000..d3f32361dd
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/o/openssl_wrapper.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for OpenSSL_wrapper
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_OpenSSL_wrapper(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/p/perl.py b/test/framework/sandbox/easybuild/easyblocks/p/perl.py
new file mode 100644
index 0000000000..463b14b57d
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/p/perl.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for Perl
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_Perl(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/p/python.py b/test/framework/sandbox/easybuild/easyblocks/p/python.py
new file mode 100644
index 0000000000..e6e43ed0cf
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/p/python.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for Python
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_Python(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
index 3ded1247f4..1fb37fbe5c 100644
--- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
+++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
index 04186a37ef..5c434e31c7 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,7 +38,7 @@
from easybuild.tools.environment import setvar
from easybuild.tools.filetools import mkdir, write_file
from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts):
@@ -73,6 +73,10 @@ def __init__(self, *args, **kwargs):
setvar('TOY', '%s-%s' % (self.name, self.version))
+ # extra paths for environment variables to consider
+ if self.name == 'toy':
+ self.module_load_environment.CPATH.append('toy-headers')
+
def prepare_for_extensions(self):
"""
Prepare for installing toy extensions.
@@ -108,7 +112,7 @@ def configure_step(self, name=None, cfg=None):
'echo "Configured"',
cfg['configopts']
])
- run_cmd(cmd)
+ run_shell_cmd(cmd)
if os.path.exists("%s.source" % name):
os.rename('%s.source' % name, '%s.c' % name)
@@ -124,8 +128,8 @@ def build_step(self, name=None, cfg=None):
cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts'])
# purposely run build command without checking exit code;
# we rely on this in test_toy_build_hooks
- (out, ec) = run_cmd(cmd, log_ok=False, log_all=False)
- if ec:
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ if res.exit_code:
print_warning("Command '%s' failed, but we'll ignore it..." % cmd)
def install_step(self, name=None):
@@ -142,6 +146,12 @@ def install_step(self, name=None):
mkdir(libdir, parents=True)
write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper())
+ def post_processing_step(self):
+ """Any postprocessing for toy"""
+ libdir = os.path.join(self.installdir, 'lib')
+ write_file(os.path.join(libdir, 'lib%s_post.a' % self.name), self.name.upper())
+ super(EB_toy, self).post_processing_step()
+
@property
def required_deps(self):
"""Return list of required dependencies for this extension."""
@@ -150,27 +160,29 @@ def required_deps(self):
else:
raise EasyBuildError("Dependencies for %s are unknown!", self.name)
- def prerun(self):
+ def pre_install_extension(self):
"""
Prepare installation of toy as extension.
"""
- super(EB_toy, self).run(unpack_src=True)
+ super(EB_toy, self).install_extension(unpack_src=True)
self.configure_step()
- def run(self):
+ def install_extension(self):
"""
Install toy as extension.
"""
self.build_step()
- def run_async(self):
+ def install_extension_async(self, thread_pool):
"""
Asynchronous installation of toy as extension.
"""
cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
- self.async_cmd_start(cmd)
+ task_id = f'ext_{self.name}_{self.version}'
+ return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
+ fail_on_error=False, task_id=task_id, work_dir=os.getcwd())
- def postrun(self):
+ def post_install_extension(self):
"""
Wrap up installation of toy as extension.
"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
index 6445655707..ba3efd236f 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -41,7 +41,7 @@ def configure_step(self):
def build_step(self):
"""Build toy."""
# note: import is (purposely) missing, so this will go down hard
- run_cmd('gcc toy.c -o toy') # noqa
+ run_shell_cmd('gcc toy.c -o toy') # noqa
def install_step(self):
"""Install toy."""
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_deprecated.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_deprecated.py
new file mode 100644
index 0000000000..80891ff152
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_deprecated.py
@@ -0,0 +1,40 @@
+##
+# Copyright 2009-2024 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+(deprecated) EasyBuild support for building and installing toy, implemented as an easyblock
+
+@author: Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
+"""
+
+from easybuild.easyblocks.toy import EB_toy
+
+
+class EB_toy_deprecated(EB_toy):
+ """Support for building/installing toy with deprecated post_install step."""
+
+ def post_install_step(self):
+ """Any postprocessing for toy (deprecated)"""
+ print("This step is deprecated.")
+ super(EB_toy, self).post_install_step()
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
index 09442d1a79..add589838d 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
@@ -1,5 +1,5 @@
##
-# Copyright 2020-2024 Ghent University
+# Copyright 2020-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,6 +37,6 @@ def prepare_step(self, *args, **kwargs):
"""Constructor"""
super(EB_toy_eula, self).prepare_step(*args, **kwargs)
- # EULA for toy must be accepted via --accept-eula EasyBuild configuration option,
+ # EULA for toy must be accepted via --accept-eula-for EasyBuild configuration option,
# or via 'accept_eula = True' in easyconfig file
self.check_accepted_eula(more_info='https://example.com/toy_eula.txt')
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
index a3ef746d53..2cb97836c3 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/x/xcrysden.py b/test/framework/sandbox/easybuild/easyblocks/x/xcrysden.py
new file mode 100644
index 0000000000..4c46df68a0
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/x/xcrysden.py
@@ -0,0 +1,32 @@
+##
+# Copyright 2009-2025 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Dummy easyblock for XCrySDen
+"""
+from easybuild.framework.easyblock import EasyBlock
+
+
+class EB_XCrySDen(EasyBlock):
+ pass
diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py
index 389c5ff367..b1a57625bf 100644
--- a/test/framework/sandbox/easybuild/tools/__init__.py
+++ b/test/framework/sandbox/easybuild/tools/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2024 Ghent University
+# Copyright 2009-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
index a519340298..7eca596ebb 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2024 Ghent University
+# Copyright 2011-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
index 9113d89045..5fd30dadba 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
index 424fb55d43..47d1b55ccc 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
index a0e0429bfe..dbfb79a6f8 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz b/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz
new file mode 100644
index 0000000000..a1583ec2c0
Binary files /dev/null and b/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz differ
diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch
new file mode 100644
index 0000000000..7512447168
--- /dev/null
+++ b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch
@@ -0,0 +1,10 @@
+--- a/toy-0.0.orig/toy.source 2014-03-06 18:48:16.000000000 +0100
++++ b/toy-0.0/toy.source 2020-08-18 12:19:35.000000000 +0200
+@@ -2,6 +2,6 @@
+
+ int main(int argc, char* argv[]){
+
+- printf("I'm a toy, and proud of it.\n");
++ printf("I'm a toy, and proud of it.\n")
+ return 0;
+ }
diff --git a/test/framework/style.py b/test/framework/style.py
index bfdf4e026a..0c1d9b140a 100644
--- a/test/framework/style.py
+++ b/test/framework/style.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2024 Ghent University
+# Copyright 2016-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,10 +40,7 @@
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
+ pass
class StyleTest(EnhancedTestCase):
@@ -51,8 +48,8 @@ class StyleTest(EnhancedTestCase):
def test_style_conformance(self):
"""Check the easyconfigs for style"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping style checks (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_style_conformance pycodestyle is not available")
return
# all available easyconfig files
@@ -66,8 +63,8 @@ def test_style_conformance(self):
def test_check_trailing_whitespace(self):
"""Test for trailing whitespace check."""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping trailing whitespace checks (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_trailing_whitespace is not available")
return
lines = [
@@ -77,6 +74,12 @@ def test_check_trailing_whitespace(self):
'''description = """start of long description, ''', # trailing whitespace, but allowed in description
''' continuation of long description ''', # trailing whitespace, but allowed in continued description
''' end of long description"""''',
+ '''citing = """start of long citing text, ''', # trailing whitespace, but allowed in citing
+ ''' continuation of long citing text ''', # trailing whitespace, but allowed in continued citing
+ ''' end of long citing text"""''',
+ '''examples = """start of long examples, ''', # trailing whitespace, but allowed in examples
+ ''' continuation of long examples ''', # trailing whitespace, but allowed in continued examples
+ ''' end of long examples"""''',
"moduleclass = 'tools' ", # trailing whitespace
'',
]
@@ -89,6 +92,12 @@ def test_check_trailing_whitespace(self):
None,
None,
None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
(21, "W299 trailing whitespace"),
]
diff --git a/test/framework/suite.py b/test/framework/suite.py
index 3b8193469c..afec127c83 100755
--- a/test/framework/suite.py
+++ b/test/framework/suite.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -81,7 +81,6 @@
import test.framework.tweak as tw
import test.framework.utilities_test as u
import test.framework.variables as v
-import test.framework.yeb as y
# set plain text key ring to be used,
# so a GitHub token stored in it can be obtained without having to provide a password
@@ -121,7 +120,7 @@
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
tests = [gen, d, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c,
- tw, p, i, pkg, env, et, y, st, h, ct, lib, u, es, ou]
+ tw, p, i, pkg, env, et, st, h, ct, lib, u, es, ou]
SUITE = unittest.TestSuite([x.suite() for x in tests])
res = unittest.TextTestRunner().run(SUITE)
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index deece97702..7529166d72 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,8 +40,7 @@
import easybuild.tools.systemtools as st
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file
-from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import RunShellCmdResult, run_shell_cmd
from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64
from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN
from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL
@@ -321,8 +320,8 @@ def mocked_is_readable(mocked_fp, fp):
return fp == mocked_fp
-def mocked_run_cmd(cmd, **kwargs):
- """Mocked version of run_cmd, with specified output for known commands."""
+def mocked_run_shell_cmd(cmd, **kwargs):
+ """Mocked version of run_shell_cmd, with specified output for known commands."""
known_cmds = {
"gcc --version": "gcc (GCC) 5.1.1 20150618 (Red Hat 5.1.1-4)",
"ldd --version": "ldd (GNU libc) 2.12; ",
@@ -341,12 +340,10 @@ def mocked_run_cmd(cmd, **kwargs):
"ulimit -u": '40',
}
if cmd in known_cmds:
- if 'simple' in kwargs and kwargs['simple']:
- return True
- else:
- return (known_cmds[cmd], 0)
+ return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(),
+ out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None)
else:
- return run_cmd(cmd, **kwargs)
+ return run_shell_cmd(cmd, **kwargs)
def mocked_uname():
@@ -365,8 +362,8 @@ def setUp(self):
self.orig_get_os_type = st.get_os_type
self.orig_is_readable = st.is_readable
self.orig_read_file = st.read_file
- self.orig_run_cmd = st.run_cmd
- self.orig_platform_dist = getattr(st.platform, 'dist', None)
+ self.orig_run_shell_cmd = st.run_shell_cmd
+ self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None
self.orig_platform_uname = st.platform.uname
self.orig_get_tool_version = st.get_tool_version
self.orig_sys_version_info = st.sys.version_info
@@ -382,7 +379,7 @@ def tearDown(self):
st.get_cpu_architecture = self.orig_get_cpu_architecture
st.get_os_name = self.orig_get_os_name
st.get_os_type = self.orig_get_os_type
- st.run_cmd = self.orig_run_cmd
+ st.run_shell_cmd = self.orig_run_shell_cmd
if self.orig_platform_dist is not None:
st.platform.dist = self.orig_platform_dist
st.platform.uname = self.orig_platform_uname
@@ -413,13 +410,13 @@ def test_avail_core_count_linux(self):
def test_avail_core_count_darwin(self):
"""Test getting core count (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_avail_core_count(), 10)
def test_cpu_model_native(self):
"""Test getting CPU model."""
cpu_model = get_cpu_model()
- self.assertIsInstance(cpu_model, string_type)
+ self.assertIsInstance(cpu_model, str)
def test_cpu_model_linux(self):
"""Test getting CPU model (mocked for Linux)."""
@@ -451,7 +448,7 @@ def test_cpu_model_linux(self):
def test_cpu_model_darwin(self):
"""Test getting CPU model (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz")
def test_cpu_speed_native(self):
@@ -486,7 +483,7 @@ def test_cpu_speed_linux(self):
def test_cpu_speed_darwin(self):
"""Test getting CPU speed (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_cpu_speed(), 2400.0)
def test_cpu_features_native(self):
@@ -494,7 +491,7 @@ def test_cpu_features_native(self):
cpu_feat = get_cpu_features()
self.assertIsInstance(cpu_feat, list)
self.assertTrue(len(cpu_feat) >= 0)
- self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat))
+ self.assertTrue(all(isinstance(x, str) for x in cpu_feat))
def test_cpu_features_linux(self):
"""Test getting CPU features (mocked for Linux)."""
@@ -541,7 +538,7 @@ def test_cpu_features_linux(self):
def test_cpu_features_darwin(self):
"""Test getting CPU features (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
expected = ['1gbpage', 'acpi', 'aes', 'apic', 'avx1.0', 'avx2', 'bmi1', 'bmi2', 'clfsh', 'cmov', 'cx16',
'cx8', 'de', 'ds', 'dscpl', 'dtes64', 'em64t', 'erms', 'est', 'f16c', 'fma', 'fpu', 'fpu_csds',
'fxsr', 'htt', 'invpcid', 'lahf', 'lzcnt', 'mca', 'mce', 'mmx', 'mon', 'movbe', 'msr', 'mtrr',
@@ -578,7 +575,7 @@ def test_cpu_architecture(self):
def test_cpu_arch_name_native(self):
"""Test getting CPU arch name."""
arch_name = get_cpu_arch_name()
- self.assertIsInstance(arch_name, string_type)
+ self.assertIsInstance(arch_name, str)
def test_cpu_arch_name(self):
"""Test getting CPU arch name."""
@@ -635,12 +632,12 @@ def test_cpu_vendor_linux(self):
def test_cpu_vendor_darwin(self):
"""Test getting CPU vendor (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_cpu_vendor(), INTEL)
def test_cpu_family_native(self):
"""Test get_cpu_family function."""
- run_cmd.clear_cache()
+ run_shell_cmd.clear_cache()
cpu_family = get_cpu_family()
self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN)
@@ -685,8 +682,8 @@ def test_cpu_family_linux(self):
def test_cpu_family_darwin(self):
"""Test get_cpu_family function (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
- run_cmd.clear_cache()
+ st.run_shell_cmd = mocked_run_shell_cmd
+ run_shell_cmd.clear_cache()
self.assertEqual(get_cpu_family(), INTEL)
def test_os_type(self):
@@ -712,12 +709,12 @@ def test_shared_lib_ext_darwin(self):
def test_platform_name_native(self):
"""Test getting platform name."""
platform_name_nover = get_platform_name()
- self.assertIsInstance(platform_name_nover, string_type)
+ self.assertIsInstance(platform_name_nover, str)
len_nover = len(platform_name_nover.split('-'))
self.assertTrue(len_nover >= 3)
platform_name_ver = get_platform_name(withversion=True)
- self.assertIsInstance(platform_name_ver, string_type)
+ self.assertIsInstance(platform_name_ver, str)
len_ver = len(platform_name_ver.split('-'))
self.assertTrue(platform_name_ver.startswith(platform_name_ver))
self.assertTrue(len_ver >= len_nover)
@@ -737,12 +734,12 @@ def test_platform_name_darwin(self):
def test_os_name(self):
"""Test getting OS name."""
os_name = get_os_name()
- self.assertTrue(isinstance(os_name, string_type) or os_name == UNKNOWN)
+ self.assertTrue(isinstance(os_name, str) or os_name == UNKNOWN)
def test_os_version(self):
"""Test getting OS version."""
os_version = get_os_version()
- self.assertTrue(isinstance(os_version, string_type) or os_version == UNKNOWN)
+ self.assertTrue(isinstance(os_version, str) or os_version == UNKNOWN)
# make sure that bug fixed in https://github.com/easybuilders/easybuild-framework/issues/3952
# does not surface again, by mocking what's needed to make get_os_version fall into SLES-specific path
@@ -763,29 +760,33 @@ def test_gcc_version_native(self):
"""Test getting gcc version."""
gcc_version = get_gcc_version()
if gcc_version is not None:
- self.assertIsInstance(gcc_version, string_type)
+ self.assertIsInstance(gcc_version, str)
def test_gcc_version_linux(self):
"""Test getting gcc version (mocked for Linux)."""
st.get_os_type = lambda: st.LINUX
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_gcc_version(), '5.1.1')
def test_gcc_version_darwin(self):
"""Test getting gcc version (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = lambda *args, **kwargs: ("Apple LLVM version 7.0.0 (clang-700.1.76)", 0)
+ out = "Apple LLVM version 7.0.0 (clang-700.1.76)"
+ cwd = os.getcwd()
+ mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd,
+ out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None)
+ st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res
self.assertEqual(get_gcc_version(), None)
def test_glibc_version_native(self):
"""Test getting glibc version."""
glibc_version = get_glibc_version()
- self.assertTrue(isinstance(glibc_version, string_type) or glibc_version == UNKNOWN)
+ self.assertTrue(isinstance(glibc_version, str) or glibc_version == UNKNOWN)
def test_glibc_version_linux(self):
"""Test getting glibc version (mocked for Linux)."""
st.get_os_type = lambda: st.LINUX
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_glibc_version(), '2.12')
def test_glibc_version_linux_gentoo(self):
@@ -816,7 +817,7 @@ def test_get_total_memory_linux(self):
def test_get_total_memory_darwin(self):
"""Test the function that gets the total memory."""
st.get_os_type = lambda: st.DARWIN
- st.run_cmd = mocked_run_cmd
+ st.run_shell_cmd = mocked_run_shell_cmd
self.assertEqual(get_total_memory(), 8192)
def test_get_total_memory_native(self):
@@ -847,11 +848,13 @@ def test_det_parallelism_mocked(self):
# mock number of available cores to 8
st.get_avail_core_count = lambda: 8
self.assertTrue(det_parallelism(), 8)
+
# make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6)
- st.run_cmd = mocked_run_cmd
- self.assertTrue(det_parallelism(), 4)
- self.assertTrue(det_parallelism(par=6), 4)
- self.assertTrue(det_parallelism(maxpar=2), 2)
+ del det_parallelism._default_parallelism
+ st.run_shell_cmd = mocked_run_shell_cmd
+ self.assertEqual(det_parallelism(), 4)
+ self.assertEqual(det_parallelism(par=6), 6)
+ self.assertEqual(det_parallelism(maxpar=2), 2)
st.get_avail_core_count = orig_get_avail_core_count
@@ -872,19 +875,23 @@ def mock_python_ver(py_maj_ver, py_min_ver):
# mock running with different Python versions
mock_python_ver(1, 4)
- error_pattern = r"EasyBuild is not compatible \(yet\) with Python 1.4"
+ error_pattern = r"EasyBuild is not compatible with Python 1.4"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
mock_python_ver(4, 0)
error_pattern = r"EasyBuild is not compatible \(yet\) with Python 4.0"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
- mock_python_ver(2, 6)
- error_pattern = r"Python 2.7 is required when using Python 2, found Python 2.6"
+ mock_python_ver(2, 7)
+ error_pattern = r"EasyBuild is not compatible with Python 2.7"
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
+
+ mock_python_ver(3, 5)
+ error_pattern = r"Python 3.6 or higher is required, found Python 3.5"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
# no problems when running with a supported Python version
- for pyver in [(2, 7), (3, 5), (3, 6), (3, 7)]:
+ for pyver in [(3, 6), (3, 7), (3, 11)]:
mock_python_ver(*pyver)
self.assertEqual(check_python_version(), pyver)
@@ -1028,19 +1035,21 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(txt_path), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path), None)
- bin_ls_path = which('ls')
+ bin_bash_path = which('bash')
os_type = get_os_type()
if os_type == LINUX:
- out, _ = run_cmd("ldd %s" % bin_ls_path)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("ldd %s" % bin_bash_path)
elif os_type == DARWIN:
- out, _ = run_cmd("otool -L %s" % bin_ls_path)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("otool -L %s" % bin_bash_path)
else:
raise EasyBuildError("Unknown OS type: %s" % os_type)
shlib_ext = get_shared_lib_ext()
lib_path_regex = re.compile(r'(?P[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M)
- lib_path = lib_path_regex.search(out).group(1)
+ lib_path = lib_path_regex.search(res.output).group(1)
test_pattern_named_args = [
# if no patterns are specified, result is always True
@@ -1054,7 +1063,7 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
- for path in (bin_ls_path, lib_path):
+ for path in (bin_bash_path, lib_path):
# path may not exist, especially for library paths obtained via 'otool -L' on macOS
if os.path.exists(path):
error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args)
@@ -1071,7 +1080,7 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
- for path in (bin_ls_path, lib_path):
+ for path in (bin_bash_path, lib_path):
error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args)
self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg)
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index beeb2e3512..e55ccabe93 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -50,8 +50,8 @@
from easybuild.tools.environment import setvar
from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir
from easybuild.tools.filetools import read_file, symlink, write_file, which
-from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.run import run_cmd
+from easybuild.tools.modules import EnvironmentModules
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.toolchain.mpi import get_mpi_cmd_template
from easybuild.tools.toolchain.toolchain import env_vars_external_module
@@ -92,7 +92,7 @@ def get_toolchain(self, name, version=None):
def test_toolchain(self):
"""Test whether toolchain is initialized correctly."""
test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')
- ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb'))
+ ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb'))
ec = EasyConfig(ec_file, validate=False)
tc = ec.toolchain
self.assertIn('debug', tc.options)
@@ -114,24 +114,13 @@ def test_foss_toolchain(self):
self.get_toolchain("foss", version="2018a")
def test_get_variable_system_toolchain(self):
- """Test get_variable on system/dummy toolchain"""
+ """Test get_variable on system toolchain"""
# system toolchain version doesn't really matter, but fine...
for ver in ['system', '']:
tc = self.get_toolchain('system', version=ver)
- tc.prepare()
- self.assertEqual(tc.get_variable('CC'), '')
- self.assertEqual(tc.get_variable('CXX', typ=str), '')
- self.assertEqual(tc.get_variable('CFLAGS', typ=list), [])
-
- # dummy toolchain is deprecated, so we need to allow for it (and catch the warnings that get printed)
- self.allow_deprecated_behaviour()
-
- for ver in ['dummy', '']:
- self.mock_stderr(True)
- tc = self.get_toolchain('dummy', version=ver)
- self.mock_stderr(False)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), '')
self.assertEqual(tc.get_variable('CXX', typ=str), '')
self.assertEqual(tc.get_variable('CFLAGS', typ=list), [])
@@ -153,27 +142,10 @@ def test_is_system_toolchain(self):
tc = self.get_toolchain('intel', version='2018a')
self.assertFalse(tc.is_system_toolchain())
- # using dummy toolchain is deprecated, so to test for that we need to explicitely allow using deprecated stuff
- error_pattern = "Use of 'dummy' toolchain is deprecated"
- for ver in ['dummy', '']:
- self.assertErrorRegex(EasyBuildError, error_pattern, self.get_toolchain, 'dummy', version=ver)
-
- dummy_depr_warning = "WARNING: Deprecated functionality, will no longer work in v5.0: Use of 'dummy' toolchain"
-
- self.allow_deprecated_behaviour()
-
- for ver in ['dummy', '']:
- self.mock_stderr(True)
- tc = self.get_toolchain('dummy', version=ver)
- stderr = self.get_stderr()
- self.mock_stderr(False)
- self.assertTrue(tc.is_system_toolchain())
- self.assertIn(dummy_depr_warning, stderr)
-
def test_toolchain_prepare_sysroot(self):
"""Test build environment setup done by Toolchain.prepare in case --sysroot is specified."""
- sysroot = os.path.join(self.test_prefix, 'test', 'alternate', 'sysroot')
+ sysroot = os.path.join(self.test_prefix, 'test', 'alternative', 'sysroot')
sysroot_pkgconfig = os.path.join(sysroot, 'usr', 'lib', 'pkgconfig')
mkdir(sysroot_pkgconfig, parents=True)
init_config(build_options={'sysroot': sysroot})
@@ -186,7 +158,8 @@ def test_toolchain_prepare_sysroot(self):
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), None)
tc = self.get_toolchain('system', version='system')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), sysroot_pkgconfig)
# usr/lib64/pkgconfig is also picked up
@@ -195,25 +168,29 @@ def test_toolchain_prepare_sysroot(self):
init_config(build_options={'sysroot': sysroot})
del os.environ['PKG_CONFIG_PATH']
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), sysroot_pkgconfig)
# existing $PKG_CONFIG_PATH value is retained
test_pkg_config_path = os.pathsep.join([self.test_prefix, '/foo/bar'])
os.environ['PKG_CONFIG_PATH'] = test_pkg_config_path
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), test_pkg_config_path + os.pathsep + sysroot_pkgconfig)
# no duplicate paths are added
test_pkg_config_path = os.pathsep.join([self.test_prefix, sysroot_pkgconfig, '/foo/bar'])
os.environ['PKG_CONFIG_PATH'] = test_pkg_config_path
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), test_pkg_config_path)
# if no usr/lib*/pkgconfig subdirectory is present in sysroot, then $PKG_CONFIG_PATH is not touched
del os.environ['PKG_CONFIG_PATH']
init_config(build_options={'sysroot': self.test_prefix})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), None)
def unset_compiler_env_vars(self):
@@ -314,7 +291,8 @@ def test_toolchain_compiler_env_vars(self):
# check whether specification in --minimal-build-env is picked up
init_config(build_options={'minimal_build_env': 'CC:g++'})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'g++')
self.assertEqual(os.getenv('CXX'), None)
@@ -361,23 +339,28 @@ def test_toolchain_compiler_env_vars(self):
# incorrect spec in minimal_build_env results in an error
init_config(build_options={'minimal_build_env': 'CC=gcc'})
error_pattern = "Incorrect mapping in --minimal-build-env value: 'CC=gcc'"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
init_config(build_options={'minimal_build_env': 'foo:bar:baz'})
error_pattern = "Incorrect mapping in --minimal-build-env value: 'foo:bar:baz'"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
init_config(build_options={'minimal_build_env': 'CC:gcc,foo'})
error_pattern = "Incorrect mapping in --minimal-build-env value: 'foo'"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
init_config(build_options={'minimal_build_env': 'foo:bar:baz,CC:gcc'})
error_pattern = "Incorrect mapping in --minimal-build-env value: 'foo:bar:baz'"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
init_config(build_options={'minimal_build_env': 'CC:gcc,'})
error_pattern = "Incorrect mapping in --minimal-build-env value: ''"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
# for a full toolchain, a more extensive build environment is set up (incl. $CFLAGS & co),
# and the specs in --minimal-build-env are ignored
@@ -385,9 +368,8 @@ def test_toolchain_compiler_env_vars(self):
tc.set_options({})
# catch potential warning about too long $TMPDIR value that causes trouble for Open MPI (irrelevant here)
- self.mock_stderr(True)
- tc.prepare()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'gcc')
self.assertEqual(os.getenv('CXX'), 'g++')
@@ -415,7 +397,8 @@ def test_get_variable_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'gcc')
self.assertEqual(tc.get_variable('CXX'), 'g++')
@@ -458,7 +441,8 @@ def test_get_variable_mpi_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.check_vars_foss_usempi(tc)
@@ -467,15 +451,18 @@ def test_prepare_iterate(self):
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.check_vars_foss_usempi(tc)
# without a reset, the value is wrong...
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertFalse(tc.get_variable('MPICC') == 'mpicc')
tc.reset()
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.check_vars_foss_usempi(tc)
def test_cray_reset(self):
@@ -484,25 +471,34 @@ def test_cray_reset(self):
init_config(build_options={'optarch': 'test', 'silent': True})
for tcname in ['CrayGNU', 'CrayCCE', 'CrayIntel']:
+ # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+ # makes environment inconsistent which is not allowed by Environment Modules tool
+ if isinstance(self.modtool, EnvironmentModules):
+ self.modtool.purge()
tc = self.get_toolchain(tcname, version='2015.06-XC')
tc.set_options({'dynamic': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS'), '')
tc.reset()
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS'), '')
tc.reset()
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS'), '')
tc.reset()
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS'), '')
def test_get_variable_seq_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC_SEQ'), 'gcc')
self.assertEqual(tc.get_variable('CXX_SEQ'), 'g++')
@@ -513,12 +509,13 @@ def test_get_variable_seq_compilers(self):
def test_get_variable_libs_list(self):
"""Test get_variable function to obtain list of libraries."""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
ldflags = tc.get_variable('LDFLAGS', typ=list)
self.assertIsInstance(ldflags, list)
if len(ldflags) > 0:
- self.assertIsInstance(ldflags[0], string_type)
+ self.assertIsInstance(ldflags[0], str)
def test_validate_pass_by_value(self):
"""
@@ -526,7 +523,8 @@ def test_validate_pass_by_value(self):
which is required to ensure correctness.
"""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
pass_by_value = True
ids = []
@@ -551,7 +549,8 @@ def test_optimization_flags(self):
# check default optimization flag (e.g. -O2)
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
self.assertIn(tc.COMPILER_SHARED_OPTION_MAP['defaultopt'], flags)
@@ -561,7 +560,8 @@ def test_optimization_flags(self):
tc = self.get_toolchain('foss', version='2018a')
for enable in [True, False]:
tc.set_options({opt: enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
@@ -579,28 +579,31 @@ def test_optimization_flags_combos(self):
# lowest optimization should always be picked
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'lowopt': True, 'opt': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
- flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt']
+ flag = tc.COMPILER_SHARED_OPTION_MAP['lowopt']
self.assertIn(flag, flags)
self.modtool.purge()
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'noopt': True, 'lowopt': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
- flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
+ flag = tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertIn(flag, flags)
self.modtool.purge()
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'noopt': True, 'lowopt': True, 'opt': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
- flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
+ flag = tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertIn(flag, flags)
def test_misc_flags_shared(self):
@@ -613,9 +616,10 @@ def test_misc_flags_shared(self):
for enable in [True, False]:
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({opt: enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# we need to make sure we check for flags, not letter (e.g. 'v' vs '-v')
- flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt]
+ flag = tc.COMPILER_SHARED_OPTION_MAP[opt]
for var in flag_vars:
flags = tc.get_variable(var).split()
if enable:
@@ -629,7 +633,8 @@ def test_misc_flags_shared(self):
opt = 'extra_' + var.lower()
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({opt: value})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertTrue(tc.get_variable(var).endswith(' ' + value))
self.modtool.purge()
@@ -637,7 +642,8 @@ def test_misc_flags_shared(self):
flag_vars.remove('CXXFLAGS')
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'extra_cxxflags': value})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertTrue(tc.get_variable('CXXFLAGS').endswith(' ' + value))
for var in flag_vars:
self.assertNotIn(value, tc.get_variable(var))
@@ -656,7 +662,8 @@ def test_misc_flags_unique(self):
for enable in [True, False]:
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({opt: enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
if opt == 'optarch':
option = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)]
else:
@@ -665,8 +672,7 @@ def test_misc_flags_unique(self):
option = {True: option}
for var in flag_vars:
flags = tc.get_variable(var)
- for key, value in option.items():
- flag = "-%s" % value
+ for key, flag in option.items():
if enable == key:
self.assertIn(flag, flags, "%s: %s means %s in %s" % (opt, enable, flag, flags))
else:
@@ -676,15 +682,16 @@ def test_misc_flags_unique(self):
def test_override_optarch(self):
"""Test whether overriding the optarch flag works."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']
- for optarch_var in ['march=lovelylovelysandybridge', None]:
+ for optarch_var in ['-march=lovelylovelysandybridge', None]:
init_config(build_options={'optarch': optarch_var, 'silent': True})
for enable in [True, False]:
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'optarch': enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
flag = None
if optarch_var is not None:
- flag = '-%s' % optarch_var
+ flag = optarch_var
else:
# default optarch flag
flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)]
@@ -730,7 +737,8 @@ def test_optarch_generic(self):
tcopts.update(custom_tcopts)
tc.set_options(tcopts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
val = tc.get_variable(var)
tup = (key, tcversion, tcopts, generic_flags, val)
@@ -751,32 +759,35 @@ def test_optarch_aarch64_heuristic(self):
st.get_cpu_vendor = lambda: st.ARM
tc = self.get_toolchain("GCC", version="4.6.4")
tc.set_options({})
- tc.prepare()
- self.assertEqual(tc.options.options_map['optarch'], 'mcpu=cortex-a53')
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ self.assertEqual(tc.options.options_map['optarch'], '-mcpu=cortex-a53')
self.assertIn('-mcpu=cortex-a53', os.environ['CFLAGS'])
self.modtool.purge()
tc = self.get_toolchain("GCCcore", version="6.2.0")
tc.set_options({})
- tc.prepare()
- self.assertEqual(tc.options.options_map['optarch'], 'mcpu=native')
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ self.assertEqual(tc.options.options_map['optarch'], '-mcpu=native')
self.assertIn('-mcpu=native', os.environ['CFLAGS'])
self.modtool.purge()
st.get_cpu_model = lambda: 'ARM Cortex-A53 + Cortex-A72'
tc = self.get_toolchain("GCC", version="4.6.4")
tc.set_options({})
- tc.prepare()
- self.assertEqual(tc.options.options_map['optarch'], 'mcpu=cortex-a72.cortex-a53')
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ self.assertEqual(tc.options.options_map['optarch'], '-mcpu=cortex-a72.cortex-a53')
self.assertIn('-mcpu=cortex-a72.cortex-a53', os.environ['CFLAGS'])
self.modtool.purge()
def test_compiler_dependent_optarch(self):
"""Test whether specifying optarch on a per compiler basis works."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']
- intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')]
- gcc_options = [('gccflag', 'gccflag'), ('march=nocona', 'march=nocona'), ('', '')]
- gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')]
+ intel_options = [('intelflag', '-intelflag'), ('GENERIC', '-xSSE2'), ('', '')]
+ gcc_options = [('gccflag', '-gccflag'), ('-march=nocona', '-march=nocona'), ('', '')]
+ gcccore_options = [('gcccoreflag', '-gcccoreflag'), ('GENERIC', '-march=x86-64 -mtune=generic'), ('', '')]
tc_intel = ('iccifort', '2018.1.163')
tc_gcc = ('GCC', '6.4.0-2.28')
@@ -784,6 +795,7 @@ def test_compiler_dependent_optarch(self):
tc_pgi = ('PGI', '16.7-GCC-5.4.0-2.26')
enabled = [True, False]
+ self.allow_deprecated_behaviour() # when testing optarch flags without initial "-" (remove in EB 6.0)
test_cases = []
for i, (tc, options) in enumerate(zip((tc_intel, tc_gcc, tc_gcccore),
(intel_options, gcc_options, gcccore_options))):
@@ -811,7 +823,8 @@ def test_compiler_dependent_optarch(self):
init_config(build_options={'optarch': optarch_var, 'silent': True})
tc = self.get_toolchain(toolchain_name, version=toolchain_ver)
tc.set_options({'optarch': enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
flags = None
if toolchain_name == 'iccifort':
flags = intel_flags_exp
@@ -837,8 +850,8 @@ def test_compiler_dependent_optarch(self):
gcc_options[1][1],
gcccore_options[0][1],
gcccore_options[1][1],
- 'xHost', # default optimal for Intel
- 'march=native', # default optimal for GCC
+ '-xHost', # default optimal for Intel
+ '-march=native', # default optimal for GCC
]
else:
blacklist = [flags]
@@ -868,14 +881,16 @@ def test_easyconfig_optarch_flags(self):
toy_txt = read_file(eb_file)
# check that an optarch map raises an error
- write_file(test_ec, toy_txt + "\ntoolchainopts = {'optarch': 'GCC:march=sandrybridge;Intel:xAVX'}")
+ write_file(test_ec, toy_txt + "\ntoolchainopts = {'optarch': 'GCC:-march=sandrybridge;Intel:-xAVX'}")
msg = "syntax is not allowed"
- self.assertErrorRegex(EasyBuildError, msg, self.eb_main, [test_ec], raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, msg, self.eb_main, [test_ec], raise_error=True, do_build=True)
# check that setting optarch flags work
- write_file(test_ec, toy_txt + "\ntoolchainopts = {'optarch': 'march=sandybridge'}")
- out = self.eb_main([test_ec], raise_error=True, do_build=True)
- regex = re.compile("_set_optimal_architecture: using march=sandybridge as optarch for x86_64")
+ write_file(test_ec, toy_txt + "\ntoolchainopts = {'optarch': '-march=sandybridge'}")
+ with self.mocked_stdout_stderr():
+ out = self.eb_main([test_ec], raise_error=True, do_build=True)
+ regex = re.compile("_set_optimal_architecture: using -march=sandybridge as optarch for x86_64")
self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_misc_flags_unique_fortran(self):
@@ -888,12 +903,13 @@ def test_misc_flags_unique_fortran(self):
for enable in [True, False]:
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({opt: enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
flag = tc.COMPILER_UNIQUE_OPTION_MAP[opt]
if isinstance(flag, list):
- flag = ' '.join('-%s' % x for x in flag)
+ flag = ' '.join(flag)
else:
- flag = '-%s' % flag
+ flag = flag
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
@@ -910,7 +926,8 @@ def test_precision_flags(self):
# check default precision: -fno-math-errno flag for GCC
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
flags_regex = re.compile(r"-O2 -ftree-vectorize -m(arch|cpu)=native -fno-math-errno")
for var in flag_vars:
val = os.getenv(var)
@@ -920,13 +937,14 @@ def test_precision_flags(self):
precs = ['strict', 'precise', 'loose', 'veryloose']
prec_flags = {}
for prec in precs:
- prec_flags[prec] = ' '.join('-%s' % x for x in Gcc.COMPILER_UNIQUE_OPTION_MAP[prec])
+ prec_flags[prec] = ' '.join(Gcc.COMPILER_UNIQUE_OPTION_MAP[prec])
for prec in prec_flags:
for enable in [True, False]:
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({prec: enable})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for var in flag_vars:
if enable:
regex = re.compile(r"-O2 -ftree-vectorize -m(arch|cpu)=native %s" % prec_flags[prec])
@@ -937,10 +955,96 @@ def test_precision_flags(self):
self.modtool.purge()
+ def test_search_path_cpp_headers(self):
+ """Test functionality behind search-path-cpp-headers option"""
+ cpp_headers_mode = {
+ "flags": ["CPPFLAGS"],
+ "cpath": ["CPATH"],
+ "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+ }
+ # test without toolchain option
+ for build_opt in cpp_headers_mode:
+ init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
+ tc = self.get_toolchain("foss", version="2018a")
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ for env_var in cpp_headers_mode[build_opt]:
+ assert_fail_msg = (
+ f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' "
+ "not found in toolchain environment"
+ )
+ self.assertIn(env_var, tc.variables, assert_fail_msg)
+ self.modtool.purge()
+ # test with toolchain option
+ for build_opt in cpp_headers_mode:
+ init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
+ for tc_opt in cpp_headers_mode:
+ tc = self.get_toolchain("foss", version="2018a")
+ tc.set_options({"search-path-cpp-headers": tc_opt})
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ for env_var in cpp_headers_mode[tc_opt]:
+ assert_fail_msg = (
+ f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' "
+ "not found in toolchain environment"
+ )
+ self.assertIn(env_var, tc.variables, assert_fail_msg)
+ self.modtool.purge()
+ # test wrong toolchain option
+ tc = self.get_toolchain("foss", version="2018a")
+ tc.set_options({"search-path-cpp-headers": "WRONG_MODE"})
+ with self.mocked_stdout_stderr():
+ error_pattern = "Unknown value selected for toolchain option search-path-cpp-headers"
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ self.modtool.purge()
+
+ def test_search_path_linker(self):
+ """Test functionality behind search-path-linker option"""
+ linker_mode = {
+ "flags": ["LDFLAGS"],
+ "library_path": ["LIBRARY_PATH"],
+ }
+ # test without toolchain option
+ for build_opt in linker_mode:
+ init_config(build_options={"search_path_linker": build_opt, "silent": True})
+ tc = self.get_toolchain("foss", version="2018a")
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ for env_var in linker_mode[build_opt]:
+ assert_fail_msg = (
+ f"Variable {env_var} required by search-path-linker build option '{build_opt}' "
+ "not found in toolchain environment"
+ )
+ self.assertIn(env_var, tc.variables, assert_fail_msg)
+ self.modtool.purge()
+ # test with toolchain option
+ for build_opt in linker_mode:
+ init_config(build_options={"search_path_linker": build_opt, "silent": True})
+ for tc_opt in linker_mode:
+ tc = self.get_toolchain("foss", version="2018a")
+ tc.set_options({"search-path-linker": tc_opt})
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ for env_var in linker_mode[tc_opt]:
+ assert_fail_msg = (
+ f"Variable {env_var} required by search-path-linker toolchain option '{tc_opt}' "
+ "not found in toolchain environment"
+ )
+ self.assertIn(env_var, tc.variables, assert_fail_msg)
+ self.modtool.purge()
+ # test wrong toolchain option
+ tc = self.get_toolchain("foss", version="2018a")
+ tc.set_options({"search-path-linker": "WRONG_MODE"})
+ with self.mocked_stdout_stderr():
+ error_pattern = "Unknown value selected for toolchain option search-path-linker"
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ self.modtool.purge()
+
def test_cgoolf_toolchain(self):
"""Test for cgoolf toolchain."""
tc = self.get_toolchain("cgoolf", version="1.1.6")
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'clang')
self.assertEqual(tc.get_variable('CXX'), 'clang++')
@@ -951,20 +1055,23 @@ def test_cgoolf_toolchain(self):
def test_comp_family(self):
"""Test determining compiler family."""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.comp_family(), "GCC")
def test_mpi_family(self):
"""Test determining MPI family."""
# check subtoolchain w/o MPI
tc = self.get_toolchain("GCC", version="6.4.0-2.28")
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_family(), None)
self.modtool.purge()
# check full toolchain including MPI
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_family(), "OpenMPI")
self.modtool.purge()
@@ -972,28 +1079,32 @@ def test_mpi_family(self):
self.setup_sandbox_for_intel_fftw(self.test_prefix)
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain("intel", version="2018a")
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_family(), "IntelMPI")
def test_blas_lapack_family(self):
"""Test determining BLAS/LAPACK family."""
# check compiler-only (sub)toolchain
tc = self.get_toolchain("GCC", version="6.4.0-2.28")
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.blas_family(), None)
self.assertEqual(tc.lapack_family(), None)
self.modtool.purge()
# check compiler/MPI-only (sub)toolchain
tc = self.get_toolchain('gompi', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.blas_family(), None)
self.assertEqual(tc.lapack_family(), None)
self.modtool.purge()
# check full toolchain including BLAS/LAPACK
tc = self.get_toolchain('fosscuda', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.blas_family(), 'OpenBLAS')
self.assertEqual(tc.lapack_family(), 'OpenBLAS')
self.modtool.purge()
@@ -1002,14 +1113,16 @@ def test_blas_lapack_family(self):
self.setup_sandbox_for_intel_fftw(self.test_prefix)
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.blas_family(), 'IntelMKL')
self.assertEqual(tc.lapack_family(), 'IntelMKL')
def test_fft_env_vars_foss(self):
"""Test setting of $FFT* environment variables using foss toolchain."""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1024,7 +1137,8 @@ def test_fft_env_vars_foss(self):
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'openmp': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
@@ -1037,7 +1151,8 @@ def test_fft_env_vars_foss(self):
tc = self.get_toolchain('foss', version='2018a')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3_mpi.a,libfftw3.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1054,7 +1169,8 @@ def test_fft_env_vars_foss(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('foss', version='2018a-FFTW.MPI')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1072,7 +1188,8 @@ def test_fft_env_vars_foss(self):
tc = self.get_toolchain('foss', version='2018a-FFTW.MPI')
tc.set_options({'openmp': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
@@ -1088,7 +1205,8 @@ def test_fft_env_vars_foss(self):
tc = self.get_toolchain('foss', version='2018a-FFTW.MPI')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3_mpi.a,libfftw3.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1111,7 +1229,8 @@ def test_fft_env_vars_intel(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3xc_intel.a,libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1132,7 +1251,8 @@ def test_fft_env_vars_intel(self):
tc = self.get_toolchain('intel', version='2018a')
tc.set_options({'openmp': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
@@ -1145,7 +1265,8 @@ def test_fft_env_vars_intel(self):
tc = self.get_toolchain('intel', version='2018a')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3xc_intel.a,libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,'
fft_static_libs += 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
@@ -1169,7 +1290,8 @@ def test_fft_env_vars_intel(self):
self.modtool.purge()
self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0')
tc = self.get_toolchain('intel', version='2021b')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
@@ -1190,7 +1312,8 @@ def test_fft_env_vars_intel(self):
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'openmp': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
@@ -1206,7 +1329,8 @@ def test_fft_env_vars_intel(self):
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'usempi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
fft_static_libs = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,'
fft_static_libs += 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
@@ -1235,10 +1359,11 @@ def test_fosscuda(self):
tc = self.get_toolchain("fosscuda", version="2018a")
opts = {'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10'], 'openmp': True}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
archflags = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)]
- optflags = "-O2 -ftree-vectorize -%s -fno-math-errno -fopenmp" % archflags
+ optflags = "-O2 -ftree-vectorize %s -fno-math-errno -fopenmp" % archflags
nvcc_flags = r' '.join([
r'-Xcompiler="%s"' % optflags,
# the use of -lcudart in -Xlinker is a bit silly but hard to avoid
@@ -1344,7 +1469,8 @@ def test_intel_toolchain(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain("intel", version="2018a")
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'icc')
self.assertEqual(tc.get_variable('CXX'), 'icpc')
@@ -1356,7 +1482,8 @@ def test_intel_toolchain(self):
tc = self.get_toolchain("intel", version="2018a")
opts = {'usempi': True}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'mpiicc')
self.assertEqual(tc.get_variable('CXX'), 'mpiicpc')
@@ -1373,7 +1500,8 @@ def test_intel_toolchain(self):
tc = self.get_toolchain("intel", version="2018a")
opts = {'usempi': True, 'openmp': True}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
for flag in ['-mt_mpi', '-fopenmp']:
for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']:
@@ -1402,7 +1530,8 @@ def test_intel_toolchain(self):
tc = self.get_toolchain('intel', version='2012a')
opts = {'openmp': True}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('MPIFC'), 'mpiifort')
for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']:
self.assertIn('-openmp', tc.get_variable(var))
@@ -1417,7 +1546,8 @@ def test_intel_toolchain(self):
tc = self.get_toolchain('intel-compilers', version='2021.4.0')
tc.set_options({})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icc')
self.assertEqual(os.getenv('CXX'), 'icpc')
@@ -1436,7 +1566,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# default remains classic compilers for now
self.assertEqual(os.getenv('CC'), 'icc')
@@ -1454,7 +1585,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icx')
self.assertEqual(os.getenv('CXX'), 'icpx')
@@ -1470,7 +1602,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel-compilers', version='2022.2.0')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# by default (for version >= 2022.2.0): oneAPI C/C++ compiler + classic Fortran compiler
self.assertEqual(os.getenv('CC'), 'icx')
@@ -1482,7 +1615,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel-compilers', version='2022.2.0')
tc.set_options({'oneapi_fortran': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icx')
self.assertEqual(os.getenv('CXX'), 'icpx')
self.assertEqual(os.getenv('F77'), 'ifx')
@@ -1492,7 +1626,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel-compilers', version='2022.2.0')
tc.set_options({'oneapi_c_cxx': False, 'oneapi_fortran': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icc')
self.assertEqual(os.getenv('CXX'), 'icpc')
self.assertEqual(os.getenv('F77'), 'ifx')
@@ -1533,7 +1668,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi_c_cxx': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icx')
self.assertEqual(os.getenv('CXX'), 'icpx')
self.assertEqual(os.getenv('F77'), 'ifort')
@@ -1543,7 +1679,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi_fortran': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icc')
self.assertEqual(os.getenv('CXX'), 'icpc')
self.assertEqual(os.getenv('F77'), 'ifx')
@@ -1553,7 +1690,8 @@ def test_intel_toolchain_oneapi(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi_c_cxx': True, 'oneapi_fortran': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.getenv('CC'), 'icx')
self.assertEqual(os.getenv('CXX'), 'icpx')
self.assertEqual(os.getenv('F77'), 'ifx')
@@ -1563,25 +1701,29 @@ def test_intel_toolchain_oneapi(self):
def test_toolchain_verification(self):
"""Test verification of toolchain definition."""
tc = self.get_toolchain('foss', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.modtool.purge()
# toolchain modules missing a toolchain element should fail verification
error_msg = "List of toolchain dependency modules and toolchain definition do not match"
tc = self.get_toolchain('foss', version='2018a-brokenFFTW')
- self.assertErrorRegex(EasyBuildError, error_msg, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, tc.prepare)
self.modtool.purge()
# missing optional toolchain elements are fine
tc = self.get_toolchain('fosscuda', version='2018a')
opts = {'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10']}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
def test_nosuchtoolchain(self):
"""Test preparing for a toolchain for which no module is available."""
tc = self.get_toolchain('intel', version='1970.01')
- self.assertErrorRegex(EasyBuildError, "No module found for toolchain", tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "No module found for toolchain", tc.prepare)
def test_mpi_cmd_prefix(self):
"""Test mpi_exec_nranks function."""
@@ -1598,7 +1740,8 @@ def test_mpi_cmd_prefix(self):
self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
tc = self.get_toolchain('gompi', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2")
self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1")
@@ -1606,7 +1749,8 @@ def test_mpi_cmd_prefix(self):
self.setup_sandbox_for_intel_fftw(self.test_prefix)
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2")
self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1")
@@ -1614,7 +1758,8 @@ def test_mpi_cmd_prefix(self):
self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038')
tc = self.get_toolchain('intel', version='2012a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4")
self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix(nr_ranks=4)))
@@ -1650,19 +1795,22 @@ def test_mpi_cmd_for(self):
self.assertEqual(tc.mpi_cmd_for('test123', 2), "mpirun -n 2 test123")
tc = self.get_toolchain('gompi', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_cmd_for('test123', 2), "mpirun -n 2 test123")
self.modtool.purge()
self.setup_sandbox_for_intel_fftw(self.test_prefix)
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.mpi_cmd_for('test123', 2), "mpirun -n 2 test123")
self.modtool.purge()
self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038')
tc = self.get_toolchain('intel', version='2012a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
mpi_cmd_for_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 test$")
self.assertTrue(mpi_cmd_for_re.match(tc.mpi_cmd_for('test', 4)))
@@ -1729,7 +1877,8 @@ def test_prepare_deps(self):
'build_only': False,
},
]
- tc.prepare(deps=deps)
+ with self.mocked_stdout_stderr():
+ tc.prepare(deps=deps)
mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28']
self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods))
@@ -1757,7 +1906,8 @@ def test_prepare_deps_external(self):
}
]
tc = self.get_toolchain('GCC', version='6.4.0-2.28')
- tc.prepare(deps=deps)
+ with self.mocked_stdout_stderr():
+ tc.prepare(deps=deps)
mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'toy/0.0']
self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods))
self.assertTrue(os.environ['EBROOTTOY'].endswith('software/toy/0.0'))
@@ -1778,7 +1928,8 @@ def test_prepare_deps_external(self):
}
tc = self.get_toolchain('GCC', version='6.4.0-2.28')
os.environ['FOOBAR_PREFIX'] = '/foo/bar'
- tc.prepare(deps=deps)
+ with self.mocked_stdout_stderr():
+ tc.prepare(deps=deps)
mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'toy/0.0']
self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods))
self.assertEqual(os.environ['EBROOTTOY'], '/foo/bar')
@@ -1872,7 +2023,8 @@ def test_old_new_iccifort(self):
scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext)
tc = self.get_toolchain('fosscuda', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda)
self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda)
self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda)
@@ -1964,7 +2116,8 @@ def test_old_new_iccifort(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('BLAS_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4)
self.assertEqual(os.environ.get('BLAS_STATIC_LIBS', "(not set)"), blas_static_libs_intel4)
self.assertEqual(os.environ.get('LAPACK_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4)
@@ -1977,19 +2130,22 @@ def test_old_new_iccifort(self):
self.modtool.purge()
tc = self.get_toolchain('intel', version='2012a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel3)
self.assertIn(libscalack_intel3, os.environ['LIBSCALAPACK'])
self.modtool.purge()
tc = self.get_toolchain('intel', version='2018a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4)
self.assertIn(libscalack_intel4, os.environ['LIBSCALAPACK'])
self.modtool.purge()
tc = self.get_toolchain('intel', version='2012a')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel3)
self.assertIn(libscalack_intel3, os.environ['LIBSCALAPACK'])
self.modtool.purge()
@@ -1998,13 +2154,15 @@ def test_old_new_iccifort(self):
tc = self.get_toolchain('intel', version='2018a')
opts = {'i8': True}
tc.set_options(opts)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertIn(libscalack_intel4, os.environ['LIBSCALAPACK'])
self.modtool.purge()
tc = self.get_toolchain('fosscuda', version='2018a')
tc.set_options({'openmp': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda)
self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda)
self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda)
@@ -2046,7 +2204,8 @@ def test_standalone_iccifort(self):
"""Test whether standalone installation of iccifort matches the iccifort toolchain definition."""
tc = self.get_toolchain('iccifort', version='2018.1.163')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.toolchain_dep_mods, ['icc/2018.1.163', 'ifort/2018.1.163'])
self.modtool.purge()
@@ -2061,7 +2220,8 @@ def test_standalone_iccifort(self):
# toolchain verification fails because icc/ifort are not dependencies of iccifort modules,
# and corresponding environment variables are not set
error_pattern = "List of toolchain dependency modules and toolchain definition do not match"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
self.modtool.purge()
# make iccifort module set $EBROOT* and $EBVERSION* to pass toolchain verification
@@ -2074,7 +2234,8 @@ def test_standalone_iccifort(self):
])
write_file(fake_iccifort, fake_iccifort_txt)
# toolchain preparation (which includes verification) works fine now
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# no dependencies found in iccifort module
self.assertEqual(tc.toolchain_dep_mods, [])
@@ -2082,7 +2243,8 @@ def test_standalone_iccifortcuda(self):
"""Test whether standalone installation of iccifortcuda matches the iccifortcuda toolchain definition."""
tc = self.get_toolchain('iccifortcuda', version='2018b')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.toolchain_dep_mods, ['icc/2018.1.163', 'ifort/2018.1.163', 'CUDA/9.1.85'])
self.modtool.purge()
@@ -2097,13 +2259,15 @@ def test_standalone_iccifortcuda(self):
# toolchain verification fails because icc/ifort are not dependencies of iccifortcuda modules,
# and corresponding environment variables are not set
error_pattern = "List of toolchain dependency modules and toolchain definition do not match"
- self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
self.modtool.purge()
# Verify that it works loading a module that contains a combined iccifort module
tc = self.get_toolchain('iccifortcuda', version='2019a')
# toolchain preparation (which includes verification) works fine now
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# dependencies found in iccifortcuda module
self.assertEqual(tc.toolchain_dep_mods, ['iccifort/2019.5.281', 'CUDA/9.1.85'])
@@ -2111,42 +2275,55 @@ def test_independence(self):
"""Test independency of toolchain instances."""
# tweaking --optarch is required for Cray toolchains (craypre- module must be available)
- init_config(build_options={'optarch': 'test', 'silent': True})
-
- tc_cflags = {
- 'CrayCCE': "-O2 -homp -craype-verbose",
- 'CrayGNU': "-O2 -fno-math-errno -fopenmp -craype-verbose",
- 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose",
- 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp",
- 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp",
- 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -qopenmp",
- }
-
- toolchains = [
- ('CrayCCE', '2015.06-XC'),
- ('CrayGNU', '2015.06-XC'),
- ('CrayIntel', '2015.06-XC'),
- ('GCC', '6.4.0-2.28'),
- ('iccifort', '2018.1.163'),
- ('intel-compilers', '2022.1.0'),
- ]
+ custom_optarchs = ['test', '-test'] # specifying without initial "-" is deprecated but should still work
+ for custom_optarch in custom_optarchs:
+ init_config(build_options={'optarch': custom_optarch, 'silent': True})
+
+ tc_cflags = {
+ 'CrayCCE': "-O2 -homp -craype-verbose",
+ 'CrayGNU': "-O2 -fno-math-errno -fopenmp -craype-verbose",
+ 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose",
+ 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp",
+ 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp",
+ 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -qopenmp",
+ }
- # purposely obtain toolchains several times in a row, value for $CFLAGS should not change
- for _ in range(3):
- for tcname, tcversion in toolchains:
- tc = get_toolchain({'name': tcname, 'version': tcversion}, {},
- mns=ActiveMNS(), modtool=self.modtool)
- # also check whether correct compiler flag for OpenMP is used while we're at it
- # and options for oneAPI compiler for Intel
- if tcname == 'intel-compilers':
- tc.set_options({'oneapi': True, 'openmp': True})
- else:
- tc.set_options({'openmp': True})
- tc.prepare()
- expected_cflags = tc_cflags[tcname]
- msg = "Expected $CFLAGS found for toolchain %s: %s" % (tcname, expected_cflags)
- self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags, msg)
- self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
+ toolchains = [
+ ('GCC', '6.4.0-2.28'),
+ ('iccifort', '2018.1.163'),
+ ('intel-compilers', '2022.1.0'),
+ ]
+ if custom_optarch == 'test': # optarch is not used as a flag for Cray
+ toolchains += [
+ ('CrayCCE', '2015.06-XC'),
+ ('CrayGNU', '2015.06-XC'),
+ ('CrayIntel', '2015.06-XC'),
+ ]
+ self.allow_deprecated_behaviour() # test will be automatically converted to -test (remove in 6.0)
+ else:
+ self.disallow_deprecated_behaviour()
+
+ # purposely obtain toolchains several times in a row, value for $CFLAGS should not change
+ for _ in range(3):
+ for tcname, tcversion in toolchains:
+ # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+ # makes environment inconsistent which is not allowed by Environment Modules tool
+ if isinstance(self.modtool, EnvironmentModules):
+ self.modtool.purge()
+ tc = get_toolchain({'name': tcname, 'version': tcversion}, {},
+ mns=ActiveMNS(), modtool=self.modtool)
+ # also check whether correct compiler flag for OpenMP is used while we're at it
+ # and options for oneAPI compiler for Intel
+ if tcname == 'intel-compilers':
+ tc.set_options({'oneapi': True, 'openmp': True})
+ else:
+ tc.set_options({'openmp': True})
+ with self.mocked_stdout_stderr():
+ tc.prepare()
+ expected_cflags = tc_cflags[tcname]
+ msg = "Expected $CFLAGS found for toolchain %s: %s" % (tcname, expected_cflags)
+ self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags, msg)
+ self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def test_pgi_toolchain(self):
"""Tests for PGI toolchain."""
@@ -2158,7 +2335,8 @@ def test_pgi_toolchain(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('PGI', version='14.9')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'pgcc')
self.assertEqual(tc.get_variable('CXX'), 'pgCC')
@@ -2169,7 +2347,8 @@ def test_pgi_toolchain(self):
for pgi_ver in ['14.10', '16.3', '19.1']:
tc = self.get_toolchain('PGI', version=pgi_ver)
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'pgcc')
self.assertEqual(tc.get_variable('CXX'), 'pgc++')
@@ -2205,7 +2384,8 @@ def test_pgi_imkl(self):
self.modtool.prepend_module_path(self.test_prefix)
tc = self.get_toolchain('pomkl', version='2016.03')
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
liblapack = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core "
liblapack += "-Wl,--end-group -Wl,-Bdynamic -ldl"
@@ -2222,12 +2402,14 @@ def test_compiler_cache(self):
"--force",
"--debug",
"--disable-cleanup-tmpdir",
+ "--disable-rpath",
]
ccache = which('ccache')
if ccache is None:
msg = r"ccache binary not found in \$PATH, required by --use-ccache"
- self.assertErrorRegex(EasyBuildError, msg, self.eb_main, args, raise_error=True, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, msg, self.eb_main, args, raise_error=True, do_build=True)
# generate shell script to mock ccache/f90cache
for cache_tool in ['ccache', 'f90cache']:
@@ -2256,7 +2438,8 @@ def test_compiler_cache(self):
ccache_dir = os.path.join(self.test_prefix, 'ccache')
mkdir(ccache_dir, parents=True)
- out = self.eb_main(args, raise_error=True, do_build=True, reset_env=False)
+ with self.mocked_stdout_stderr():
+ out = self.eb_main(args, raise_error=True, do_build=True, reset_env=False)
patterns = [
"This is a ccache wrapper",
@@ -2287,7 +2470,8 @@ def test_compiler_cache(self):
mkdir(f90cache_dir, parents=True)
args.append("--use-f90cache=%s" % f90cache_dir)
- out = self.eb_main(args, raise_error=True, do_build=True, reset_env=False)
+ with self.mocked_stdout_stderr():
+ out = self.eb_main(args, raise_error=True, do_build=True, reset_env=False)
for pattern in patterns:
regex = re.compile(pattern)
self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out))
@@ -2315,8 +2499,9 @@ def test_rpath_args_script(self):
])
# simplest possible compiler command
- out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -c foo.c")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2327,11 +2512,12 @@ def test_rpath_args_script(self):
"'-c'",
"'foo.c'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# linker command, --enable-new-dtags should be replaced with --disable-new-dtags
- out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} ld '' '{rpath_inc}' --enable-new-dtags foo.o")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-rpath=%s/lib'" % self.test_prefix,
"'-rpath=%s/lib64'" % self.test_prefix,
@@ -2342,11 +2528,12 @@ def test_rpath_args_script(self):
"'--disable-new-dtags'",
"'foo.o'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# compiler command, -Wl,--enable-new-dtags should be replaced with -Wl,--disable-new-dtags
- out, ec = run_cmd("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -Wl,--enable-new-dtags foo.c")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2357,11 +2544,12 @@ def test_rpath_args_script(self):
"'-Wl,--disable-new-dtags'",
"'foo.c'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test passing no arguments
- out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}'")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2370,11 +2558,12 @@ def test_rpath_args_script(self):
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test passing a single empty argument
- out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} ld.gold '' '{rpath_inc}' ''")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-rpath=%s/lib'" % self.test_prefix,
"'-rpath=%s/lib64'" % self.test_prefix,
@@ -2384,12 +2573,13 @@ def test_rpath_args_script(self):
"'--disable-new-dtags'",
"''",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# single -L argument, but non-existing path => not used in RPATH, but -L option is retained
- cmd = "%s gcc '' '%s' foo.c -L%s/foo -lfoo" % (script, rpath_inc, self.test_prefix)
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L{self.test_prefix}/foo -lfoo"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2401,12 +2591,13 @@ def test_rpath_args_script(self):
"'-L%s/foo'" % self.test_prefix,
"'-lfoo'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# single -L argument again, with existing path
mkdir(os.path.join(self.test_prefix, 'foo'))
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2419,11 +2610,12 @@ def test_rpath_args_script(self):
"'-L%s/foo'" % self.test_prefix,
"'-lfoo'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# relative paths passed to -L are *not* RPATH'ed in
- out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' foo.c -L../lib -lfoo")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2435,12 +2627,13 @@ def test_rpath_args_script(self):
"'-L../lib'",
"'-lfoo'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# single -L argument, with value separated by a space
- cmd = "%s gcc '' '%s' foo.c -L %s/foo -lfoo" % (script, rpath_inc, self.test_prefix)
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L {self.test_prefix}/foo -lfoo"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2453,7 +2646,7 @@ def test_rpath_args_script(self):
"'-L%s/foo'" % self.test_prefix,
"'-lfoo'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
mkdir(os.path.join(self.test_prefix, 'bar'))
mkdir(os.path.join(self.test_prefix, 'lib64'))
@@ -2474,8 +2667,9 @@ def test_rpath_args_script(self):
'-L/usr/lib',
'-L%s/bar' % self.test_prefix,
])
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-rpath=%s/lib'" % self.test_prefix,
"'-rpath=%s/lib64'" % self.test_prefix,
@@ -2496,7 +2690,7 @@ def test_rpath_args_script(self):
"'-L/usr/lib'",
"'-L%s/bar'" % self.test_prefix,
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test specifying of custom rpath filter
cmd = ' '.join([
@@ -2511,8 +2705,9 @@ def test_rpath_args_script(self):
'-L/bar',
'-lbar',
])
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-rpath=%s/lib'" % self.test_prefix,
"'-rpath=%s/lib64'" % self.test_prefix,
@@ -2528,7 +2723,7 @@ def test_rpath_args_script(self):
"'-L/bar'",
"'-lbar'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# slightly trimmed down real-life example (compilation of XZ)
for subdir in ['icc/lib/intel64', 'imkl/lib', 'imkl/mkl/lib/intel64', 'gettext/lib']:
@@ -2550,8 +2745,9 @@ def test_rpath_args_script(self):
'-Wl,-rpath',
'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib',
])
- out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{script} icc '' '{rpath_inc}' {args}")
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
"'-Wl,-rpath=%s/lib64'" % self.test_prefix,
@@ -2578,7 +2774,7 @@ def test_rpath_args_script(self):
"'-Wl,-rpath'",
"'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# trimmed down real-life example involving quotes and escaped quotes (compilation of GCC)
args = [
@@ -2594,8 +2790,9 @@ def test_rpath_args_script(self):
'../../gcc/version.c',
]
cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args))
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = [
"'-Wl,-rpath=%s/lib'" % self.test_prefix,
@@ -2615,14 +2812,16 @@ def test_rpath_args_script(self):
"'-o' 'build/version.o'",
"'../../gcc/version.c'",
]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# verify that no -rpath arguments are injected when command is run in 'version check' mode
for extra_args in ["-v", "-V", "--version", "-dumpversion", "-v -L/test/lib"]:
cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, extra_args)
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(["'%s'" % x for x in extra_args.split(' ')]))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
+ cmd_args = ' '.join(["'%s'" % x for x in extra_args.split(' ')])
+ self.assertEqual(res.output.strip(), f"CMD_ARGS=({cmd_args})")
# if a compiler command includes "-x c++-header" or "-x c-header" (which imply no linking is done),
# we should *not* inject -Wl,-rpath options, since those enable linking as a side-effect;
@@ -2634,10 +2833,11 @@ def test_rpath_args_script(self):
]
for extra_args in test_cases:
cmd = "%s g++ '' '%s' foo.c -O2 %s" % (script, rpath_inc, extra_args)
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')]
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# check whether $LIBRARY_PATH is taken into account
test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc)
@@ -2702,15 +2902,17 @@ def test_rpath_args_script(self):
os.environ['LIBRARY_PATH'] = ':'.join(library_path)
- out, ec = run_cmd(test_cmd_gcc, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd_gcc)
+ self.assertEqual(res.exit_code, 0)
cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
- out, ec = run_cmd(test_cmd_ld, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd_ld)
+ self.assertEqual(res.exit_code, 0)
cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# paths already listed via -L don't get included again as RPATH option
new_lib64 = os.path.join(self.test_prefix, 'new', 'lib64')
@@ -2728,18 +2930,20 @@ def test_rpath_args_script(self):
]
os.environ['LIBRARY_PATH'] = ':'.join(library_path)
- out, ec = run_cmd(test_cmd_gcc, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd_gcc)
+ self.assertEqual(res.exit_code, 0)
# no -L options in GCC command, so all $LIBRARY_PATH entries are retained except for last one (lib symlink)
cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path[:-1] if x] + post_cmd_args_gcc
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
- out, ec = run_cmd(test_cmd_ld, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(test_cmd_ld)
+ self.assertEqual(res.exit_code, 0)
# only new path from $LIBRARY_PATH is included as -rpath option,
# since others are already included via corresponding -L flag
cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % new_lib64] + post_cmd_args_ld
- self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
+ self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
def test_toolchain_prepare_rpath(self):
"""Test toolchain.prepare under --rpath"""
@@ -2763,7 +2967,8 @@ def test_toolchain_prepare_rpath(self):
# setting 'rpath' toolchain option to false implies no RPATH wrappers being used
tc.set_options({'rpath': False})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
res = which('g++', retain_all=True)
self.assertTrue(len(res) >= 1)
self.assertFalse(tc.is_rpath_wrapper(res[0]))
@@ -2772,7 +2977,8 @@ def test_toolchain_prepare_rpath(self):
# enable 'rpath' toolchain option again (equivalent to the default setting)
tc.set_options({'rpath': True})
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
# check that wrapper is indeed in place
res = which('g++', retain_all=True)
@@ -2854,8 +3060,9 @@ def test_toolchain_prepare_rpath(self):
"'$FOO'",
'-DX="\\"\\""',
])
- out, ec = run_cmd(cmd)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
expected = ' '.join([
'-Wl,--disable-new-dtags',
'-Wl,-rpath=%s/foo' % self.test_prefix,
@@ -2865,7 +3072,7 @@ def test_toolchain_prepare_rpath(self):
'$FOO',
'-DX=""',
])
- self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
+ self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')})
# check whether 'stubs' library directory are correctly filtered out
paths = [
@@ -2890,8 +3097,9 @@ def test_toolchain_prepare_rpath(self):
args = ['-L%s' % x for x in paths]
cmd = "g++ ${USER}.c %s" % ' '.join(args)
- out, ec = run_cmd(cmd, simple=False)
- self.assertEqual(ec, 0)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+ self.assertEqual(res.exit_code, 0)
expected = ' '.join([
'-Wl,--disable-new-dtags',
@@ -2916,11 +3124,12 @@ def test_toolchain_prepare_rpath(self):
'-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix,
'-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix,
])
- self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
+ self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')})
# calling prepare() again should *not* result in wrapping the existing RPATH wrappers
# this can happen when building extensions
- tc.prepare()
+ with self.mocked_stdout_stderr():
+ tc.prepare()
res = which('g++', retain_all=True)
self.assertTrue(len(res) >= 2)
self.assertTrue(tc.is_rpath_wrapper(res[0]))
@@ -2960,16 +3169,13 @@ def prep():
# $TMPDIR is left untouched with OpenMPI 2.x if $TMPDIR is sufficiently short
os.environ['TMPDIR'] = orig_tmpdir
tc, stdout, stderr = prep()
- self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
self.assertEqual(os.environ.get('TMPDIR'), orig_tmpdir)
# warning is printed and $TMPDIR is set to shorter path if existing $TMPDIR is too long
os.environ['TMPDIR'] = long_tmpdir
tc, stdout, stderr = prep()
- self.assertEqual(stdout, '')
- # basename of tmpdir will be 6 chars in Python 2, 8 chars in Python 3
- regex = re.compile(r"^WARNING: Long \$TMPDIR .* problems with OpenMPI 2.x, using shorter path: /tmp/.{6,8}$")
+ regex = re.compile(r"^WARNING: Long \$TMPDIR .* problems with OpenMPI 2.x, using shorter path: /tmp/.{8}$")
self.assertTrue(regex.match(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
# new $TMPDIR should be /tmp/xxxxxx
@@ -2999,14 +3205,12 @@ def prep():
# $TMPDIR is left untouched with OpenMPI 1.6.4
tc, stdout, stderr = prep()
- self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
self.assertEqual(os.environ.get('TMPDIR'), orig_tmpdir)
# ... even with long $TMPDIR
os.environ['TMPDIR'] = long_tmpdir
tc, stdout, stderr = prep()
- self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
self.assertEqual(os.environ.get('TMPDIR'), long_tmpdir)
os.environ['TMPDIR'] = orig_tmpdir
@@ -3038,10 +3242,10 @@ def test_get_flag(self):
tc = self.get_toolchain('gompi', version='2018a')
checks = {
- '-a': 'a',
- '-openmp': 'openmp',
- '-foo': ['foo'],
- '-foo -bar': ['foo', 'bar'],
+ '-a': '-a',
+ '-openmp': '-openmp',
+ '-foo': ['-foo'],
+ '-foo -bar': ['-foo', '-bar'],
}
for flagstring, flags in checks.items():
diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py
index e61bb9031c..4ad56d904e 100644
--- a/test/framework/toolchainvariables.py
+++ b/test/framework/toolchainvariables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -62,16 +62,16 @@ class TCV(ToolchainVariables):
tcv.join('MPICC', 'CC')
self.assertEqual(str(tcv['MPICC']), "gcc")
- tcv['F90'] = ['gfortran', 'foo', 'bar']
- self.assertEqual(tcv['F90'].__repr__(), "[['gfortran', 'foo', 'bar']]")
+ tcv['F90'] = ['gfortran', '-foo', '-bar']
+ self.assertEqual(tcv['F90'].__repr__(), "[['gfortran', '-foo', '-bar']]")
self.assertEqual(str(tcv['F90']), "gfortran -foo -bar")
- tcv.nappend('FLAGS', ['one', 'two'])
- x = tcv.nappend('FLAGS', ['three', 'four'])
+ tcv.nappend('FLAGS', ['-one', '-two'])
+ x = tcv.nappend('FLAGS', ['-three', '-four'])
x.POSITION = -5 # sanitize will reorder, default POSITION is 0
- self.assertEqual(tcv['FLAGS'].__repr__(), "[['one', 'two'], ['three', 'four']]")
+ self.assertEqual(tcv['FLAGS'].__repr__(), "[['-one', '-two'], ['-three', '-four']]")
tcv['FLAGS'].sanitize() # sort on position, called by __str__ also
- self.assertEqual(tcv['FLAGS'].__repr__(), "[['three', 'four'], ['one', 'two']]")
+ self.assertEqual(tcv['FLAGS'].__repr__(), "[['-three', '-four'], ['-one', '-two']]")
self.assertEqual(str(tcv['FLAGS']), "-three -four -one -two")
# LIBBLAS is a LibraryList
@@ -159,7 +159,7 @@ class TCV(ToolchainVariables):
tcv.nappend('MPICH_CC', 'icc', var_class=CommandFlagList)
self.assertEqual(str(tcv['MPICH_CC']), "icc")
- tcv.nappend('MPICH_CC', 'test')
+ tcv.nappend('MPICH_CC', '-test')
self.assertEqual(str(tcv['MPICH_CC']), "icc -test")
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 1bd114d9d7..d46e755ebc 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
##
-# Copyright 2013-2024 Ghent University
+# Copyright 2013-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,8 +40,10 @@
import sys
import tempfile
import textwrap
+import filecmp
from easybuild.tools import LooseVersion
-from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
+from importlib import reload
+from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup
from test.framework.package import mock_fpm
from unittest import TextTestRunner
@@ -49,15 +51,15 @@
import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s)
from easybuild.framework.easyconfig.easyconfig import EasyConfig
from easybuild.framework.easyconfig.parser import EasyConfigParser
+from easybuild.main import main_with_hooks
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import get_module_syntax, get_repositorypath
from easybuild.tools.environment import modify_env
from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file
from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file
from easybuild.tools.module_generator import ModuleGeneratorTcl
-from easybuild.tools.modules import Lmod
-from easybuild.tools.py2vs3 import reload, string_type
-from easybuild.tools.run import run_cmd
+from easybuild.tools.modules import EnvironmentModules, Lmod
+from easybuild.tools.run import run_shell_cmd
from easybuild.tools.utilities import nub
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.version import VERSION as EASYBUILD_VERSION
@@ -156,9 +158,9 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi
devel_module_path = os.path.join(software_path, 'easybuild', '%s-%s-easybuild-devel' % (name, full_version))
self.assertExists(devel_module_path)
- def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True,
- raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True,
- raise_systemexit=False, force=True, test_report_regexs=None, debug=True):
+ def _test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True,
+ raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True,
+ raise_systemexit=False, force=True, test_report_regexs=None, debug=True):
"""Perform a toy build."""
if extra_args is None:
extra_args = []
@@ -169,9 +171,6 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
full_ver = '0.0%s' % versionsuffix
args = [
ec_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--unittest-file=%s' % self.logfile,
'--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]),
]
@@ -235,7 +234,7 @@ def run_test_toy_build_with_output(self, *args, **kwargs):
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(*args, **kwargs)
+ self._test_toy_build(*args, **kwargs)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
@@ -243,17 +242,29 @@ def run_test_toy_build_with_output(self, *args, **kwargs):
return stdout, stderr
+ def run_eb_main_capture_output(self, *args, **kwargs):
+ """Run eb_main with specified arguments, capture stdout, and return output from eb_main"""
+ self.mock_stdout(True)
+ outtxt = self.eb_main(*args, **kwargs)
+ self.mock_stdout(False)
+ return outtxt
+
+ def test_toy_build(self):
+ """Base level test build"""
+ with self.mocked_stdout_stderr():
+ self._test_toy_build()
+
def test_toy_broken(self):
"""Test deliberately broken toy build."""
tmpdir = tempfile.mkdtemp()
broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb")
toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
broken_toy_ec_txt = read_file(toy_ec_file)
- broken_toy_ec_txt += "checksums = ['clearywrongMD5checksumoflength32']"
+ broken_toy_ec_txt += "checksums = ['clearywrongSHA256checksumoflength64-0123456789012345678901234567']"
write_file(broken_toy_ec, broken_toy_ec_txt)
error_regex = "Checksum verification .* failed"
- self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build, ec_file=broken_toy_ec, tmpdir=tmpdir,
- verify=False, fails=True, verbose=False, raise_error=True)
+ self.assertErrorRegex(EasyBuildError, error_regex, self.run_test_toy_build_with_output, ec_file=broken_toy_ec,
+ tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True)
# make sure log file is retained, also for failed build
log_path_pattern = os.path.join(tmpdir, 'eb-*', 'easybuild-toy-0.0*.log')
@@ -265,12 +276,88 @@ def test_toy_broken(self):
# test dumping full test report (doesn't raise an exception)
test_report_fp = os.path.join(self.test_buildpath, 'full_test_report.md')
- self.test_toy_build(ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False,
- raise_error=True, test_report=test_report_fp)
+ self.run_test_toy_build_with_output(ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True,
+ verbose=False, raise_error=True, test_report=test_report_fp)
# cleanup
shutil.rmtree(tmpdir)
+ def test_toy_broken_copy_log_build_dir(self):
+ """
+ Test whether log files and the build directory are copied to a permanent location
+ after a failed installation.
+ """
+ toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ toy_ec_txt = read_file(toy_ec)
+
+ test_ec_txt = re.sub(
+ r'toy-0\.0_fix-silly-typo-in-printf-statement\.patch',
+ r'toy-0.0_add-bug.patch',
+ toy_ec_txt
+ )
+ test_ec = os.path.join(self.test_prefix, 'toy-0.0-buggy.eb')
+ write_file(test_ec, test_ec_txt)
+
+ # set up subdirectories where stuff should go
+ tmpdir = os.path.join(self.test_prefix, 'tmp')
+ tmp_log_dir = os.path.join(self.test_prefix, 'tmp-logs')
+ failed_install_build_dirs_path = os.path.join(self.test_prefix, 'failed-install-build-dirs')
+ failed_install_logs_path = os.path.join(self.test_prefix, 'failed-install-logs')
+
+ extra_args = [
+ f'--failed-install-build-dirs-path={failed_install_build_dirs_path}',
+ f'--failed-install-logs-path={failed_install_logs_path}',
+ f'--tmp-logdir={tmp_log_dir}',
+ ]
+ with self.mocked_stdout_stderr():
+ outtxt = self._test_toy_build(ec_file=test_ec, extra_args=extra_args, tmpdir=tmpdir,
+ verify=False, fails=True, verbose=False)
+
+ # find path to temporary log file
+ log_files = glob.glob(os.path.join(tmp_log_dir, '*.log'))
+ self.assertTrue(len(log_files) == 1, f"Expected exactly one log file, found {len(log_files)}: {log_files}")
+ log_file = log_files[0]
+
+ # check that log files were copied
+ saved_log_files = glob.glob(os.path.join(failed_install_logs_path, log_file))
+ self.assertTrue(len(saved_log_files) == 1, f"Unique copy of log file '{log_file}' made")
+ saved_log_file = saved_log_files[0]
+ self.assertTrue(filecmp.cmp(log_file, saved_log_file, shallow=False),
+ f"Log file '{log_file}' copied successfully")
+
+ # check that build directories were copied
+ build_dir = self.test_buildpath
+ topdir = failed_install_build_dirs_path
+
+ app_build_dir = os.path.join(build_dir, 'toy', '0.0', 'system-system')
+ # pattern: --
+ subdir_pattern = '????????-??????-?????'
+
+ # find path to toy.c
+ toy_c_files = glob.glob(os.path.join(app_build_dir, '**', 'toy.c'))
+ self.assertTrue(len(toy_c_files) == 1, f"Exactly one toy.c file found: {toy_c_files}")
+ toy_c_file = toy_c_files[0]
+
+ path = os.path.join(topdir, 'toy-0.0', subdir_pattern, 'toy-0.0', os.path.basename(toy_c_file))
+ res = glob.glob(path)
+ self.assertTrue(len(res) == 1, f"Exactly one hit found for {path}: {res}")
+ copied_toy_c_file = res[0]
+ self.assertTrue(filecmp.cmp(toy_c_file, copied_toy_c_file, shallow=False),
+ f"Copy of {toy_c_file} should be found under {topdir}")
+
+ # check whether compiler error messages are present in build log
+
+ # compiler error because of missing semicolon at end of line, could be:
+ # "error: expected ; before ..."
+ # "error: expected ';' after expression"
+ output_regexs = [r"^\s*toy\.c:5:44: error: expected (;|.;.)"]
+
+ log_txt = read_file(log_file)
+ for regex_pattern in output_regexs:
+ regex = re.compile(regex_pattern, re.M)
+ self.assertRegex(outtxt, regex)
+ self.assertRegex(log_txt, regex)
+
def test_toy_tweaked(self):
"""Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
@@ -284,8 +371,10 @@ def test_toy_tweaked(self):
# tweak easyconfig by appending to it
ec_extra = '\n'.join([
"versionsuffix = '-tweaked'",
- "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}",
- "modextrapaths_append = {'SOMEPATH_APPEND': ['qux/fred', 'thud', '']}",
+ "modextrapaths = {",
+ " 'SOMEPATH': ['foo/bar', 'baz', ''],",
+ " 'SOMEPATH_APPEND': {'paths': ['qux/fred', 'thud', ''], 'prepend': False},",
+ "}",
"modextravars = {'FOO': 'bar'}",
"modloadmsg = '%s'" % modloadmsg,
"modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua
@@ -297,18 +386,29 @@ def test_toy_tweaked(self):
"docurls = ['https://easybuilders.github.io/easybuild/toy/docs.html']",
"upstream_contacts = 'support@toy.org'",
"site_contacts = ['Jim Admin', 'Jane Admin']",
+ "postinstallcmds = [",
+ " 'mkdir \"%(installdir)s/foo\"',"
+ " 'touch \"%(installdir)s/foo/bar\"',"
+ " 'touch \"%(installdir)s/baz\"',"
+ " 'mkdir \"%(installdir)s/qux\"',"
+ " 'touch \"%(installdir)s/qux/fred\"',"
+ " 'touch \"%(installdir)s/thud\"',"
+ "]",
])
write_file(ec_file, ec_extra, append=True)
+ # populate test install dir
+ toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-tweaked')
+ mkdir(os.path.join(toy_installdir, 'foo'), parents=True)
+ write_file(os.path.join(toy_installdir, 'foo', 'bar'), "Test install file")
+ mkdir(os.path.join(toy_installdir, 'baz'), parents=True)
+
args = [
ec_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--force',
]
- outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
+ outtxt = self.run_eb_main_capture_output(args, do_build=True, verbose=True, raise_error=True)
self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked')
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked')
if get_module_syntax() == 'Lua':
@@ -369,8 +469,8 @@ def test_toy_buggy_easyblock(self):
'verify': False,
'verbose': False,
}
- err_regex = r"Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*name 'run_cmd' is not defined"
- self.assertErrorRegex(EasyBuildError, err_regex, self.test_toy_build, **kwargs)
+ err_regex = r"name 'run_shell_cmd' is not defined"
+ self.assertErrorRegex(NameError, err_regex, self.run_test_toy_build_with_output, **kwargs)
def test_toy_build_formatv2(self):
"""Perform a toy build (format v2)."""
@@ -380,9 +480,6 @@ def test_toy_build_formatv2(self):
args = [
os.path.join(os.path.dirname(__file__), 'easyconfigs', 'v2.0', 'toy.eb'),
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
@@ -391,7 +488,7 @@ def test_toy_build_formatv2(self):
'--toolchain=system,system',
'--experimental',
]
- outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
+ outtxt = self.run_eb_main_capture_output(args, logfile=self.dummylogfn, do_build=True, verbose=True)
self.check_toy(self.test_installpath, outtxt)
@@ -416,14 +513,12 @@ def test_toy_build_with_blocks(self):
args = [
'toy-0.0-multiple.eb',
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
]
- outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
for toy_prefix, toy_version, toy_suffix in [
('', '0.0', '-somesuffix'),
@@ -451,9 +546,6 @@ def test_toy_build_formatv2_sections(self):
for version, specs in versions.items():
args = [
os.path.join(os.path.dirname(__file__), 'easyconfigs', 'v2.0', 'toy-with-sections.eb'),
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
@@ -462,7 +554,8 @@ def test_toy_build_formatv2_sections(self):
'--toolchain=system,system',
'--experimental',
]
- outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ outtxt = self.run_eb_main_capture_output(args, logfile=self.dummylogfn, do_build=True, verbose=True,
+ raise_error=True)
specs['version'] = version
@@ -490,7 +583,7 @@ def test_toy_download_sources(self):
'--unittest-file=%s' % self.logfile,
'--force',
]
- outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
+ outtxt = self.run_eb_main_capture_output(args, logfile=self.dummylogfn, do_build=True, verbose=True)
self.check_toy(tmpdir, outtxt)
@@ -510,9 +603,6 @@ def test_toy_permissions(self):
write_file(test_ec, test_ec_txt)
args = [
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
@@ -523,7 +613,8 @@ def test_toy_permissions(self):
# test specifying a non-existing group
allargs = [test_ec] + args + ['--group=thisgroupdoesnotexist']
- outtxt, err = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, return_error=True)
+ outtxt, err = self.run_eb_main_capture_output(allargs, logfile=self.dummylogfn, do_build=True,
+ return_error=True)
err_regex = re.compile("Failed to get group ID .* group does not exist")
self.assertTrue(err_regex.search(outtxt), "Pattern '%s' found in '%s'" % (err_regex.pattern, outtxt))
@@ -556,7 +647,7 @@ def test_toy_permissions(self):
allargs.append("--umask=%s" % umask)
if cfg_group is not None:
allargs.append("--group=%s" % cfg_group)
- outtxt = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, verbose=True)
+ outtxt = self.run_eb_main_capture_output(allargs, logfile=self.dummylogfn, do_build=True, verbose=True)
# verify that installation was correct
self.check_toy(self.test_installpath, outtxt)
@@ -620,7 +711,7 @@ def test_toy_permissions_installdir(self):
write_file(test_ec, test_ec_txt)
# first check default behaviour
- self.test_toy_build(ec_file=test_ec)
+ self.run_test_toy_build_with_output(ec_file=test_ec)
toy_install_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
toy_bin = os.path.join(toy_install_dir, 'bin', 'toy')
@@ -649,9 +740,8 @@ def test_toy_permissions_installdir(self):
['--rebuild', '--fetch'],
)
for extra_args in test_cases:
- self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False)
- self.mock_stdout(False)
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args,
+ force=False)
installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777
self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir))
@@ -672,7 +762,7 @@ def test_toy_permissions_installdir(self):
shutil.rmtree(self.test_installpath)
# also check --group-writable-installdir
- self.test_toy_build(ec_file=test_ec, extra_args=['--group-writable-installdir'])
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=['--group-writable-installdir'])
installdir_perms = os.stat(toy_install_dir).st_mode & 0o777
self.assertEqual(installdir_perms, 0o775, "%s has group write permissions" % self.test_installpath)
@@ -683,7 +773,8 @@ def test_toy_permissions_installdir(self):
# this happens when for example using ModuleRC easyblock (because no devel module is created)
test_ec_txt += "\nmake_module = False"
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'], verify=False, raise_error=True)
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=['--read-only-installdir'], verify=False,
+ raise_error=True)
# restore original umask
os.umask(orig_umask)
@@ -699,7 +790,7 @@ def test_toy_gid_sticky_bits(self):
(('modules', 'all', 'toy'), False),
]
# no gid/sticky bits by default
- self.test_toy_build()
+ self.run_test_toy_build_with_output()
for subdir, _ in subdirs:
fullpath = os.path.join(self.test_installpath, *subdir)
perms = os.stat(fullpath).st_mode
@@ -707,7 +798,7 @@ def test_toy_gid_sticky_bits(self):
self.assertFalse(perms & stat.S_ISVTX, "no sticky bit on %s" % fullpath)
# git/sticky bits are set, but only on (re)created directories
- self.test_toy_build(extra_args=['--set-gid-bit', '--sticky-bit'])
+ self.run_test_toy_build_with_output(extra_args=['--set-gid-bit', '--sticky-bit'])
for subdir, bits_set in subdirs:
fullpath = os.path.join(self.test_installpath, *subdir)
perms = os.stat(fullpath).st_mode
@@ -720,7 +811,7 @@ def test_toy_gid_sticky_bits(self):
# start with a clean slate, now gid/sticky bits should be set on everything
shutil.rmtree(self.test_installpath)
- self.test_toy_build(extra_args=['--set-gid-bit', '--sticky-bit'])
+ self.run_test_toy_build_with_output(extra_args=['--set-gid-bit', '--sticky-bit'])
for subdir, _ in subdirs:
fullpath = os.path.join(self.test_installpath, *subdir)
perms = os.stat(fullpath).st_mode
@@ -733,9 +824,10 @@ def test_toy_group_check(self):
os.close(fd)
# figure out a group that we're a member of to use in the test
- out, ec = run_cmd('groups', simple=False)
- self.assertEqual(ec, 0, "Failed to select group to use in test")
- group_name = out.split(' ')[0].strip()
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd('groups')
+ self.assertEqual(res.exit_code, 0, "Failed to select group to use in test")
+ group_name = res.output.split(' ')[0].strip()
toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
test_ec = os.path.join(self.test_prefix, 'test.eb')
@@ -747,7 +839,7 @@ def test_toy_group_check(self):
for group in [group_name, (group_name, "Hey, you're not in the '%s' group!" % group_name)]:
- if isinstance(group, string_type):
+ if isinstance(group, str):
write_file(test_ec, read_file(toy_ec) + "\ngroup = '%s'\n" % group)
else:
write_file(test_ec, read_file(toy_ec) + "\ngroup = %s\n" % str(group))
@@ -757,14 +849,9 @@ def test_toy_group_check(self):
self.mock_stdout(False)
if get_module_syntax() == 'Tcl':
- pattern = "Can't generate robust check in TCL modules for users belonging to group %s." % group_name
- regex = re.compile(pattern, re.M)
- self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt))
-
- elif get_module_syntax() == 'Lua':
- lmod_version = os.getenv('LMOD_VERSION', 'NOT_FOUND')
- if LooseVersion(lmod_version) >= LooseVersion('6.0.8'):
- toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua')
+ module_version = LooseVersion(self.modtool.version)
+ if isinstance(self.modtool, EnvironmentModules) and module_version >= LooseVersion('4.6.0'):
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
toy_mod_txt = read_file(toy_mod)
if isinstance(group, tuple):
@@ -775,17 +862,35 @@ def test_toy_group_check(self):
error_msg_pattern = "You are not part of '%s' group of users" % group_name
pattern = '\n'.join([
- r'^if not \( userInGroup\("%s"\) \) then' % group_name,
- r' LmodError\("%s[^"]*"\)' % error_msg_pattern,
- r'end$',
+ r'^if \{ \!\[ module-info usergroups %s \] \} \{' % group_name,
+ r' error "%s[^"]*"' % error_msg_pattern,
+ r'\}$',
])
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt))
else:
- pattern = r"Can't generate robust check in Lua modules for users belonging to group %s. "
- pattern += r"Lmod version not recent enough \(%s\), should be >= 6.0.8" % lmod_version
- regex = re.compile(pattern % group_name, re.M)
+ pattern = "Can't generate robust check in Tcl modules for users belonging to group %s." % group_name
+ regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt))
+
+ elif get_module_syntax() == 'Lua':
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua')
+ toy_mod_txt = read_file(toy_mod)
+
+ if isinstance(group, tuple):
+ group_name = group[0]
+ error_msg_pattern = "Hey, you're not in the '%s' group!" % group_name
+ else:
+ group_name = group
+ error_msg_pattern = "You are not part of '%s' group of users" % group_name
+
+ pattern = '\n'.join([
+ r'^if not \( userInGroup\("%s"\) \) then' % group_name,
+ r' LmodError\("%s[^"]*"\)' % error_msg_pattern,
+ r'end$',
+ ])
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt))
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
@@ -801,7 +906,8 @@ def test_allow_system_deps(self):
shutil.copy2(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), tmpdir)
ec_file = os.path.join(tmpdir, 'toy-0.0.eb')
write_file(ec_file, "\nallow_system_deps = [('Python', SYS_PYTHON_VERSION)]\n", append=True)
- self.test_toy_build(ec_file=ec_file)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=ec_file)
shutil.rmtree(tmpdir)
def test_toy_hierarchical(self):
@@ -813,9 +919,6 @@ def test_toy_hierarchical(self):
args = [
os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'),
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
@@ -830,7 +933,8 @@ def test_toy_hierarchical(self):
# resolution) we must add an additional build option
'--disable-map-toolchains',
]
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2', 'toy', '0.0')
@@ -841,9 +945,9 @@ def test_toy_hierarchical(self):
# check that toolchain load is expanded to loads for toolchain dependencies,
# except for the ones that extend $MODULEPATH to make the toy module available
if get_module_syntax() == 'Tcl':
- load_regex_template = "load %s"
+ load_regex_template = "(load|depends-on) %s"
elif get_module_syntax() == 'Lua':
- load_regex_template = r'load\("%s/.*"\)'
+ load_regex_template = r'(load|depends_on)\("%s/.*"\)'
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
@@ -861,7 +965,8 @@ def test_toy_hierarchical(self):
extra_args = [
'--try-toolchain=GCC,6.4.0-2.28',
]
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '6.4.0-2.28', 'toy', '0.0')
@@ -878,7 +983,8 @@ def test_toy_hierarchical(self):
'--try-toolchain=GCC,6.4.0-2.28',
'--try-amend=moduleclass=mpi',
]
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '6.4.0-2.28', 'toy', '0.0')
@@ -901,7 +1007,8 @@ def test_toy_hierarchical(self):
# ... unless they shouldn't be
extra_args.append('--try-amend=include_modpath_extensions=') # pass empty string as equivalent to False
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
modtxt = read_file(toy_module_path)
modpath_extension = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'toy', '0.0')
if get_module_syntax() == 'Tcl':
@@ -918,7 +1025,8 @@ def test_toy_hierarchical(self):
extra_args = [
'--try-toolchain=system,system',
]
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0')
@@ -936,7 +1044,8 @@ def test_toy_hierarchical(self):
'--try-toolchain=system,system',
'--try-amend=moduleclass=compiler',
]
- self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0')
@@ -969,7 +1078,8 @@ def test_toy_hierarchical(self):
args[0] = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb')
self.modtool.purge()
- self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
self.assertExists(gompi_module_path)
def test_toy_hierarchical_subdir_user_modules(self):
@@ -1009,15 +1119,14 @@ def test_toy_hierarchical_subdir_user_modules(self):
args = [
os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a.eb'),
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
'--installpath=%s' % home,
'--unittest-file=%s' % self.logfile,
'--force',
'--module-naming-scheme=HierarchicalMNS',
'--try-toolchain=foss,2018a',
]
- self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
mod_ext = ''
if get_module_syntax() == 'Lua':
@@ -1074,7 +1183,8 @@ def test_toy_hierarchical_subdir_user_modules(self):
])
write_file(openmpi_mod + '.lua', openmpi_mod_txt)
- self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
toy_modtxt = read_file(toy_mod)
# No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary
@@ -1093,7 +1203,8 @@ def test_toy_advanced(self):
test_dir = os.path.abspath(os.path.dirname(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
- self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=['--debug'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=['--debug'])
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-gompi-2018a-test')
if get_module_syntax() == 'Lua':
@@ -1109,6 +1220,13 @@ def test_toy_advanced(self):
for pattern in patterns:
self.assertTrue(re.search(pattern, toy_mod_txt, re.M), "Pattern '%s' found in: %s" % (pattern, toy_mod_txt))
+ toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-gompi-2018a-test')
+ toy_libs_path = os.path.join(toy_installdir, 'toy_libs_path.txt')
+ self.assertTrue(os.path.exists(toy_libs_path))
+ txt = read_file(toy_libs_path)
+ regex = re.compile('^TOY_EXAMPLES=.*/examples$')
+ self.assertTrue(regex.match(txt), f"Pattern '{regex.pattern}' should match in: {txt}")
+
def test_toy_advanced_filter_deps(self):
"""Test toy build with extensions, and filtered build dependency."""
# test case for bug https://github.com/easybuilders/easybuild-framework/pull/2515
@@ -1124,7 +1242,8 @@ def test_toy_advanced_filter_deps(self):
test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, toy_ec_txt)
- self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=["--filter-deps=FFTW"])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=["--filter-deps=FFTW"])
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-gompi-2018a-test')
if get_module_syntax() == 'Lua':
@@ -1135,7 +1254,8 @@ def test_toy_hidden_cmdline(self):
"""Test installing a hidden module using the '--hidden' command line option."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
ec_file = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
- self.test_toy_build(ec_file=ec_file, extra_args=['--hidden'], verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=ec_file, extra_args=['--hidden'], verify=False)
# module file is hidden
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0')
if get_module_syntax() == 'Lua':
@@ -1153,7 +1273,8 @@ def test_toy_hidden_easyconfig(self):
shutil.copy2(ec_file, self.test_prefix)
ec_file = os.path.join(self.test_prefix, 'toy-0.0.eb')
write_file(ec_file, "\nhidden = True\n", append=True)
- self.test_toy_build(ec_file=ec_file, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=ec_file, verify=False)
# module file is hidden
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0')
if get_module_syntax() == 'Lua':
@@ -1172,15 +1293,13 @@ def test_module_filepath_tweaking(self):
eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
args = [
eb_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--force',
'--debug',
'--suffix-modules-path=foobarbaz',
'--module-naming-scheme=TestModuleNamingScheme',
]
- self.eb_main(args, do_build=True, verbose=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, verbose=True)
mod_file_prefix = os.path.join(self.test_installpath, 'modules')
mod_file_suffix = ''
if get_module_syntax() == 'Lua':
@@ -1199,7 +1318,8 @@ def test_toy_archived_easyconfig(self):
'--repository=FileRepository',
'--repositorypath=%s' % repositorypath,
]
- self.test_toy_build(raise_error=True, extra_args=extra_args)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(raise_error=True, extra_args=extra_args)
archived_ec = os.path.join(repositorypath, 'toy', 'toy-0.0.eb')
self.assertExists(archived_ec)
@@ -1214,7 +1334,8 @@ def test_toy_patches(self):
'--repository=FileRepository',
'--repositorypath=%s' % repositorypath,
]
- self.test_toy_build(raise_error=True, extra_args=extra_args)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(raise_error=True, extra_args=extra_args)
installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
@@ -1250,14 +1371,16 @@ def test_toy_extension_patches_postinstallcmds(self):
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
# make sure that patches were actually applied (without them the message producded by 'bar' is different)
bar_bin = os.path.join(installdir, 'bin', 'bar')
- out, _ = run_cmd(bar_bin)
- self.assertEqual(out, "I'm a bar, and very very proud of it.\n")
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(bar_bin)
+ self.assertEqual(res.output, "I'm a bar, and very very proud of it.\n")
# verify that post-install command for 'bar' extension was executed
fn = 'created-via-postinstallcmds.txt'
@@ -1289,7 +1412,8 @@ def test_toy_extension_sources(self):
']',
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
# copy bar-0.0.tar.gz to /bar-0.0-local.tar.gz, to be used below
test_source_path = os.path.join(self.test_prefix, 'sources')
@@ -1317,7 +1441,8 @@ def test_toy_extension_sources(self):
']',
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, raise_error=True)
# check that checksums are picked up and verified
test_ec_txt = '\n'.join([
@@ -1338,8 +1463,9 @@ def test_toy_extension_sources(self):
write_file(test_ec, test_ec_txt)
error_pattern = r"Checksum verification for extension source bar-0.0-local.tar.gz failed"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ raise_error=True, verbose=False)
# test again with correct checksum for bar-0.0.tar.gz, but faulty checksum for patch file
test_ec_txt = '\n'.join([
@@ -1360,8 +1486,9 @@ def test_toy_extension_sources(self):
write_file(test_ec, test_ec_txt)
error_pattern = r"Checksum verification for extension patch bar-0.0_fix-local.patch failed"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ raise_error=True, verbose=False)
# test again with correct checksums
test_ec_txt = '\n'.join([
@@ -1379,7 +1506,8 @@ def test_toy_extension_sources(self):
']',
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, raise_error=True)
def test_toy_extension_extract_cmd(self):
"""Test for custom extract_cmd specified for an extension."""
@@ -1402,9 +1530,12 @@ def test_toy_extension_extract_cmd(self):
])
write_file(test_ec, test_ec_txt)
- error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- raise_error=True, verbose=False)
+ error_pattern = r"shell command 'unzip \.\.\.' failed with exit code 9 in extensions step for test.eb"
+ with self.mocked_stdout_stderr():
+ # for now, we expect subprocess.CalledProcessError, but eventually 'run' function will
+ # do proper error reporting
+ self.assertErrorRegex(EasyBuildError, error_pattern,
+ self._test_toy_build, ec_file=test_ec, raise_error=True, verbose=False)
def test_toy_extension_sources_git_config(self):
"""Test install toy that includes extensions with 'sources' spec including 'git_config'."""
@@ -1440,7 +1571,8 @@ def test_toy_extension_sources_git_config(self):
']',
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
def test_toy_module_fulltxt(self):
"""Strict text comparison of generated module file."""
@@ -1510,21 +1642,22 @@ def test_toy_module_fulltxt(self):
r'',
r'conflict\("toy"\)',
r'',
+ r'prepend_path\("CMAKE_LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
r'prepend_path\("CMAKE_PREFIX_PATH", root\)',
r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)',
- r'setenv\("EBROOTTOY", root\)',
- r'setenv\("EBVERSIONTOY", "0.0"\)',
- r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)',
- r'',
- r'setenv\("FOO", "bar"\)',
r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)',
r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)',
r'prepend_path\("SOMEPATH", root\)',
r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)',
r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)',
r'append_path\("SOMEPATH_APPEND", root\)',
+ r'setenv\("EBROOTTOY", root\)',
+ r'setenv\("EBVERSIONTOY", "0.0"\)',
+ r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)',
+ r'',
+ r'setenv\("FOO", "bar"\)',
r'',
r'if mode\(\) == "load" then',
] + modloadmsg_lua + [
@@ -1551,21 +1684,22 @@ def test_toy_module_fulltxt(self):
r'',
r'conflict toy',
r'',
+ r'prepend-path CMAKE_LIBRARY_PATH \$root/lib',
r'prepend-path CMAKE_PREFIX_PATH \$root',
r'prepend-path LD_LIBRARY_PATH \$root/lib',
r'prepend-path LIBRARY_PATH \$root/lib',
r'prepend-path PATH \$root/bin',
- r'setenv EBROOTTOY "\$root"',
- r'setenv EBVERSIONTOY "0.0"',
- r'setenv EBDEVELTOY "\$root/easybuild/toy-0.0-tweaked-easybuild-devel"',
- r'',
- r'setenv FOO "bar"',
r'prepend-path SOMEPATH \$root/foo/bar',
r'prepend-path SOMEPATH \$root/baz',
r'prepend-path SOMEPATH \$root',
r'append-path SOMEPATH_APPEND \$root/qux/fred',
r'append-path SOMEPATH_APPEND \$root/thud',
r'append-path SOMEPATH_APPEND \$root',
+ r'setenv EBROOTTOY "\$root"',
+ r'setenv EBVERSIONTOY "0.0"',
+ r'setenv EBDEVELTOY "\$root/easybuild/toy-0.0-tweaked-easybuild-devel"',
+ r'',
+ r'setenv FOO "bar"',
r'',
r'if { \[ module-info mode load \] } {',
] + modloadmsg_tcl + [
@@ -1605,7 +1739,8 @@ def test_external_dependencies(self):
start_env = copy.deepcopy(os.environ)
- self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True)
self.modtool.load(['toy/0.0-external-deps'])
# note build dependency is not loaded
@@ -1625,8 +1760,9 @@ def test_external_dependencies(self):
else:
err_msg = r"Unable to locate a modulefile for 'nosuchbuilddep/0.0.0'"
- self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self._test_toy_build, ec_file=toy_ec,
+ raise_error=True, verbose=False)
extraectxt = "\ndependencies += [('nosuchmodule/1.2.3', EXTERNAL_MODULE)]"
extraectxt += "\nversionsuffix = '-external-deps-broken2'"
@@ -1637,11 +1773,13 @@ def test_external_dependencies(self):
else:
err_msg = r"Unable to locate a modulefile for 'nosuchmodule/1.2.3'"
- self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self._test_toy_build, ec_file=toy_ec,
+ raise_error=True, verbose=False)
# --dry-run still works when external modules are missing; external modules are treated as if they were there
- outtxt = self.test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False)
+ with self.mocked_stdout_stderr():
+ outtxt = self._test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False)
regex = re.compile(r"^ \* \[ \] .* \(module: toy/0.0-external-deps-broken2\)", re.M)
self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt))
@@ -1657,9 +1795,6 @@ def test_module_only(self):
# sanity check fails without --force if software is not installed yet
common_args = [
ec_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--robot=%s' % ec_files_path,
@@ -1667,30 +1802,36 @@ def test_module_only(self):
]
args = common_args + ['--module-only']
err_msg = "Sanity check failed"
- self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
self.assertNotExists(toy_mod)
- self.eb_main(args + ['--force'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + ['--force'], do_build=True, raise_error=True)
self.assertExists(toy_mod)
# make sure load statements for dependencies are included in additional module file generated with --module-only
modtxt = read_file(toy_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
- self.assertTrue(re.search('load.*GCC/6.4.0-2.28', modtxt), "load statement for GCC/6.4.0-2.28 found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*GCC/6.4.0-2.28', modtxt),
+ "load statement for GCC/6.4.0-2.28 found in module")
os.remove(toy_mod)
# --module-only --rebuild should run sanity check
rebuild_args = args + ['--rebuild']
err_msg = "Sanity check failed"
- self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, rebuild_args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, rebuild_args, do_build=True, raise_error=True)
self.assertNotExists(toy_mod)
# installing another module under a different naming scheme and using Lua module syntax works fine
# first actually build and install toy software + module
prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps')
- self.eb_main(common_args + ['--force'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(common_args + ['--force'], do_build=True, raise_error=True)
self.assertExists(toy_mod)
self.assertExists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))
modtxt = read_file(toy_mod)
@@ -1705,7 +1846,8 @@ def test_module_only(self):
]
toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0-deps')
self.assertNotExists(toy_core_mod)
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_core_mod)
# existing install is reused
modtxt2 = read_file(toy_core_mod)
@@ -1715,14 +1857,16 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_core_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
# Test we can create a module even for an installation where we don't have write permissions
os.remove(toy_core_mod)
# remove the write permissions on the installation
adjust_permissions(prefix, stat.S_IRUSR | stat.S_IXUSR, relative=False)
self.assertNotExists(toy_core_mod)
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_core_mod)
# existing install is reused
modtxt2 = read_file(toy_core_mod)
@@ -1732,7 +1876,8 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_core_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
os.remove(toy_core_mod)
os.remove(toy_mod)
@@ -1747,7 +1892,8 @@ def test_module_only(self):
'--modules-tool=Lmod',
]
self.assertNotExists(toy_mod + '.lua')
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_mod + '.lua')
# existing install is reused
modtxt3 = read_file(toy_mod + '.lua')
@@ -1757,7 +1903,8 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_mod + '.lua')
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
def test_module_only_extensions(self):
"""
@@ -1791,7 +1938,8 @@ def test_module_only_extensions(self):
self.assertEqual(self.modtool.available('toy'), [])
# install toy/0.0
- self.eb_main([test_ec], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([test_ec], do_build=True, raise_error=True)
# remove module file so we can try --module-only
remove_file(toy_mod)
@@ -1802,7 +1950,8 @@ def test_module_only_extensions(self):
del os.environ['EASYBUILD_SOURCEPATH']
# first try normal --module-only, should work fine
- self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
self.assertExists(toy_mod)
remove_file(toy_mod)
@@ -1813,18 +1962,21 @@ def test_module_only_extensions(self):
# check whether sanity check fails now when using --module-only
error_pattern = 'Sanity check failed: command "ls -l lib/libbarbar.a" failed'
for extra_args in (['--module-only'], ['--module-only', '--rebuild']):
- self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args,
- do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args,
+ do_build=True, raise_error=True)
self.assertNotExists(toy_mod)
# failing sanity check for barbar extension is ignored when using --module-only --skip-extensions
for extra_args in (['--module-only'], ['--module-only', '--rebuild']):
- self.eb_main([test_ec, '--skip-extensions'] + extra_args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([test_ec, '--skip-extensions'] + extra_args, do_build=True, raise_error=True)
self.assertExists(toy_mod)
remove_file(toy_mod)
# we can force module generation via --force (which skips sanity check entirely)
- self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True)
self.assertExists(toy_mod)
def test_toy_exts_parallel(self):
@@ -1855,7 +2007,7 @@ def test_toy_exts_parallel(self):
write_file(test_ec, test_ec_txt)
args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3']
- stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True)
self.assertEqual(stderr, '')
# take into account that each of these lines may appear multiple times,
@@ -1874,7 +2026,7 @@ def test_toy_exts_parallel(self):
# also test skipping of extensions in parallel
args.append('--skip')
- stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True)
self.assertEqual(stderr, '')
# order in which these patterns occur is not fixed, so check them one by one
@@ -1900,7 +2052,7 @@ def test_toy_exts_parallel(self):
write_file(toy_ext_eb, toy_ext_eb_txt)
args[-1] = '--include-easyblocks=%s' % toy_ext_eb
- stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True)
self.assertEqual(stderr, '')
# take into account that each of these lines may appear multiple times,
# in case no progress was made between checks
@@ -1927,9 +2079,6 @@ def test_backup_modules(self):
common_args = [
ec_file,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--robot=%s' % ec_files_path,
@@ -1939,11 +2088,13 @@ def test_backup_modules(self):
args = common_args + ['--module-syntax=Tcl']
# install module once (without --module-only), so it can be backed up
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_mod)
# forced reinstall, no backup of module file because --backup-modules (or --module-only) is not used
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_mod)
toy_mod_backups = glob.glob(os.path.join(toy_mod_dir, '.' + toy_mod_fn + '.bak_*'))
self.assertEqual(len(toy_mod_backups), 0)
@@ -1972,7 +2123,8 @@ def test_backup_modules(self):
self.assertEqual(stderr, '')
# no backup of existing module file if --disable-backup-modules is used
- self.eb_main(args + ['--disable-backup-modules'], do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + ['--disable-backup-modules'], do_build=True, raise_error=True)
toy_mod_backups = glob.glob(os.path.join(toy_mod_dir, '.' + toy_mod_fn + '.bak_*'))
self.assertEqual(len(toy_mod_backups), 1)
@@ -2007,7 +2159,8 @@ def test_backup_modules(self):
toy_mod = os.path.join(toy_mod_dir, toy_mod_fn + '.lua')
# initial installation of Lua module file
- self.eb_main(args, do_build=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
self.assertExists(toy_mod)
lua_toy_mods = glob.glob(os.path.join(toy_mod_dir, '*.lua*'))
self.assertEqual(len(lua_toy_mods), 1)
@@ -2101,7 +2254,8 @@ def test_package(self):
'--packagepath=%s' % pkgpath,
]
- self.test_toy_build(extra_args=extra_args)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=extra_args)
toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.321.foo' % EASYBUILD_VERSION)
self.assertExists(toypkg)
@@ -2111,11 +2265,12 @@ def test_package_skip(self):
mock_fpm(self.test_prefix)
pkgpath = os.path.join(self.test_prefix, 'packages') # default path
- self.test_toy_build(['--packagepath=%s' % pkgpath])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(['--packagepath=%s' % pkgpath])
self.assertNotExists(pkgpath, "%s is not created without use of --package" % pkgpath)
self.mock_stdout(True)
- self.test_toy_build(extra_args=['--package', '--skip'], verify=False)
+ self._test_toy_build(extra_args=['--package', '--skip'], verify=False)
self.mock_stdout(False)
toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.1.rpm' % EASYBUILD_VERSION)
@@ -2123,14 +2278,8 @@ def test_package_skip(self):
def test_regtest(self):
"""Test use of --regtest."""
-
- # skip test when using Python 2, since it somehow fails then,
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333
- if sys.version_info[0] == 2:
- print("Skipping test_regtest because Python 2.x is being used")
- return
-
- self.test_toy_build(extra_args=['--regtest', '--sequential'], verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=['--regtest', '--sequential'], verify=False)
# just check whether module exists
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
@@ -2141,17 +2290,12 @@ def test_regtest(self):
def test_minimal_toolchains(self):
"""Test toy build with --minimal-toolchains."""
# this test doesn't check for anything specific to using minimal toolchains, only side-effects
- self.test_toy_build(extra_args=['--minimal-toolchains'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=['--minimal-toolchains'])
def test_reproducibility(self):
"""Test toy build produces expected reproducibility files"""
- # skip test when using Python 2, since it somehow fails then,
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333
- if sys.version_info[0] == 2:
- print("Skipping test_reproducibility because Python 2.x is being used")
- return
-
# We need hooks for a complete test
hooks_filename = 'my_hooks.py'
hooks_file = os.path.join(self.test_prefix, hooks_filename)
@@ -2169,7 +2313,7 @@ def test_reproducibility(self):
# also use the easyblock with inheritance to fully test
self.mock_stdout(True)
- self.test_toy_build(extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy', '--hooks=%s' % hooks_file])
+ self._test_toy_build(extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy', '--hooks=%s' % hooks_file])
self.mock_stdout(False)
# Check whether easyconfig is dumped to reprod/ subdir
@@ -2234,7 +2378,9 @@ def test_reproducibility_ext_easyblocks(self):
])
write_file(ec1, ec1_txt)
- self.test_toy_build(ec_file=ec1, verify=False, extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=ec1, verify=False,
+ extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy'])
# Check whether easyconfig is dumped to reprod/ subdir
reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod')
@@ -2259,6 +2405,7 @@ def test_reproducibility_ext_easyblocks(self):
def test_toy_toy(self):
"""Test building two easyconfigs in a single go, with one depending on the other."""
+
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
toy_ec_txt = read_file(toy_ec_file)
@@ -2270,18 +2417,27 @@ def test_toy_toy(self):
])
write_file(ec1, ec1_txt)
+ # adapt toy easyconfig for toy2 to produce a modulefile with a dedicated
+ # name ('toy2' instead of 'toy')
ec2 = os.path.join(self.test_prefix, 'toy2.eb')
ec2_txt = '\n'.join([
toy_ec_txt,
+ "name = 'toy2'",
+ "easyblock = 'EB_toy'",
+ "sources = ['toy/toy-0.0.tar.gz']",
+ "patches = []",
+ "sanity_check_paths = { 'files': ['bin/toy2'], 'dirs': ['bin']}",
+ "prebuildopts = 'mv toy.source toy2.c &&'",
"versionsuffix = '-two'",
"dependencies = [('toy', '0.0', '-one')]",
])
write_file(ec2, ec2_txt)
- self.test_toy_build(ec_file=self.test_prefix, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=self.test_prefix, verify=False)
mod1 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-one')
- mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-two')
+ mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy2', '0.0-two')
if get_module_syntax() == 'Lua':
mod1 += '.lua'
mod2 += '.lua'
@@ -2290,19 +2446,19 @@ def test_toy_toy(self):
mod2_txt = read_file(mod2)
- load1_regex = re.compile('load.*toy/0.0-one', re.M)
+ load1_regex = re.compile('(load|depends[-_]on).*toy/0.0-one', re.M)
self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt))
# Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load
- reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod')
- dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env')
+ reprod_dir = os.path.join(self.test_installpath, 'software', 'toy2', '0.0-two', 'easybuild', 'reprod')
+ dumpenv_script = os.path.join(reprod_dir, 'toy2-0.0-two.env')
reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script)
self.assertExists(reprod_dumpenv)
# Check contents of the dumpenv script
patterns = [
"""#!/bin/bash""",
- """# usage: source toy-0.0-two.env""",
+ """# usage: source toy2-0.0-two.env""",
# defining build env
"""module load toy/0.0-one""",
"""# (no build environment defined)""",
@@ -2319,6 +2475,8 @@ def test_toy_sanity_check_commands(self):
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))
+ out_file = os.path.join(self.test_prefix, 'out.txt')
+
toy_ec_txt = '\n'.join([
toy_ec_txt,
"toolchain = {'name': 'foss', 'version': '2018a'}",
@@ -2334,6 +2492,8 @@ def test_toy_sanity_check_commands(self):
" True,",
# test command to make sure that '-h' is not passed to commands specified as string ('env -h' fails)
" 'env',"
+ # print current working directory, should *not* be software install directory, but empty dir
+ f" '(pwd && ls | wc -l) > {out_file}',",
"]",
])
@@ -2342,16 +2502,14 @@ def test_toy_sanity_check_commands(self):
args = [
tweaked_toy_ec,
- '--sourcepath=%s' % self.test_sourcepath,
- '--buildpath=%s' % self.test_buildpath,
- '--installpath=%s' % self.test_installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
'--robot=%s' % test_easyconfigs,
'--module-naming-scheme=HierarchicalMNS',
]
- self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
modpath = os.path.join(self.test_installpath, 'modules', 'all')
toy_modfile = os.path.join(modpath, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2', 'toy', '0.0')
@@ -2360,6 +2518,13 @@ def test_toy_sanity_check_commands(self):
self.assertExists(toy_modfile)
+ # check contents of output file created by sanity check commands
+ self.assertExists(out_file)
+ out_txt = read_file(out_file)
+ # working dir for sanity check command should be an empty custom temporary directory
+ regex = re.compile('^.*/eb-[^/]+/eb-sanity-check-[^/]+\n[ ]*0$')
+ self.assertTrue(regex.match(out_txt), f"Pattern '{regex.pattern}' should match in: {out_txt}")
+
def test_sanity_check_paths_lib64(self):
"""Test whether fallback in sanity check for lib64/ equivalents of library files works."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
@@ -2368,7 +2533,14 @@ def test_sanity_check_paths_lib64(self):
# modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a
ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt)
- postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a"
+ postinstallcmd = ' && '.join([
+ # remove lib64 symlink (if it's there)
+ "rm -f %(installdir)s/lib64",
+ # create empty lib64 dir
+ "mkdir %(installdir)s/lib64",
+ # move libtoy*.a
+ "mv %(installdir)s/lib/libtoy*.a %(installdir)s/lib64/",
+ ])
ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt)
test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb')
@@ -2376,12 +2548,14 @@ def test_sanity_check_paths_lib64(self):
# sanity check fails if lib64 fallback in sanity check is disabled
error_pattern = r"Sanity check failed: no file found at 'lib/libtoy.a' or 'lib/libfoo.a' in "
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+ raise_error=True, verbose=False)
# all is fine is lib64 fallback check is enabled (which it is by default)
- self.test_toy_build(ec_file=test_ec, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, raise_error=True)
# also check with 'lib' in sanity check dirs (special case)
ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy'],", ectxt)
@@ -2389,11 +2563,13 @@ def test_sanity_check_paths_lib64(self):
write_file(test_ec, ectxt)
error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib' in "
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+ raise_error=True, verbose=False)
- self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
# also check other way around (lib64 -> lib)
ectxt = read_file(ec_file)
@@ -2402,12 +2578,14 @@ def test_sanity_check_paths_lib64(self):
# sanity check fails if lib64 fallback in sanity check is disabled, since lib64/libtoy.a is not there
error_pattern = r"Sanity check failed: no file found at 'lib64/libtoy.a' in "
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+ raise_error=True, verbose=False)
# sanity check passes when lib64 fallback is enabled (by default), since lib/libtoy.a is also considered
- self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
# also check with 'lib64' in sanity check dirs (special case)
ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy'],", ectxt)
@@ -2415,11 +2593,13 @@ def test_sanity_check_paths_lib64(self):
write_file(test_ec, ectxt)
error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib64' in "
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
- raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+ raise_error=True, verbose=False)
- self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
# check whether fallback works for files that's more than 1 subdir deep
ectxt = read_file(ec_file)
@@ -2428,7 +2608,8 @@ def test_sanity_check_paths_lib64(self):
postinstallcmd += "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/test/libtoy.a"
ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt)
write_file(test_ec, ectxt)
- self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
def test_toy_build_enhanced_sanity_check(self):
"""Test enhancing of sanity check."""
@@ -2476,7 +2657,7 @@ def test_toy_build_enhanced_sanity_check(self):
# by default, sanity check commands & paths specified by easyblock are used
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
stdout = self.get_stdout()
self.mock_stdout(False)
@@ -2509,7 +2690,7 @@ def test_toy_build_enhanced_sanity_check(self):
write_file(test_ec, test_ec_txt)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
stdout = self.get_stdout()
self.mock_stdout(False)
@@ -2533,7 +2714,7 @@ def test_toy_build_enhanced_sanity_check(self):
write_file(test_ec, test_ec_txt)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
stdout = self.get_stdout()
self.mock_stdout(False)
@@ -2566,7 +2747,7 @@ def test_toy_build_enhanced_sanity_check(self):
]
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
stdout = self.get_stdout()
self.mock_stdout(False)
@@ -2587,37 +2768,128 @@ def test_toy_build_enhanced_sanity_check(self):
test_ec_txt = test_ec_txt + '\nenhance_sanity_check = False'
write_file(test_ec, test_ec_txt)
- error_pattern = r" Missing mandatory key 'dirs' in sanity_check_paths."
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
- extra_args=eb_args, raise_error=True, verbose=False)
+ error_pattern = r"Missing mandatory key 'dirs' in sanity_check_paths."
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec,
+ extra_args=eb_args, raise_error=True, verbose=False)
del sys.modules['easybuild.easyblocks.toy']
+ def test_toy_build_enhanced_sanity_check_templated_multi_dep(self):
+ """Test enhancing of sanity check by easyblocks with templates and in the presence of multi_deps."""
+
+ # if toy easyblock was imported, get rid of corresponding entry in sys.modules,
+ # to avoid that it messes up the use of --include-easyblocks=toy.py below...
+ if 'easybuild.easyblocks.toy' in sys.modules:
+ del sys.modules['easybuild.easyblocks.toy']
+
+ test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)))
+ toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ toy_ec_txt = read_file(toy_ec)
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+
+ # get rid of custom sanity check paths in test easyconfig
+ regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M)
+ test_ec_txt = regex.sub('', toy_ec_txt)
+ self.assertNotIn('sanity_check_', test_ec_txt)
+
+ test_ec_txt += "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}"
+ write_file(test_ec, test_ec_txt)
+
+ # create custom easyblock for toy that has a custom sanity_check_step
+ toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py')
+
+ toy_easyblock_txt = read_file(toy_easyblock)
+
+ toy_custom_sanity_check_step = textwrap.dedent("""
+ # Add to class to indent
+ def sanity_check_step(self):
+ paths = {
+ 'files': ['bin/python%(pyshortver)s'],
+ 'dirs': ['lib/py-%(pyshortver)s'],
+ }
+ cmds = ['python%(pyshortver)s']
+ return super(EB_toy, self).sanity_check_step(custom_paths=paths, custom_commands=cmds)
+ """)
+ test_toy_easyblock = os.path.join(self.test_prefix, 'toy.py')
+ write_file(test_toy_easyblock, toy_easyblock_txt + toy_custom_sanity_check_step)
+
+ eb_args = [
+ '--extended-dry-run',
+ '--include-easyblocks=%s' % test_toy_easyblock,
+ ]
+
+ # by default, sanity check commands & paths specified by easyblock are used
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ stdout = self.get_stdout()
+ # Cut output to start of the toy-ec, after the Python installations
+ stdout = stdout[stdout.index(test_ec):]
+
+ pattern_template = textwrap.dedent(r"""
+ Sanity check paths - file.*
+ \s*\* bin/python{pyshortver}
+ Sanity check paths - \(non-empty\) directory.*
+ \s*\* lib/py-{pyshortver}
+ Sanity check commands
+ \s*\* python{pyshortver}
+ """)
+ for pyshortver in ('2.7', '3.7'):
+ regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+ # Enhance sanity check by extra paths to check for, the ones from the easyblock should be kept
+ test_ec_txt += textwrap.dedent("""
+ enhance_sanity_check = True
+ sanity_check_paths = {
+ 'files': ['bin/pip%(pyshortver)s'],
+ 'dirs': ['bin'],
+ }
+ """)
+ write_file(test_ec, test_ec_txt)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+ stdout = self.get_stdout()
+ # Cut output to start of the toy-ec, after the Python installations
+ stdout = stdout[stdout.index(test_ec):]
+
+ pattern_template = textwrap.dedent(r"""
+ Sanity check paths - file.*
+ \s*\* bin/pip{pyshortver}
+ \s*\* bin/python{pyshortver}
+ Sanity check paths - \(non-empty\) directory.*
+ \s*\* bin
+ \s*\* lib/py-{pyshortver}
+ Sanity check commands
+ \s*\* python{pyshortver}
+ """)
+ for pyshortver in ('2.7', '3.7'):
+ regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
def test_toy_dumped_easyconfig(self):
- """ Test dumping of file in eb_filerepo in both .eb and .yeb format """
+ """ Test dumping of file in eb_filerepo in both .eb format """
filename = 'toy-0.0'
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
paths = [
os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', '%s.eb' % filename),
- os.path.join(test_ecs_dir, 'yeb', '%s.yeb' % filename),
]
for path in paths:
- if path.endswith('.yeb') and 'yaml' not in sys.modules:
- print("Skipping .yeb part of test_toy_dumped_easyconfig (no PyYAML available)")
- continue
-
args = [
path,
'--experimental',
'--force',
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
# test eb build with dumped file
args[0] = os.path.join(get_repositorypath()[0], 'toy', 'toy-0.0%s' % os.path.splitext(path)[-1])
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
easybuild.tools.build_log.EXPERIMENTAL = True
ec = EasyConfig(args[0])
buildstats = ec.parser.get_config_dict()['buildstats']
@@ -2636,12 +2908,14 @@ def test_toy_filter_env_vars(self):
re.compile("prepend[-_]path.*PATH.*bin", re.M),
]
- self.test_toy_build()
+ with self.mocked_stdout_stderr():
+ self._test_toy_build()
toy_mod_txt = read_file(toy_mod_path)
for regex in regexs:
self.assertTrue(regex.search(toy_mod_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt))
- self.test_toy_build(extra_args=['--filter-env-vars=LD_LIBRARY_PATH,PATH'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=['--filter-env-vars=LD_LIBRARY_PATH,PATH'])
toy_mod_txt = read_file(toy_mod_path)
self.assertFalse(regexs[0].search(toy_mod_txt), "Pattern '%s' found in: %s" % (regexs[0].pattern, toy_mod_txt))
self.assertTrue(regexs[1].search(toy_mod_txt), "Pattern '%s' found in: %s" % (regexs[1].pattern, toy_mod_txt))
@@ -2656,7 +2930,8 @@ def test_toy_iter(self):
for extra_args in [None, ['--minimal-toolchains']]:
# sanity check will make sure all entries in buildopts list were taken into account
- self.test_toy_build(ec_file=toy_ec, extra_args=extra_args, versionsuffix='-iter')
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, extra_args=extra_args, versionsuffix='-iter')
# verify whether dumped easyconfig contains original value for buildopts
dumped_toy_ec = os.path.join(self.test_prefix, 'ebfiles_repo', 'toy', os.path.basename(toy_ec))
@@ -2693,7 +2968,8 @@ def grab_gcc_rpath_wrapper_args():
return {'filter_paths': res_filter.group(1), 'include_paths': res_include.group(1)}
args = ['--rpath']
- self.test_toy_build(extra_args=args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=args, raise_error=True)
# by default, /lib and /usr are included in RPATH filter,
# together with temporary directory and build directory
@@ -2705,7 +2981,8 @@ def grab_gcc_rpath_wrapper_args():
# Check that we can use --rpath-override-dirs
args = ['--rpath', '--rpath-override-dirs=/opt/eessi/2021.03/lib:/opt/eessi/lib']
- self.test_toy_build(extra_args=args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=args, raise_error=True)
rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',')
# Make sure our directories appear in dirs to be included in the rpath (and in the right order)
self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib')
@@ -2713,7 +2990,8 @@ def grab_gcc_rpath_wrapper_args():
# Check that when we use --rpath-override-dirs empty values are filtered
args = ['--rpath', '--rpath-override-dirs=/opt/eessi/2021.03/lib::/opt/eessi/lib']
- self.test_toy_build(extra_args=args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=args, raise_error=True)
rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',')
# Make sure our directories appear in dirs to be included in the rpath (and in the right order)
self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib')
@@ -2722,12 +3000,14 @@ def grab_gcc_rpath_wrapper_args():
# Check that when we use --rpath-override-dirs we can only provide absolute paths
eb_args = ['--rpath', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib']
error_pattern = r"Path used in rpath_override_dirs is not an absolute path: eessi/lib"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=eb_args, raise_error=True,
- verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, extra_args=eb_args,
+ raise_error=True, verbose=False)
# also test use of --rpath-filter
args.extend(['--rpath-filter=/test.*,/foo/bar.*', '--disable-cleanup-tmpdir'])
- self.test_toy_build(extra_args=args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=args, raise_error=True)
# check whether rpath filter was set correctly
rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',')
@@ -2736,13 +3016,15 @@ def grab_gcc_rpath_wrapper_args():
self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths))
self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths))
- # test use of rpath toolchain option
- test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
- toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'))
- toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n"
- toy_ec = os.path.join(self.test_prefix, 'toy.eb')
- write_file(toy_ec, toy_ec_txt)
- self.test_toy_build(ec_file=toy_ec, extra_args=['--rpath'], raise_error=True)
+ # test use of rpath toolchain option with SYSTEM and gompi 2018b toolchains
+ for toolchain in ['', '-gompi-2018a']:
+ test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', f'toy-0.0{toolchain}.eb'))
+ toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" # overwrites existing toolchainopts
+ toy_ec = os.path.join(self.test_prefix, 'toy.eb')
+ write_file(toy_ec, toy_ec_txt)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, extra_args=['--rpath'], raise_error=True)
def test_toy_filter_rpath_sanity_libs(self):
"""Test use of --filter-rpath-sanity-libs."""
@@ -2751,58 +3033,83 @@ def test_toy_filter_rpath_sanity_libs(self):
toy_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb')
# This should just build succesfully
- args = ['--rpath']
- self.test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
+ rpath_args = ['--rpath', '--strict-rpath-sanity-check']
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=rpath_args, raise_error=True)
libtoy_libdir = os.path.join(self.test_installpath, 'software', 'libtoy', '0.0', 'lib')
toyapp_bin = os.path.join(self.test_installpath, 'software', 'toy-app', '0.0', 'bin', 'toy-app')
- rpath_regex = re.compile(r"RPATH.*%s" % libtoy_libdir, re.M)
- out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False)
- self.assertTrue(rpath_regex.search(out), "Pattern '%s' should be found in: %s" % (rpath_regex.pattern, out))
+ rpath_regex = re.compile(r"\(RPATH\).*" + libtoy_libdir, re.M)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"readelf -d {toyapp_bin}")
+ self.assertTrue(rpath_regex.search(res.output),
+ f"Pattern '{rpath_regex.pattern}' should be found in: {res.output}")
- out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"ldd {toyapp_bin}")
+ out = res.output
libtoy_regex = re.compile(r"libtoy.so => /.*/libtoy.so", re.M)
notfound = re.compile(r"libtoy\.so\s*=>\s*not found", re.M)
- self.assertTrue(libtoy_regex.search(out), "Pattern '%s' should be found in: %s" % (libtoy_regex.pattern, out))
- self.assertFalse(notfound.search(out), "Pattern '%s' should not be found in: %s" % (notfound.pattern, out))
+ self.assertTrue(libtoy_regex.search(out), f"Pattern '{libtoy_regex.pattern}' should be found in: {out}")
+ self.assertFalse(notfound.search(out), f"Pattern '{notfound.pattern}' should not be found in: {out}")
# test sanity error when --rpath-filter is used to filter a required library
# In this test, libtoy.so will be linked, but not RPATH-ed due to the --rpath-filter
# Thus, the RPATH sanity check is expected to fail with libtoy.so not being found
+ args = rpath_args + ['--rpath-filter=.*libtoy.*']
error_pattern = r"Sanity check failed\: Library libtoy\.so not found"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=toy_ec,
- extra_args=['--rpath', '--rpath-filter=.*libtoy.*'],
- name='toy-app', raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=toy_ec,
+ extra_args=args, name='toy-app', raise_error=True, verbose=False)
# test use of --filter-rpath-sanity-libs option. In this test, we use --rpath-filter to make sure libtoy.so is
# not rpath-ed. Then, we use --filter-rpath-sanity-libs to make sure the RPATH sanity checks ignores
# the fact that libtoy.so is not found. Thus, this build should complete succesfully
- args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libtoy.so']
- self.test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
+ args = rpath_args + ['--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libtoy.so']
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
- out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False)
- self.assertFalse(rpath_regex.search(out),
- "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"readelf -d {toyapp_bin}")
+ self.assertFalse(rpath_regex.search(res.output),
+ f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}")
- out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False)
- self.assertFalse(libtoy_regex.search(out),
- "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out))
- self.assertTrue(notfound.search(out),
- "Pattern '%s' should be found in: %s" % (notfound.pattern, out))
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"ldd {toyapp_bin}")
+ self.assertFalse(libtoy_regex.search(res.output),
+ f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}")
+ self.assertTrue(notfound.search(res.output),
+ f"Pattern '{notfound.pattern}' should be found in: {res.output}")
# test again with list of library names passed to --filter-rpath-sanity-libs
- args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so']
- self.test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
+ args = rpath_args + ['--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so']
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
- out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"readelf -d {toyapp_bin}")
self.assertFalse(rpath_regex.search(out),
- "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out))
+ f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}")
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"ldd {toyapp_bin}")
+ self.assertFalse(libtoy_regex.search(res.output),
+ f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}")
+ self.assertTrue(notfound.search(res.output),
+ f"Pattern '{notfound.pattern}' should be found in: {res.output}")
+
+ # by default, without using --strict-rpath-sanity-check, there's no failure since RPATH sanity check
+ # doesn't check for missing libraries with $LD_LIBRARY_PATH unset
+ args = ['--rpath', '--rpath-filter=.*libtoy.*']
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True, verbose=False)
- out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False)
- self.assertFalse(libtoy_regex.search(out),
- "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out))
- self.assertTrue(notfound.search(out),
- "Pattern '%s' should be found in: %s" % (notfound.pattern, out))
+ # trouble again when $LD_LIBRARY_PATH is not used in generated module file
+ args = ['libtoy-0.0.eb', '--rebuild', '--rpath', '--rpath-filter=.*libtoy.*',
+ '--filter-env-vars=LD_LIBRARY_PATH']
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=toy_ec,
+ extra_args=args, name='toy-app', raise_error=True, verbose=False)
def test_toy_modaltsoftname(self):
"""Build two dependent toys as in test_toy_toy but using modaltsoftname"""
@@ -2833,7 +3140,8 @@ def test_toy_modaltsoftname(self):
'--module-naming-scheme=HierarchicalMNS',
'--robot-paths=%s' % self.test_prefix,
]
- self.test_toy_build(ec_file=self.test_prefix, verify=False, extra_args=extra_args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=self.test_prefix, verify=False, extra_args=extra_args, raise_error=True)
software_path = os.path.join(self.test_installpath, 'software')
modules_path = os.path.join(self.test_installpath, 'modules', 'all', 'Core')
@@ -2872,7 +3180,7 @@ def test_toy_build_trace(self):
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=['--trace'], verify=False, testing=False)
+ self._test_toy_build(ec_file=test_ec, extra_args=['--trace'], verify=False, testing=False)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
@@ -2882,14 +3190,15 @@ def test_toy_build_trace(self):
patterns = [
r"^ >> installation prefix: .*/software/toy/0\.0$",
- r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$",
+ r"^== fetching files and verifying checksums\.\.\.\n"
+ r" >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$",
r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$",
r'\n'.join([
- r"^ >> running command:",
+ r"^ >> running shell command:",
+ r"\tgcc toy.c -o toy\n"
r"\t\[started at: .*\]",
r"\t\[working dir: .*\]",
- r"\t\[output logged in .*\]",
- r"\tgcc toy.c -o toy\n"
+ r"\t\[output and state saved to .*\]",
r'',
]),
r" >> command completed: exit 0, ran in .*",
@@ -2960,7 +3269,7 @@ def end_hook():
print('end hook triggered, all done!')
def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
- if cmd.strip() == TOY_COMP_CMD:
+ if isinstance(cmd, str) and cmd.strip() == TOY_COMP_CMD:
print("pre_run_shell_cmd_hook triggered for '%s'" % cmd)
# 'copy_toy_file' command doesn't exist, but don't worry,
# this problem will be fixed in post_run_shell_cmd_hook
@@ -2971,7 +3280,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
exit_code = kwargs['exit_code']
output = kwargs['output']
work_dir = kwargs['work_dir']
- if cmd.strip().startswith(TOY_COMP_CMD) and exit_code:
+ if isinstance(cmd, str) and cmd.strip().startswith(TOY_COMP_CMD) and exit_code:
cwd = change_dir(work_dir)
copy_file('toy', 'copy_of_toy')
change_dir(cwd)
@@ -2979,9 +3288,15 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
""")
write_file(hooks_file, hooks_file_txt)
+ extra_args = [
+ '--hooks=%s' % hooks_file,
+ # disable trace output to make checking of generated output produced by hooks easier
+ '--disable-trace',
+ ]
+
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False)
+ self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True, debug=False)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
@@ -2991,6 +3306,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0')
if get_module_syntax() == 'Lua':
toy_mod_file += '.lua'
+ mod_name = os.path.basename(toy_mod_file)
warnings = nub([x for x in stderr.strip().splitlines() if x])
self.assertEqual(warnings, ["WARNING: Command ' gcc toy.c -o toy ' failed, but we'll ignore it..."])
@@ -3001,7 +3317,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
# - for fake module file being created during sanity check (triggered twice, for main + toy install)
# - for final module file
# - for devel module file
- expected_output = textwrap.dedent("""
+ expected_output = textwrap.dedent(f"""
start hook triggered
toy 0.0
['%(name)s-%(version)s.tar.gz']
@@ -3051,6 +3367,7 @@ def test_toy_multi_deps(self):
test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]"
test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}"
+
write_file(test_ec, test_ec_txt)
test_mod_path = os.path.join(self.test_installpath, 'modules', 'all')
@@ -3067,7 +3384,8 @@ def test_toy_multi_deps(self):
# to make situation where GCC/7.3.0-2.30 is loaded when GCC/4.6.3 is already loaded (by default) fail
os.environ['LMOD_DISABLE_SAME_NAME_AUTOSWAP'] = 'yes'
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0')
if get_module_syntax() == 'Lua':
@@ -3078,15 +3396,25 @@ def test_toy_multi_deps(self):
# check whether (guarded) load statement for first version listed in multi_deps is there
if get_module_syntax() == 'Lua':
expected = '\n'.join([
- 'if not ( isloaded("GCC/4.6.3") ) and not ( isloaded("GCC/7.3.0-2.30") ) then',
- ' load("GCC/4.6.3")',
+ 'if mode() == "unload" or isloaded("GCC/7.3.0-2.30") then',
+ ' depends_on("GCC")',
+ 'else',
+ ' depends_on("GCC/4.6.3")',
'end',
])
else:
+ if isinstance(self.modtool, EnvironmentModules):
+ load_stmt = "module load"
+ else:
+ load_stmt = "depends-on"
expected = '\n'.join([
- 'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {',
- ' module load GCC/4.6.3',
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded GCC/7.3.0-2.30 ] } {",
+ " %s GCC" % load_stmt,
+ '} else {',
+ " %s GCC/4.6.3" % load_stmt,
'}',
+ '',
])
self.assertIn(expected, toy_mod_txt)
@@ -3175,7 +3503,8 @@ def check_toy_load(depends_on=False):
write_file(test_ec, test_ec_txt + "\nmulti_deps_load_default = False")
remove_file(toy_mod_file)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
toy_mod_txt = read_file(toy_mod_file)
self.assertNotIn(expected, toy_mod_txt)
@@ -3217,7 +3546,8 @@ def check_toy_load(depends_on=False):
if self.modtool.supports_depends_on:
remove_file(toy_mod_file)
- self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'], raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'], raise_error=True)
toy_mod_txt = read_file(toy_mod_file)
@@ -3321,7 +3651,8 @@ def test_fix_shebang(self):
"fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']",
])
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, raise_error=True)
toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')
@@ -3355,7 +3686,8 @@ def test_fix_shebang(self):
# now test with a custom env command
extra_args = ['--env-for-shebang=/usr/bin/env -S']
- self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')
@@ -3397,7 +3729,8 @@ def check_shebangs():
'--env-for-shebang=/usr/bin/env -S',
'--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it
]
- self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/bin/env -S python\n# test$')
regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/bin/env -S perl\n# test$')
@@ -3409,7 +3742,8 @@ def check_shebangs():
'--env-for-shebang=/usr/../usr/../usr/bin/env -S',
'--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it
]
- self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True)
regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S python\n# test$')
regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S perl\n# test$')
@@ -3435,7 +3769,8 @@ def test_toy_system_toolchain_alias(self):
test_ec_txt = tc_regex.sub(tc, toy_ec_txt)
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
def test_toy_ghost_installdir(self):
"""Test whether ghost installation directory is removed under --force."""
@@ -3459,7 +3794,6 @@ def test_toy_ghost_installdir(self):
stdout, stderr = self.run_test_toy_build_with_output()
# by default, a warning is printed for ghost installation directories (but they're left untouched)
- self.assertFalse(stdout)
regex = re.compile("WARNING: Likely ghost installation directory detected: %s" % toy_installdir)
self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
self.assertExists(toy_installdir)
@@ -3470,8 +3804,8 @@ def test_toy_ghost_installdir(self):
self.assertFalse(stderr)
- regex = re.compile("^== Ghost installation directory %s removed" % toy_installdir)
- self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ regex = re.compile("== Ghost installation directory %s removed" % toy_installdir)
+ self.assertRegex(stdout, regex)
self.assertNotExists(toy_installdir)
@@ -3486,29 +3820,34 @@ def test_toy_build_lock(self):
mkdir(toy_lock_path, parents=True)
error_pattern = "Lock .*_software_toy_0.0.lock already exists, aborting!"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, raise_error=True, verbose=False)
# lock should still be there after it was hit
self.assertExists(toy_lock_path)
# trying again should give same result
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, raise_error=True, verbose=False)
self.assertExists(toy_lock_path)
locks_dir = os.path.join(self.test_prefix, 'locks')
# no lock in place, so installation proceeds as normal
extra_args = ['--locks-dir=%s' % locks_dir]
- self.test_toy_build(extra_args=extra_args, verify=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=extra_args, verify=True, raise_error=True)
# put lock in place in custom locks dir, try again
toy_lock_path = os.path.join(locks_dir, toy_lock_fn)
mkdir(toy_lock_path, parents=True)
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build,
- extra_args=extra_args, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build,
+ extra_args=extra_args, raise_error=True, verbose=False)
# also test use of --ignore-locks
- self.test_toy_build(extra_args=extra_args + ['--ignore-locks'], verify=True, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=extra_args + ['--ignore-locks'], verify=True, raise_error=True)
orig_sigalrm_handler = signal.getsignal(signal.SIGALRM)
@@ -3531,28 +3870,17 @@ def __exit__(self, type, value, traceback):
signal.alarm(0)
# wait for lock to be removed, with 1 second interval of checking;
- # check with both --wait-on-lock-interval and deprecated --wait-on-lock options
wait_regex = re.compile("^== lock .*_software_toy_0.0.lock exists, waiting 1 seconds", re.M)
ok_regex = re.compile("^== COMPLETED: Installation ended successfully", re.M)
test_cases = [
- ['--wait-on-lock=1'],
- ['--wait-on-lock=1', '--wait-on-lock-interval=60'],
- ['--wait-on-lock=100', '--wait-on-lock-interval=1'],
- ['--wait-on-lock-limit=100', '--wait-on-lock=1'],
['--wait-on-lock-limit=100', '--wait-on-lock-interval=1'],
- ['--wait-on-lock-limit=-1', '--wait-on-lock=1'],
['--wait-on-lock-limit=-1', '--wait-on-lock-interval=1'],
]
for opts in test_cases:
- if any('--wait-on-lock=' in x for x in opts):
- self.allow_deprecated_behaviour()
- else:
- self.disallow_deprecated_behaviour()
-
if not os.path.exists(toy_lock_path):
mkdir(toy_lock_path)
@@ -3564,15 +3892,12 @@ def __exit__(self, type, value, traceback):
with RemoveLockAfter(3, toy_lock_path):
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False)
+ self._test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False)
stderr, stdout = self.get_stderr(), self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
- if any('--wait-on-lock=' in x for x in all_args):
- self.assertIn("Use of --wait-on-lock is deprecated", stderr)
- else:
- self.assertEqual(stderr, '')
+ self.assertEqual(stderr, '')
wait_matches = wait_regex.findall(stdout)
# we can't rely on an exact number of 'waiting' messages, so let's go with a range...
@@ -3586,7 +3911,7 @@ def __exit__(self, type, value, traceback):
self.mock_stderr(True)
self.mock_stdout(True)
error_pattern = r"Maximum wait time for lock /.*toy_0.0.lock to be released reached: [0-9]+ sec >= 3 sec"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=all_args,
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, extra_args=all_args,
verify=False, raise_error=True, testing=False)
stderr, stdout = self.get_stderr(), self.get_stdout()
self.mock_stderr(False)
@@ -3597,12 +3922,12 @@ def __exit__(self, type, value, traceback):
# when there is no lock in place, --wait-on-lock* has no impact
remove_dir(toy_lock_path)
- for opt in ['--wait-on-lock=1', '--wait-on-lock-limit=3', '--wait-on-lock-interval=1']:
+ for opt in ['--wait-on-lock-limit=3', '--wait-on-lock-interval=1']:
all_args = extra_args + [opt]
self.assertNotExists(toy_lock_path)
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False)
+ self._test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False)
stderr, stdout = self.get_stderr(), self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
@@ -3615,18 +3940,13 @@ def __exit__(self, type, value, traceback):
extra_args = ['--locks-dir=/']
error_pattern = r"Failed to create lock /.*_software_toy_0.0.lock:.* "
error_pattern += r"(Read-only file system|Permission denied)"
- self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build,
- extra_args=extra_args, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build,
+ extra_args=extra_args, raise_error=True, verbose=False)
def test_toy_lock_cleanup_signals(self):
"""Test cleanup of locks after EasyBuild session gets a cancellation signal."""
- # skip test when using Python 2, since it somehow fails then,
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333
- if sys.version_info[0] == 2:
- print("Skipping test_toy_lock_cleanup_signals because Python 2.x is being used")
- return
-
orig_wd = os.getcwd()
locks_dir = os.path.join(self.test_installpath, 'software', '.locks')
@@ -3657,7 +3977,9 @@ def __exit__(self, type, value, traceback):
toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'))
test_ec = os.path.join(self.test_prefix, 'test.eb')
- write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 5"]')
+ write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 10"]')
+
+ extra_args = ['--locks-dir=%s' % locks_dir, '--wait-on-lock-limit=3', '--wait-on-lock-interval=3']
signums = [
(signal.SIGABRT, SystemExit),
@@ -3670,15 +3992,15 @@ def __exit__(self, type, value, traceback):
# avoid recycling stderr of previous test
stderr = ''
- with WaitAndSignal(1, signum):
+ with WaitAndSignal(3, signum):
# change back to original working directory before each test
change_dir(orig_wd)
self.mock_stderr(True)
self.mock_stdout(True)
- self.assertErrorRegex(exc, '.*', self.test_toy_build, ec_file=test_ec, verify=False,
- raise_error=True, testing=False, raise_systemexit=True)
+ self.assertErrorRegex(exc, '.*', self._test_toy_build, ec_file=test_ec, verify=False,
+ extra_args=extra_args, raise_error=True, testing=False, raise_systemexit=True)
stderr = self.get_stderr().strip()
self.mock_stderr(False)
@@ -3707,7 +4029,8 @@ def test_toy_build_unicode_description(self):
test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, raise_error=True)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, raise_error=True)
def test_toy_build_lib64_lib_symlink(self):
"""Check whether lib64 symlink to lib subdirectory is created."""
@@ -3715,7 +4038,8 @@ def test_toy_build_lib64_lib_symlink(self):
# see https://github.com/easybuilders/easybuild-easyconfigs/issues/5776
# by default, lib64 -> lib symlink is created (--lib64-lib-symlink is enabled by default)
- self.test_toy_build()
+ with self.mocked_stdout_stderr():
+ self._test_toy_build()
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
lib_path = os.path.join(toy_installdir, 'lib')
@@ -3736,7 +4060,8 @@ def test_toy_build_lib64_lib_symlink(self):
# cleanup and try again with --disable-lib64-lib-symlink
remove_dir(self.test_installpath)
- self.test_toy_build(extra_args=['--disable-lib64-lib-symlink'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(extra_args=['--disable-lib64-lib-symlink'])
self.assertExists(lib_path)
self.assertNotExists(lib64_path)
@@ -3751,41 +4076,42 @@ def test_toy_build_lib_lib64_symlink(self):
toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
test_ec_txt = read_file(toy_ec)
- test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']"
test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, test_ec_txt)
# by default, lib -> lib64 symlink is created (--lib-lib64-symlink is enabled by default)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
lib_path = os.path.join(toy_installdir, 'lib')
lib64_path = os.path.join(toy_installdir, 'lib64')
- # lib64 subdir exists, is not a symlink
- self.assertExists(lib64_path)
- self.assertTrue(os.path.isdir(lib64_path))
- self.assertFalse(os.path.islink(lib64_path))
-
- # lib subdir is a symlink to lib64 subdir
+ # lib subdir exists, is not a symlink
self.assertExists(lib_path)
self.assertTrue(os.path.isdir(lib_path))
- self.assertTrue(os.path.islink(lib_path))
- self.assertTrue(os.path.samefile(lib_path, lib64_path))
+ self.assertFalse(os.path.islink(lib_path))
- # lib symlink should point to a relative path
- self.assertFalse(os.path.isabs(os.readlink(lib_path)))
+ # lib64 subdir is a symlink to lib subdir
+ self.assertExists(lib64_path)
+ self.assertTrue(os.path.isdir(lib64_path))
+ self.assertTrue(os.path.islink(lib64_path))
+ self.assertTrue(os.path.samefile(lib64_path, lib_path))
+
+ # lib64 symlink should point to a relative path
+ self.assertFalse(os.path.isabs(os.readlink(lib64_path)))
# cleanup and try again with --disable-lib-lib64-symlink
remove_dir(self.test_installpath)
- self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink'])
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'])
- self.assertExists(lib64_path)
- self.assertNotExists(lib_path)
- self.assertNotIn('lib', os.listdir(toy_installdir))
- self.assertTrue(os.path.isdir(lib64_path))
- self.assertFalse(os.path.islink(lib64_path))
+ self.assertExists(lib_path)
+ self.assertNotExists(lib64_path)
+ self.assertNotIn('lib64', os.listdir(toy_installdir))
+ self.assertTrue(os.path.isdir(lib_path))
+ self.assertFalse(os.path.islink(lib_path))
def test_toy_build_sanity_check_linked_libs(self):
"""Test sanity checks for banned/requires libraries."""
@@ -3805,53 +4131,62 @@ def test_toy_build_sanity_check_linked_libs(self):
error_msg = "Check for banned/required shared libraries failed for"
# default check is done via EB_libtoy easyblock, which specifies several banned/required libraries
- self.test_toy_build(ec_file=libtoy_ec, raise_error=True, verbose=False, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=libtoy_ec, raise_error=True, verbose=False, verify=False)
remove_file(libtoy_modfile_path)
# we can make the check fail by defining environment variables picked up by the EB_libtoy easyblock
os.environ['EB_LIBTOY_BANNED_SHARED_LIBS'] = 'libtoy'
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
del os.environ['EB_LIBTOY_BANNED_SHARED_LIBS']
os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS'] = 'thisisnottheremostlikely'
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
del os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS']
# make sure default check passes (so we know better what triggered a failing test)
- self.test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False,
- raise_error=True, verbose=False, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False,
+ raise_error=True, verbose=False, verify=False)
remove_file(libtoy_modfile_path)
# check specifying banned/required libraries via EasyBuild configuration option
args = ['--banned-linked-shared-libs=%s,foobarbaz' % libtoy_fn, '--module-only']
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False)
args = ['--required-linked-shared=libs=foobarbazisnotthereforsure', '--module-only']
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False)
# check specifying banned/required libraries via easyconfig parameter
test_ec_txt = read_file(libtoy_ec)
test_ec_txt += "\nbanned_linked_shared_libs = ['toy']"
write_file(test_ec, test_ec_txt)
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
test_ec_txt = read_file(libtoy_ec)
test_ec_txt += "\nrequired_linked_shared_libs = ['thereisnosuchlibraryyoudummy']"
write_file(test_ec, test_ec_txt)
- self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False,
- ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
+ ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
- # check behaviour when alternate subdirectories are specified
+ # check behaviour when alternative subdirectories are specified
test_ec_txt = read_file(libtoy_ec)
test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']"
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec, extra_args=['--module-only'], force=False,
- raise_error=True, verbose=False, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=['--module-only'], force=False,
+ raise_error=True, verbose=False, verify=False)
# one last time: supercombo (with patterns that should pass the check)
test_ec_txt = read_file(libtoy_ec)
@@ -3864,8 +4199,47 @@ def test_toy_build_sanity_check_linked_libs(self):
'--required-linked-shared-libs=.*',
'--module-only',
]
- self.test_toy_build(ec_file=test_ec, extra_args=args, force=False,
- raise_error=True, verbose=False, verify=False)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=args, force=False,
+ raise_error=True, verbose=False, verify=False)
+
+ def test_toy_mod_files(self):
+ """Check detection of .mod files"""
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
+
+ test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']"
+ write_file(test_ec, test_ec_txt)
+
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
+
+ args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains']
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ self._test_toy_build(ec_file=test_ec, extra_args=args)
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ pattern = r"WARNING: One or more \.mod files found in .*/software/toy/0.0-GCCcore-6.2.0: .*/lib64/file.mod"
+ self.assertRegex(stderr.strip(), pattern)
+
+ args += ['--fail-on-mod-files-gcccore']
+ pattern = r"Sanity check failed: One or more \.mod files found in .*/toy/0.0-GCCcore-6.2.0: .*/lib/file.mod"
+ self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec,
+ extra_args=args, verify=False, fails=True, verbose=False, raise_error=True)
+
+ test_ec_txt += "\nskip_mod_files_sanity_check = True"
+ write_file(test_ec, test_ec_txt)
+
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec, extra_args=args)
def test_toy_ignore_test_failure(self):
"""Check whether use of --ignore-test-failure is mentioned in build output."""
@@ -3887,7 +4261,8 @@ def test_toy_post_install_patches(self):
test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, test_ec_txt)
- self.test_toy_build(ec_file=test_ec)
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=test_ec)
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
toy_readme_txt = read_file(os.path.join(toy_installdir, 'README'))
@@ -3912,9 +4287,9 @@ def test_toy_unavailable_os_dep(self):
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, force=False, raise_error=False, verify=False,
- test_report_regexs=[r"One or more OS dependencies were not found"],
- test_report=test_report_fp)
+ self._test_toy_build(ec_file=test_ec, force=False, raise_error=False, verify=False,
+ test_report_regexs=[r"One or more OS dependencies were not found"],
+ test_report=test_report_fp)
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
@@ -3943,7 +4318,7 @@ def test_toy_post_install_messages(self):
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, test_report=test_report_fp)
+ self._test_toy_build(ec_file=test_ec, test_report=test_report_fp)
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
@@ -3969,7 +4344,7 @@ def test_toy_build_info_msg(self):
write_file(test_ec, test_ec_txt)
with self.mocked_stdout_stderr():
- self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True)
+ self._test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True)
stdout = self.get_stdout()
pattern = '\n'.join([
@@ -3980,6 +4355,186 @@ def test_toy_build_info_msg(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+ def test_toy_failing_test_step(self):
+ """
+ Test behaviour when test step fails, using toy easyconfig.
+ """
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += '\nruntest = "false"'
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+
+ error_pattern = r"shell command 'false \.\.\.' failed in test step"
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output,
+ ec_file=test_ec, raise_error=True)
+
+ def test_eb_crash(self):
+ """
+ Test behaviour when EasyBuild crashes, for example due to a buggy hook
+ """
+ hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
+ hooks_file_txt = textwrap.dedent("""
+ def pre_configure_hook(self, *args, **kwargs):
+ no_such_thing
+ """)
+ write_file(hooks_file, hooks_file_txt)
+
+ topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+ toy_eb = os.path.join(topdir, 'test', 'framework', 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py')
+ toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+ args = [
+ toy_ec,
+ f'--hooks={hooks_file}',
+ '--force',
+ f'--installpath={self.test_prefix}',
+ f'--include-easyblocks={toy_eb}',
+ ]
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ cleanup()
+ try:
+ main_with_hooks(args=args)
+ self.assertFalse("This should never be reached, main function should have crashed!")
+ except NameError as err:
+ self.assertEqual(str(err), "name 'no_such_thing' is not defined")
+
+ regex = re.compile(r"EasyBuild crashed! Please consider reporting a bug, this should not happen")
+ stderr = stderr.getvalue()
+ self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}")
+
+ def test_eb_error(self):
+ """
+ Test whether main function as run by 'eb' command print error messages to stderr.
+ """
+ topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+ toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "\ndependencies = [('nosuchdep', '1.0')]"
+ write_file(test_ec, test_ec_txt)
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ cleanup()
+ try:
+ main_with_hooks(args=[test_ec, '--robot', '--force'])
+ except SystemExit:
+ pass
+
+ regex = re.compile("^ERROR: Missing dependencies", re.M)
+ stderr = stderr.getvalue()
+ self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}")
+
+ def test_toy_python(self):
+ """
+ Test whether $PYTHONPATH or $EBPYTHONPREFIXES are set correctly.
+ """
+ # generate fake Python modules that we can use as runtime dependency for toy
+ # (required condition for use of $EBPYTHONPREFIXES)
+ fake_mods_path = os.path.join(self.test_prefix, 'modules')
+ for pyver in ('2.7', '3.6'):
+ fake_python_mod = os.path.join(fake_mods_path, 'Python', pyver)
+ if get_module_syntax() == 'Lua':
+ fake_python_mod += '.lua'
+ write_file(fake_python_mod, '')
+ else:
+ write_file(fake_python_mod, '#%Module')
+ self.modtool.use(fake_mods_path)
+
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "\npostinstallcmds.append('mkdir -p %(installdir)s/lib/python3.6/site-packages')"
+ test_ec_txt += "\npostinstallcmds.append('touch %(installdir)s/lib/python3.6/site-packages/foo.py')"
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+ self.run_test_toy_build_with_output(ec_file=test_ec)
+
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+ if get_module_syntax() == 'Lua':
+ toy_mod += '.lua'
+ toy_mod_txt = read_file(toy_mod)
+
+ pythonpath_regex = re.compile('^prepend.path.*PYTHONPATH.*lib/python3.6/site-packages', re.M)
+
+ self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+ f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+ # also check when opting in to use $EBPYTHONPREFIXES instead of $PYTHONPATH
+ args = ['--prefer-python-search-path=EBPYTHONPREFIXES']
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ toy_mod_txt = read_file(toy_mod)
+ # if Python is not listed as a runtime dependency then $PYTHONPATH is still used,
+ # because the Python dependency used must be aware of $EBPYTHONPREFIXES
+ # (see sitecustomize.py installed by Python easyblock)
+ self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+ f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+ # if Python is listed as runtime dependency, then $EBPYTHONPREFIXES is used if it's preferred
+ write_file(test_ec, test_ec_txt + "\ndependencies = [('Python', '3.6', '', SYSTEM)]")
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ toy_mod_txt = read_file(toy_mod)
+
+ ebpythonprefixes_regex = re.compile('^prepend.path.*EBPYTHONPREFIXES.*root', re.M)
+ self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+ f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
+ # if Python is listed in multi_deps, then $EBPYTHONPREFIXES is used, even if it's not explicitely preferred
+ write_file(test_ec, test_ec_txt + "\nmulti_deps = {'Python': ['2.7', '3.6']}")
+ self.run_test_toy_build_with_output(ec_file=test_ec)
+ toy_mod_txt = read_file(toy_mod)
+
+ self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+ f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
+ def test_toy_multiple_ecs_module(self):
+ """
+ Verify whether module file is correct when multiple easyconfigs are being installed.
+ """
+ test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+ # modify 'toy' easyconfig so toy-headers subdirectory is created,
+ # which is taken into account by EB_toy easyblock for $CPATH
+ test_toy_ec = os.path.join(self.test_prefix, 'test-toy.eb')
+ toy_ec_txt = read_file(toy_ec)
+ toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']"
+ toy_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/toy-headers/toy.h']"
+ write_file(test_toy_ec, toy_ec_txt)
+
+ # modify 'toy-app' easyconfig so toy-headers subdirectory is created,
+ # which is consider by EB_toy easyblock for $CPATH,
+ # but should *not* be actually used because software name is not 'toy'
+ toy_app_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb')
+ test_toy_app_ec = os.path.join(self.test_prefix, 'test-toy-app.eb')
+ toy_ec_txt = read_file(toy_app_ec)
+ toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']"
+ toy_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/toy-headers/toy.h']"
+ write_file(test_toy_app_ec, toy_ec_txt)
+
+ self.run_test_toy_build_with_output(ec_file=test_toy_ec, extra_args=[test_toy_app_ec], raise_error=True)
+
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+ if get_module_syntax() == 'Lua':
+ toy_mod += '.lua'
+ toy_modtxt = read_file(toy_mod)
+ regex = re.compile('prepend[-_]path.*CPATH.*toy-headers', re.M)
+ self.assertTrue(regex.search(toy_modtxt),
+ f"Pattern '{regex.pattern}' should be found in: {toy_modtxt}")
+
+ toy_app_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy-app', '0.0')
+ if get_module_syntax() == 'Lua':
+ toy_app_mod += '.lua'
+ toy_app_modtxt = read_file(toy_app_mod)
+ self.assertFalse(regex.search(toy_app_modtxt),
+ f"Pattern '{regex.pattern}' should *not* be found in: {toy_app_modtxt}")
+
def suite():
""" return all the tests in this file """
diff --git a/test/framework/tweak.py b/test/framework/tweak.py
index 24ec85b2c8..79d39ef440 100644
--- a/test/framework/tweak.py
+++ b/test/framework/tweak.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2024 Ghent University
+# Copyright 2014-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -510,7 +510,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self):
update_build_specs={'version': new_version},
update_dep_versions=False)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
- extensions = tweaked_ec['ec']['exts_list']
+ extensions = tweaked_ec['ec'].get_ref('exts_list')
# check one extension with the same name exists and that the version has been updated
hit_extension = 0
for extension in extensions:
diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py
index 8b7edd3215..a9ce5df15b 100644
--- a/test/framework/type_checking.py
+++ b/test/framework/type_checking.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2024 Ghent University
+# Copyright 2015-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -41,7 +41,6 @@
from easybuild.framework.easyconfig.types import to_list_of_strings_and_tuples_and_dicts
from easybuild.framework.easyconfig.types import to_sanity_check_paths_dict, to_toolchain_dict
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.py2vs3 import string_type
class TypeCheckingTest(EnhancedTestCase):
@@ -172,16 +171,18 @@ def test_check_type_of_param_value_sanity_check_paths(self):
out = {'files': ['bin/foo', ('bin/bar', 'bin/baz')], 'dirs': [('lib', 'lib64', 'lib32')]}
self.assertEqual(check_type_of_param_value('sanity_check_paths', inp, auto_convert=True), (True, out))
- def test_check_type_of_param_value_checksums(self):
- """Test check_type_of_param_value function for checksums."""
-
- md5_checksum = 'fa618be8435447a017fd1bf2c7ae9224'
- sha256_checksum1 = 'fa618be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265'
- sha256_checksum2 = 'b5f9cb06105c1d2d30719db5ffb3ea67da60919fb68deaefa583deccd8813551'
- sha256_checksum3 = '033be54514a03e255df75c5aee8f9e672f663f93abb723444caec8fe43437bde'
+ @staticmethod
+ def get_valid_checksums_values():
+ """Return list of values valid for the 'checksums' EC parameter"""
+ # Using (actually invalid) prefix to better detect those in case of errors
+ md5_checksum = 'md518be8435447a017fd1bf2c7ae9224'
+ sha256_checksum1 = 'sha18be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265'
+ sha256_checksum2 = 'sha2cb06105c1d2d30719db5ffb3ea67da60919fb68deaefa583deccd8813551'
+ sha256_checksum3 = 'sha3e54514a03e255df75c5aee8f9e672f663f93abb723444caec8fe43437bde'
+ filesize = 45617379
# valid values for 'checksums' easyconfig parameters
- inputs = [
+ return [
[],
# single checksum (one file)
[md5_checksum],
@@ -191,6 +192,7 @@ def test_check_type_of_param_value_checksums(self):
# one checksum of specific type (as 2-tuple)
[('md5', md5_checksum)],
[('sha256', sha256_checksum1)],
+ [('size', filesize)],
# alternative checksums for a single file (n-tuple)
[(sha256_checksum1, sha256_checksum2)],
[(sha256_checksum1, sha256_checksum2, sha256_checksum3)],
@@ -214,17 +216,37 @@ def test_check_type_of_param_value_checksums(self):
# two checksums for a single file, *both* should match
[sha256_checksum1, md5_checksum],
# three checksums for a single file, *all* should match
- [sha256_checksum1, ('md5', md5_checksum), {'foo.txt': sha256_checksum1}],
+ [sha256_checksum1, ('md5', md5_checksum), ('size', filesize)],
# single checksum for a single file
sha256_checksum1,
# filename-to-checksum mapping
- {'foo.txt': sha256_checksum1, 'bar.txt': sha256_checksum2},
+ {'foo.txt': sha256_checksum1, 'bar.txt': sha256_checksum2, 'baz.txt': ('size', filesize)},
# 3 alternative checksums for a single file, one match is sufficient
(sha256_checksum1, sha256_checksum2, sha256_checksum3),
- ]
+ # two alternative checksums for a single file (not to be confused by checksum-type & -value tuple)
+ (sha256_checksum1, md5_checksum),
+ # three alternative checksums for a single file of different types
+ (sha256_checksum1, ('md5', md5_checksum), ('size', filesize)),
+ # alternative checksums in dicts are also allowed
+ {'foo.txt': (sha256_checksum2, sha256_checksum3), 'bar.txt': (sha256_checksum1, md5_checksum)},
+ # Same but with lists -> all must match for each file
+ {'foo.txt': [sha256_checksum2, sha256_checksum3], 'bar.txt': [sha256_checksum1, md5_checksum]},
+ ],
+ # None is allowed, meaning skip the checksum
+ [
+ None,
+ # Also in mappings
+ {'foo.txt': sha256_checksum1, 'bar.txt': None},
+ ],
]
- for inp in inputs:
- self.assertEqual(check_type_of_param_value('checksums', inp), (True, inp))
+
+ def test_check_type_of_param_value_checksums(self):
+ """Test check_type_of_param_value function for checksums."""
+
+ for inp in TypeCheckingTest.get_valid_checksums_values():
+ type_ok, newval = check_type_of_param_value('checksums', inp)
+ self.assertIs(type_ok, True, 'Failed for ' + str(inp))
+ self.assertEqual(newval, inp)
def test_check_type_of_param_value_patches(self):
"""Test check_type_of_param_value function for patches."""
@@ -248,9 +270,9 @@ def test_check_type_of_param_value_patches(self):
def test_convert_value_type(self):
"""Test convert_value_type function."""
# to string
- self.assertEqual(convert_value_type(100, string_type), '100')
+ self.assertEqual(convert_value_type(100, str), '100')
self.assertEqual(convert_value_type((100,), str), '(100,)')
- self.assertEqual(convert_value_type([100], string_type), '[100]')
+ self.assertEqual(convert_value_type([100], str), '[100]')
self.assertEqual(convert_value_type(None, str), 'None')
# to int/float
@@ -269,7 +291,7 @@ def test_convert_value_type(self):
self.assertEqual(convert_value_type((), LIST_OF_STRINGS), [])
# idempotency
- self.assertEqual(convert_value_type('foo', string_type), 'foo')
+ self.assertEqual(convert_value_type('foo', str), 'foo')
self.assertEqual(convert_value_type('foo', str), 'foo')
self.assertEqual(convert_value_type(100, int), 100)
self.assertEqual(convert_value_type(1.6, float), 1.6)
@@ -318,9 +340,9 @@ def test_to_toolchain_dict(self):
self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', 'False', '7'])
# invalid truth value
- errstr = "invalid truth value .*"
- self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, "intel, 2015, foo")
- self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, ['gcc', '4', '7'])
+ errstr = "Invalid truth value .*"
+ self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel, 2015, foo")
+ self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', '7'])
# missing keys
self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_toolchain_dict, {'name': 'intel'})
@@ -706,19 +728,94 @@ def test_to_sanity_check_paths_dict(self):
def test_to_checksums(self):
"""Test to_checksums function."""
+ # Some hand-crafted examples. Only the types are important, values are for easier verification
test_inputs = [
- ['be662daa971a640e40be5c804d9d7d10'],
- ['be662daa971a640e40be5c804d9d7d10', ('md5', 'be662daa971a640e40be5c804d9d7d10')],
- [['be662daa971a640e40be5c804d9d7d10', ('md5', 'be662daa971a640e40be5c804d9d7d10')]],
- [('md5', 'be662daa971a640e40be5c804d9d7d10')],
- ['be662daa971a640e40be5c804d9d7d10', ('adler32', '0x998410035'), ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'), ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273)],
+ ['checksumvalue'],
+ [('md5', 'md5checksumvalue')],
+ ['file_1_checksum', ('md5', 'file_2_md5_checksum')],
+ # One checksum per file, some with checksum type
+ [
+ 'be662daa971a640e40be5c804d9d7d10',
+ ('adler32', '0x998410035'),
+ ('crc32', '0x1553842328'),
+ ('md5', 'be662daa971a640e40be5c804d9d7d10'),
+ ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
+ # int type as the 2nd value
+ ('size', 273),
+ ],
# None values should not be filtered out, but left in place
- [None, 'fa618be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265', None],
+ [None, 'checksum', None],
+ # Alternative checksums, not to be confused with multiple checksums for a file
+ [('main_checksum', 'alternative_checksum')],
+ [('1st_of_3', '2nd_of_3', '3rd_of_3')],
+ # Lists must be kept: This means all must match
+ [['checksum_1_in_list']],
+ [['checksum_must_match', 'this_must_also_match']],
+ [['1st_of_3_list', '2nd_of_3_list', '3rd_of_3_list']],
+ # Alternative checksums with types
+ [
+ (('adler32', '1st_adler'), ('crc32', '1st_crc')),
+ (('adler32', '2nd_adler'), ('crc32', '2nd_crc'), ('sha1', '2nd_sha')),
+ ],
+ # Entries can be dicts even containing `None`
+ [
+ {
+ 'src-arm.tgz': 'arm_checksum',
+ 'src-x86.tgz': ('mainchecksum', 'altchecksum'),
+ 'src-ppc.tgz': ('mainchecksum', ('md5', 'altchecksum')),
+ 'git-clone.tgz': None,
+ },
+ {
+ 'src': ['checksum_must_match', 'this_must_also_match']
+ },
+ # 2nd required checksum a dict
+ ['first_checksum', {'src-arm': 'arm_checksum'}]
+ ],
]
for checksums in test_inputs:
self.assertEqual(to_checksums(checksums), checksums)
+ # Also reuse the checksums we use in test_check_type_of_param_value_checksums
+ # When a checksum is valid it must not be modified
+ for checksums in TypeCheckingTest.get_valid_checksums_values():
+ self.assertEqual(to_checksums(checksums), checksums)
+
+ # List in list converted to tuple -> alternatives or checksum with type
+ checksums = [['1stchecksum', ['md5', 'md5sum']]]
+ checksums_expected = [['1stchecksum', ('md5', 'md5sum')]]
+ self.assertEqual(to_checksums(checksums), checksums_expected)
+
+ # Error detection
+ wrong_nesting = [('1stchecksum', ('md5', ('md5sum', 'altmd5sum')))]
+ self.assertErrorRegex(EasyBuildError, 'Unexpected type.*md5', to_checksums, wrong_nesting)
+ correct_nesting = [('1stchecksum', ('md5', 'md5sum'), ('md5', 'altmd5sum'))]
+ self.assertEqual(to_checksums(correct_nesting), correct_nesting)
+ # YEB (YAML EC) doesn't has tuples so it uses lists instead which need to get converted
+ correct_nesting_yeb = [[['1stchecksum', ['md5', 'md5sum'], ['md5', 'altmd5sum']]]]
+ correct_nesting_yeb_conv = [[('1stchecksum', ('md5', 'md5sum'), ('md5', 'altmd5sum'))]]
+ self.assertEqual(to_checksums(correct_nesting_yeb), correct_nesting_yeb_conv)
+ self.assertEqual(to_checksums(correct_nesting_yeb_conv), correct_nesting_yeb_conv)
+
+ unexpected_set = [('1stchecksum', {'md5', 'md5sum'})]
+ self.assertErrorRegex(EasyBuildError, 'Unexpected type.*md5', to_checksums, unexpected_set)
+ unexpected_dict = [{'src': ('md5sum', {'src': 'shasum'})}]
+ self.assertErrorRegex(EasyBuildError, 'Unexpected type.*shasum', to_checksums, unexpected_dict)
+ correct_dict = [{'src': ('md5sum', 'shasum')}]
+ self.assertEqual(to_checksums(correct_dict), correct_dict)
+ correct_dict_1 = [{'src': [['md5', 'md5sum'], ['sha', 'shasum']]}]
+ correct_dict_2 = [{'src': [('md5', 'md5sum'), ('sha', 'shasum')]}]
+ self.assertEqual(to_checksums(correct_dict_2), correct_dict_2)
+ self.assertEqual(to_checksums(correct_dict_1), correct_dict_2) # inner lists to tuples
+
+ unexpected_Nones = [
+ [('1stchecksum', None)],
+ [['1stchecksum', None]],
+ [{'src': ('md5sum', None)}],
+ [{'src': ['md5sum', None]}],
+ ]
+ self.assertErrorRegex(EasyBuildError, 'Unexpected None', to_checksums, unexpected_Nones[0])
+ self.assertErrorRegex(EasyBuildError, 'Unexpected None', to_checksums, unexpected_Nones[1])
+ self.assertErrorRegex(EasyBuildError, 'Unexpected None', to_checksums, unexpected_Nones[2])
+ self.assertErrorRegex(EasyBuildError, 'Unexpected None', to_checksums, unexpected_Nones[3])
def test_ensure_iterable_license_specs(self):
"""Test ensure_iterable_license_specs function."""
diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index d07c2202b1..27ba444d96 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,6 +37,7 @@
import tempfile
import unittest
from contextlib import contextmanager
+from importlib import reload
from easybuild.base import fancylogger
from easybuild.base.testing import TestCase
@@ -48,13 +49,12 @@
from easybuild.framework.easyblock import EasyBlock
from easybuild.main import main
from easybuild.tools import config
-from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes, update_build_option
+from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes
from easybuild.tools.configobj import ConfigObj
from easybuild.tools.environment import modify_env
from easybuild.tools.filetools import copy_dir, mkdir, read_file, which
from easybuild.tools.modules import curr_module_paths, modules_tool, reset_module_caches
from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir
-from easybuild.tools.py2vs3 import reload
# make sure tests are robust against any non-default configuration settings;
@@ -106,8 +106,8 @@ def setUp(self):
os.close(fd)
self.cwd = os.getcwd()
- # keep track of original environment to restore
- self.orig_environ = copy.deepcopy(os.environ)
+ # keep track of original environment to restore after tests
+ self._initial_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
@@ -131,17 +131,19 @@ def setUp(self):
if eb_path is not None:
os.environ['EB_SCRIPT_PATH'] = eb_path
+ # disable progress bars when running the tests,
+ # since it messes with test suite progress output when test installations are performed
+ os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1'
+
+ # Store the environment as setup (including the above paths) for tests to restore
+ self.orig_environ = copy.deepcopy(os.environ)
+
# make sure no deprecated behaviour is being triggered (unless intended by the test)
self.orig_current_version = eb_build_log.CURRENT_VERSION
self.disallow_deprecated_behaviour()
init_config()
- # disable progress bars when running the tests,
- # since it messes with test suite progress output when test installations are performed
- os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1'
- update_build_option('show_progress_bar', False)
-
import easybuild
# try to import easybuild.easyblocks(.generic) packages
# it's OK if it fails here, but important to import first before fiddling with sys.path
@@ -210,6 +212,14 @@ def allow_deprecated_behaviour(self):
os.environ.pop('EASYBUILD_DEPRECATED', None)
eb_build_log.CURRENT_VERSION = self.orig_current_version
+ @contextmanager
+ def temporarily_allow_deprecated_behaviour(self):
+ self.allow_deprecated_behaviour()
+ try:
+ yield
+ finally:
+ self.disallow_deprecated_behaviour()
+
@contextmanager
def log_to_testlogfile(self):
"""Context manager class to capture log output in self.logfile for the scope used. Clears the file first"""
@@ -399,7 +409,7 @@ def setup_categorized_hmns_modules(self):
src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'modules', 'CategorizedHMNS', mod_subdir)
copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir))
- # create empty module file directory to make C/Tcl modules happy
+ # create empty module file directory to make Environment Modules <5.0 happy
mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2')
mkdir(os.path.join(mpi_pref, 'base'))
@@ -430,7 +440,8 @@ def loadTestsFromTestCase(self, test_case_class, filters):
retained_test_names = []
if len(filters) > 0:
for test_case_name in test_case_names:
- if any(filt in test_case_name for filt in filters):
+ full_test_case_name = '%s.%s' % (test_case_class.__name__, test_case_name)
+ if any(filt in full_test_case_name for filt in filters):
retained_test_names.append(test_case_name)
retained_tests = ', '.join(retained_test_names)
diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py
index a17d7beb02..ee36a0bdff 100644
--- a/test/framework/utilities_test.py
+++ b/test/framework/utilities_test.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,10 +36,10 @@
from datetime import datetime
from unittest import TextTestRunner
+import easybuild.tools.utilities as tu
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
from easybuild.tools import LooseVersion
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.utilities import time2str, natural_keys
class UtilitiesTest(EnhancedTestCase):
@@ -77,10 +77,10 @@ def test_time2str(self):
(datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"),
]
for end, expected in test_cases:
- self.assertEqual(time2str(end - start), expected)
+ self.assertEqual(tu.time2str(end - start), expected)
error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>"
- self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123)
+ self.assertErrorRegex(EasyBuildError, error_pattern, tu.time2str, 123)
def test_natural_keys(self):
"""Test the natural_keys function"""
@@ -98,7 +98,7 @@ def test_natural_keys(self):
]
shuffled_items = sorted_items[:]
random.shuffle(shuffled_items)
- shuffled_items.sort(key=natural_keys)
+ shuffled_items.sort(key=tu.natural_keys)
self.assertEqual(shuffled_items, sorted_items)
def test_LooseVersion(self):
@@ -140,11 +140,22 @@ def test_LooseVersion(self):
self.assertLess(LooseVersion('2.1.5'), LooseVersion('2.2'))
self.assertLess(LooseVersion('2.1.3'), LooseVersion('3'))
self.assertLessEqual(LooseVersion('2.1.0'), LooseVersion('2.2'))
- # Careful here: 1.0 > 1 !!!
- self.assertGreater(LooseVersion('1.0'), LooseVersion('1'))
- self.assertLess(LooseVersion('1'), LooseVersion('1.0'))
-
- # The following test is taken from Python distutils tests
+ # Missing components are either empty strings or zeroes
+ self.assertEqual(LooseVersion('1.0'), LooseVersion('1'))
+ self.assertEqual(LooseVersion('1'), LooseVersion('1.0'))
+ self.assertEqual(LooseVersion('1.0'), LooseVersion('1.'))
+ self.assertGreater(LooseVersion('2.1.a'), LooseVersion('2.1'))
+ self.assertGreater(LooseVersion('2.a'), LooseVersion('2'))
+
+ # checking prereleases
+ version_4beta = LooseVersion('4.0.0-beta')
+ self.assertGreater(version_4beta, LooseVersion('4.0.0'))
+ self.assertTrue(version_4beta.is_prerelease('4.0.0', ['-beta']))
+ self.assertTrue(version_4beta.is_prerelease(LooseVersion('4.0.0'), ['-beta']))
+ self.assertFalse(version_4beta.is_prerelease('4.0.0', ['rc']))
+ self.assertFalse(version_4beta.is_prerelease('4.0.0', ['rc, -beta']))
+
+ # The following test is based on the Python distutils tests
# licensed under the Python Software Foundation License Version 2
versions = (('1.5.1', '1.5.2b2', -1),
('161', '3.10a', 1),
@@ -154,16 +165,21 @@ def test_LooseVersion(self):
('2g6', '11g', -1),
('0.960923', '2.2beta29', -1),
('1.13++', '5.5.kw', -1),
- # Added from https://bugs.python.org/issue14894
('a.12.b.c', 'a.b.3', -1),
- ('1.0', '1', 1),
- ('1', '1.0', -1))
+ ('1.0', '1', 0),
+ ('1.a', '1', 1),
+ )
for v1, v2, wanted in versions:
res = LooseVersion(v1)._cmp(LooseVersion(v2))
self.assertEqual(res, wanted,
'cmp(%s, %s) should be %s, got %s' %
(v1, v2, wanted, res))
+ # Test the inverse
+ res = LooseVersion(v2)._cmp(LooseVersion(v1))
+ self.assertEqual(res, -wanted,
+ 'cmp(%s, %s) should be %s, got %s' %
+ (v2, v1, -wanted, res))
# vstring is the unparsed version
self.assertEqual(LooseVersion(v1).vstring, v1)
@@ -187,6 +203,26 @@ def test_LooseVersion(self):
self.assertEqual(LooseVersion('2.a').version, [2, 'a'])
self.assertEqual(LooseVersion('2.a5').version, [2, 'a', 5])
+ def test_unique_ordered_extend(self):
+ """Test unique_ordered_list_append method"""
+ base = ["potato", "tomato", "orange"]
+ base_orig = base.copy()
+
+ reference = ["potato", "tomato", "orange", "apple"]
+ self.assertEqual(tu.unique_ordered_extend(base, ["apple"]), reference)
+ self.assertEqual(tu.unique_ordered_extend(base, ["apple", "apple"]), reference)
+ self.assertNotEqual(tu.unique_ordered_extend(base, ["apple"]), sorted(reference))
+ # original list should not be modified
+ self.assertEqual(base, base_orig)
+
+ error_pattern = "given affix list is a string"
+ self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, "apple")
+ error_pattern = "given affix list is not iterable"
+ self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, 0)
+ base = "potato"
+ error_pattern = "given base cannot be extended"
+ self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference)
+
def suite():
""" return all the tests in this file """
diff --git a/test/framework/variables.py b/test/framework/variables.py
index cc804d6a65..0ff7428e94 100644
--- a/test/framework/variables.py
+++ b/test/framework/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2024 Ghent University
+# Copyright 2012-2025 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/yeb.py b/test/framework/yeb.py
deleted file mode 100644
index 346ffa057b..0000000000
--- a/test/framework/yeb.py
+++ /dev/null
@@ -1,214 +0,0 @@
-# #
-# Copyright 2015-2024 Ghent University
-#
-# This file is part of EasyBuild,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/easybuilders/easybuild
-#
-# EasyBuild is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation v2.
-#
-# EasyBuild is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with EasyBuild. If not, see .
-# #
-"""
-Unit tests for .yeb easyconfig format
-
-@author: Caroline De Brouwer (Ghent University)
-@author: Kenneth Hoste (Ghent University)
-"""
-import glob
-import os
-import platform
-import sys
-from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
-from unittest import TextTestRunner
-
-import easybuild.tools.build_log
-from easybuild.framework.easyconfig.easyconfig import EasyConfig
-from easybuild.framework.easyconfig.format.yeb import is_yeb_format
-from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import module_classes
-from easybuild.tools.filetools import read_file
-
-
-try:
- import yaml
-except ImportError:
- pass
-
-
-class YebTest(EnhancedTestCase):
- """ Testcase for run module """
-
- def setUp(self):
- """Test setup."""
- super(YebTest, self).setUp()
- self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
- easybuild.tools.build_log.EXPERIMENTAL = True
-
- def tearDown(self):
- """Test cleanup."""
- super(YebTest, self).tearDown()
- easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental
-
- def test_parse_yeb(self):
- """Test parsing of .yeb easyconfigs."""
- if 'yaml' not in sys.modules:
- print("Skipping test_parse_yeb (no PyYAML available)")
- return
-
- build_options = {
- 'check_osdeps': False,
- 'external_modules_metadata': {},
- 'silent': True,
- 'valid_module_classes': module_classes(),
- }
- init_config(build_options=build_options)
- easybuild.tools.build_log.EXPERIMENTAL = True
-
- testdir = os.path.dirname(os.path.abspath(__file__))
- test_easyconfigs = os.path.join(testdir, 'easyconfigs')
- test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
-
- # test parsing
- test_files = [
- 'bzip2-1.0.6-GCC-4.9.2',
- 'gzip-1.6-GCC-4.9.2',
- 'foss-2018a',
- 'intel-2018a',
- 'SQLite-3.8.10.2-foss-2018a',
- 'Python-2.7.10-intel-2018a',
- 'CrayCCE-5.1.29',
- 'toy-0.0',
- ]
-
- for filename in test_files:
- ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
- # compare with parsed result of .eb easyconfig
- ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0]
- ec_eb = EasyConfig(ec_file)
-
- for key in sorted(ec_yeb.asdict()):
- eb_val = ec_eb[key]
- yeb_val = ec_yeb[key]
- if key == 'description':
- # multi-line string is always terminated with '\n' in YAML, so strip it off
- yeb_val = yeb_val.strip()
-
- self.assertEqual(yeb_val, eb_val)
-
- def test_yeb_get_config_obj(self):
- """Test get_config_dict method."""
- testdir = os.path.dirname(os.path.abspath(__file__))
- test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
- ec = EasyConfig(os.path.join(test_yeb_easyconfigs, 'toy-0.0.yeb'))
- ecdict = ec.parser.get_config_dict()
-
- # changes to this dict should not affect the return value of the next call to get_config_dict
- fn = 'test.tar.gz'
- ecdict['sources'].append(fn)
-
- ecdict_bis = ec.parser.get_config_dict()
- self.assertIn(fn, ecdict['sources'])
- self.assertNotIn(fn, ecdict_bis['sources'])
-
- def test_is_yeb_format(self):
- """ Test is_yeb_format function """
- testdir = os.path.dirname(os.path.abspath(__file__))
- test_yeb = os.path.join(testdir, 'easyconfigs', 'yeb', 'bzip2-1.0.6-GCC-4.9.2.yeb')
- raw_yeb = read_file(test_yeb)
-
- self.assertTrue(is_yeb_format(test_yeb, None))
- self.assertTrue(is_yeb_format(None, raw_yeb))
-
- test_eb = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
- raw_eb = read_file(test_eb)
-
- self.assertFalse(is_yeb_format(test_eb, None))
- self.assertFalse(is_yeb_format(None, raw_eb))
-
- def test_join(self):
- """ Test yaml_join function """
- # skip test if yaml module was not loaded
- if 'yaml' not in sys.modules:
- print("Skipping test_join (no PyYAML available)")
- return
-
- stream = [
- "variables:",
- " - &f foo",
- " - &b bar",
- "",
- "fb1: !join [foo, bar]",
- "fb2: !join [*f, bar]",
- "fb3: !join [*f, *b]",
- ]
-
- # import here for testing yaml_join separately
- from easybuild.framework.easyconfig.format.yeb import yaml_join # noqa
- if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'):
- loaded = yaml.load('\n'.join(stream))
- else:
- loaded = yaml.load(u'\n'.join(stream), Loader=yaml.SafeLoader)
- for key in ['fb1', 'fb2', 'fb3']:
- self.assertEqual(loaded.get(key), 'foobar')
-
- def test_bad_toolchain_format(self):
- """ Test alternate toolchain format name,version """
- if 'yaml' not in sys.modules:
- print("Skipping test_parse_yeb (no PyYAML available)")
- return
-
- # only test bad cases - the right ones are tested with the test files in test_parse_yeb
- testdir = os.path.dirname(os.path.abspath(__file__))
- test_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
- expected = r'Can not convert list .* to toolchain dict. Expected 2 or 3 elements'
- self.assertErrorRegex(EasyBuildError, expected, EasyConfig,
- os.path.join(test_easyconfigs, 'bzip-bad-toolchain.yeb'))
-
- def test_external_module_toolchain(self):
- """Test specifying external (build) dependencies in yaml format."""
- if 'yaml' not in sys.modules:
- print("Skipping test_external_module_toolchain (no PyYAML available)")
- return
-
- ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb')
- metadata = {
- 'name': ['foo', 'bar'],
- 'version': ['1.2.3', '3.2.1'],
- 'prefix': '/foo/bar',
- }
- build_options = {
- 'external_modules_metadata': {'fftw/3.3.4.0': metadata},
- 'valid_module_classes': module_classes(),
- }
- init_config(build_options=build_options)
- easybuild.tools.build_log.EXPERIMENTAL = True
-
- ec = EasyConfig(ecpath)
-
- self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0')
- self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
-
-
-def suite():
- """ returns all the testcases in this module """
- return TestLoaderFiltered().loadTestsFromTestCase(YebTest, sys.argv[1:])
-
-
-if __name__ == '__main__':
- res = TextTestRunner(verbosity=1).run(suite())
- sys.exit(len(res.failures))