From 25bcd57bf55637801c4a17369b3042cd0eb020d8 Mon Sep 17 00:00:00 2001 From: Leah Date: Sun, 28 Jul 2024 23:43:49 +0200 Subject: [PATCH 1/5] adding easyblocks: condalock.py --- easybuild/easyblocks/generic/condalock.py | 130 ++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 easybuild/easyblocks/generic/condalock.py diff --git a/easybuild/easyblocks/generic/condalock.py b/easybuild/easyblocks/generic/condalock.py new file mode 100644 index 00000000000..ca1eeca1ad5 --- /dev/null +++ b/easybuild/easyblocks/generic/condalock.py @@ -0,0 +1,130 @@ +## +# Copyright 2009-2024 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing software using 'conda', implemented as an easyblock. + +@author: Jillian Rowe (New York University Abu Dhabi) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os + +from easybuild.easyblocks.generic.binary import Binary +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.run import run_cmd +from easybuild.tools.modules import get_software_root +from easybuild.tools.build_log import EasyBuildError + + +class CondaLock(Binary): + """Support for installing software using 'conda'.""" + + @staticmethod + def extra_options(extra_vars=None): + """Extra easyconfig parameters specific to Conda easyblock.""" + extra_vars = Binary.extra_options(extra_vars) + extra_vars.update({ + 'channels': [None, "List of conda channels to pass to 'conda install'", CUSTOM], + 'environment_file': [None, "Conda environment.yml file to use with 'conda env create'", CUSTOM], + 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], + 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], + }) + return extra_vars + + def extract_step(self): + """Copy sources via extract_step of parent, if any are specified.""" + if self.src: + super(CondaLock, self).extract_step() + + def install_step(self): + """Install software using 'conda env create' or 'conda create' & 'conda install' + (or the 'mamba', etc., equivalent).""" + if (get_software_root('anaconda2') or get_software_root('miniconda2') or + get_software_root('anaconda3') or get_software_root('miniconda3')): + conda_cmd = 'conda' + elif get_software_root('miniforge3'): + conda_cmd = 'conda-lock' + elif get_software_root('mamba'): + conda_cmd = 'mamba' + elif get_software_root('micromamba'): + conda_cmd = 'micromamba' + else: + raise EasyBuildError("No conda/mamba/micromamba available.") + + # initialize conda environment + # install conda-lock package + cmd = "pip install conda-lock" + run_cmd(cmd, log_all=True, simple=True) + + if self.cfg['environment_file'] or self.cfg['remote_environment']: + + if self.cfg['environment_file']: + env_spec = self.cfg['environment_file'] + else: + env_spec = self.cfg['remote_environment'] + + # use --force to ignore existing installation directory + cmd = "%s %s install %s --prefix %s" % (self.cfg['preinstallopts'], conda_cmd, + env_spec, self.installdir) + run_cmd(cmd, log_all=True, simple=True) + + else: + + if self.cfg['requirements']: + + install_args = "-y %s " % self.cfg['requirements'] + if self.cfg['channels']: + install_args += ' '.join('-c ' + chan for chan in self.cfg['channels']) + + self.log.info("Installed conda requirements") + + cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, + self.installdir, install_args) + run_cmd(cmd, log_all=True, simple=True) + + # clean up + # cmd = "%s clean -ya" % conda_cmd + #run_cmd(cmd, log_all=True, simple=True) + + def make_module_extra(self): + """Add the install directory to the PATH.""" + txt = super(CondaLock, self).make_module_extra() + txt += self.module_generator.set_environment('CONDA_ENV', self.installdir) + txt += self.module_generator.set_environment('CONDA_PREFIX', self.installdir) + txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir) + self.log.debug("make_module_extra added this: %s", txt) + return txt + + def make_module_req_guess(self): + """ + A dictionary of possible directories to look for. + """ + # LD_LIBRARY_PATH issue discusses here + # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl + return { + 'PATH': ['bin', 'sbin'], + 'MANPATH': ['man', os.path.join('share', 'man')], + 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], + } From 26851855cb08410eb8b73fee3992e86dece356c7 Mon Sep 17 00:00:00 2001 From: LeahHan Date: Mon, 5 Aug 2024 18:46:21 +1200 Subject: [PATCH 2/5] add conda-lock support to existing conda easyblock --- easybuild/easyblocks/generic/condaenhance.py | 184 +++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 easybuild/easyblocks/generic/condaenhance.py diff --git a/easybuild/easyblocks/generic/condaenhance.py b/easybuild/easyblocks/generic/condaenhance.py new file mode 100755 index 00000000000..03bb61adc96 --- /dev/null +++ b/easybuild/easyblocks/generic/condaenhance.py @@ -0,0 +1,184 @@ +## +# Copyright 2009-2024 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing software using 'conda', implemented as an easyblock. + +@author: Jillian Rowe (New York University Abu Dhabi) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os +import platform + +from easybuild.easyblocks.generic.binary import Binary +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.run import run_cmd +from easybuild.tools.modules import get_software_root +from easybuild.tools.build_log import EasyBuildError + + +class CondaEnhance(Binary): + """Support for installing software using 'conda'.""" + + @staticmethod + def extra_options(extra_vars=None): + """Extra easyconfig parameters specific to Conda easyblock.""" + extra_vars = Binary.extra_options(extra_vars) + extra_vars.update({ + 'channels': [None, "List of conda channels to pass to 'conda install'", CUSTOM], + 'environment_file': [None, "Conda environment.yml file to use with 'conda env create'", CUSTOM], + 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], + 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], + 'use_conda_lock': [False, "Whether to use conda-lock for reproducibility", CUSTOM], + 'lock_file': [None, "Path to a pre-generated conda-lock file", CUSTOM], + 'platform': [None, "Platform specified for conda-lock", CUSTOM], + }) + return extra_vars + + def extract_step(self): + """Copy sources via extract_step of parent, if any are specified.""" + if self.src: + super(CondaEnhance, self).extract_step() + + def install_step(self): + """Install software using 'conda env create' or 'conda create' & 'conda install' + (or the 'mamba', etc., equivalent).""" + if (get_software_root('anaconda2') or get_software_root('miniconda2') or + get_software_root('anaconda3') or get_software_root('miniconda3') or get_software_root('miniforge3')): + conda_cmd = 'conda' + # add independent entry for condaEnhance and miniforge (done) + elif get_software_root('mamba'): + conda_cmd = 'mamba' + elif get_software_root('micromamba'): + conda_cmd = 'micromamba' + else: + raise EasyBuildError("No conda/mamba/micromamba available.") + + def get_conda_lock_platform(): + system = platform.system().lower() + machine = platform.machine().lower() + + platforms = { + "linux": { + "x86_64": "linux-64", + "aarch64": "linux-aarch64", + "ppc64le": "linux-ppc64le" + }, + "darwin": { + "x86_64": "osx-64", + "arm64": "osx-arm64" + }, + "windows": { + "x86_64": "win-64", + "amd64": "win-64" + } + } + + return platforms.get(system, {}).get(machine) + + # initialize conda environment + if self.cfg['use_conda_lock']: + # install conda-lock + cmd = "%s install conda-lock" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) + + lock_file = self.cfg['lock_file'] + + if not lock_file: + # generate lock file + platform_name = self.cfg['platform'] + + if not platform_name: + platform_name = get_conda_lock_platform() + + cmd = "conda-lock -f %s -p %s" % ( + self.cfg['environment_file'], platform_name) + run_cmd(cmd, log_all=True, simple=True) + + # the default name for lock_file is 'conda-lock.yml' + lock_file = 'conda-lock.yml' + + # ship lock_file in installation + super(Binary, self).fetch_sources(sources=[lock_file]) + self.extract_step() + + # use lock_file to create environment + cmd = "%s conda-lock install %s --prefix %s" % (self.cfg['preinstallopts'], lock_file, self.installdir) + run_cmd(cmd, log_all=True, simple=True) + + elif self.cfg['environment_file'] or self.cfg['remote_environment']: + + if self.cfg['environment_file']: + env_spec = self.cfg['environment_file'] + else: + env_spec = self.cfg['remote_environment'] + + # use --force to ignore existing installation directory + cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, + env_spec, self.installdir) + run_cmd(cmd, log_all=True, simple=True) + + else: + + if self.cfg['requirements']: + + install_args = "-y %s " % self.cfg['requirements'] + if self.cfg['channels']: + install_args += ' '.join('-c ' + + chan for chan in self.cfg['channels']) + + self.log.info("Installed conda requirements") + + cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, + self.installdir, install_args) + run_cmd(cmd, log_all=True, simple=True) + + # clean up + cmd = "%s clean -ya" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) + + def make_module_extra(self): + """Add the install directory to the PATH.""" + txt = super(CondaEnhance, self).make_module_extra() + txt += self.module_generator.set_environment( + 'CONDA_ENV', self.installdir) + txt += self.module_generator.set_environment( + 'CONDA_PREFIX', self.installdir) + txt += self.module_generator.set_environment( + 'CONDA_DEFAULT_ENV', self.installdir) + self.log.debug("make_module_extra added this: %s", txt) + return txt + + def make_module_req_guess(self): + """ + A dictionary of possible directories to look for. + """ + # LD_LIBRARY_PATH issue discusses here + # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl + return { + 'PATH': ['bin', 'sbin'], + 'MANPATH': ['man', os.path.join('share', 'man')], + 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], + } From 3c38cd8ddd1dd47b256b2c17e32c83a883088994 Mon Sep 17 00:00:00 2001 From: LeahHan Date: Mon, 5 Aug 2024 19:21:04 +1200 Subject: [PATCH 3/5] rename condaenhance.py to conda.py --- easybuild/easyblocks/generic/conda.py | 83 +++++++-- easybuild/easyblocks/generic/condaenhance.py | 184 ------------------- 2 files changed, 69 insertions(+), 198 deletions(-) mode change 100644 => 100755 easybuild/easyblocks/generic/conda.py delete mode 100755 easybuild/easyblocks/generic/condaenhance.py diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py old mode 100644 new mode 100755 index 5fce4607e6f..03bb61adc96 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -30,6 +30,7 @@ """ import os +import platform from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM @@ -38,7 +39,7 @@ from easybuild.tools.build_log import EasyBuildError -class Conda(Binary): +class CondaEnhance(Binary): """Support for installing software using 'conda'.""" @staticmethod @@ -50,21 +51,24 @@ def extra_options(extra_vars=None): 'environment_file': [None, "Conda environment.yml file to use with 'conda env create'", CUSTOM], 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], + 'use_conda_lock': [False, "Whether to use conda-lock for reproducibility", CUSTOM], + 'lock_file': [None, "Path to a pre-generated conda-lock file", CUSTOM], + 'platform': [None, "Platform specified for conda-lock", CUSTOM], }) return extra_vars def extract_step(self): """Copy sources via extract_step of parent, if any are specified.""" if self.src: - super(Conda, self).extract_step() + super(CondaEnhance, self).extract_step() def install_step(self): """Install software using 'conda env create' or 'conda create' & 'conda install' (or the 'mamba', etc., equivalent).""" if (get_software_root('anaconda2') or get_software_root('miniconda2') or - get_software_root('anaconda3') or get_software_root('miniconda3') or - get_software_root('miniforge3')): + get_software_root('anaconda3') or get_software_root('miniconda3') or get_software_root('miniforge3')): conda_cmd = 'conda' + # add independent entry for condaEnhance and miniforge (done) elif get_software_root('mamba'): conda_cmd = 'mamba' elif get_software_root('micromamba'): @@ -72,15 +76,62 @@ def install_step(self): else: raise EasyBuildError("No conda/mamba/micromamba available.") + def get_conda_lock_platform(): + system = platform.system().lower() + machine = platform.machine().lower() + + platforms = { + "linux": { + "x86_64": "linux-64", + "aarch64": "linux-aarch64", + "ppc64le": "linux-ppc64le" + }, + "darwin": { + "x86_64": "osx-64", + "arm64": "osx-arm64" + }, + "windows": { + "x86_64": "win-64", + "amd64": "win-64" + } + } + + return platforms.get(system, {}).get(machine) + # initialize conda environment - # setuptools is just a choice, but *something* needs to be there - cmd = "%s config --add create_default_packages setuptools" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + if self.cfg['use_conda_lock']: + # install conda-lock + cmd = "%s install conda-lock" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) + + lock_file = self.cfg['lock_file'] + + if not lock_file: + # generate lock file + platform_name = self.cfg['platform'] + + if not platform_name: + platform_name = get_conda_lock_platform() + + cmd = "conda-lock -f %s -p %s" % ( + self.cfg['environment_file'], platform_name) + run_cmd(cmd, log_all=True, simple=True) + + # the default name for lock_file is 'conda-lock.yml' + lock_file = 'conda-lock.yml' + + # ship lock_file in installation + super(Binary, self).fetch_sources(sources=[lock_file]) + self.extract_step() + + # use lock_file to create environment + cmd = "%s conda-lock install %s --prefix %s" % (self.cfg['preinstallopts'], lock_file, self.installdir) + run_cmd(cmd, log_all=True, simple=True) - if self.cfg['environment_file'] or self.cfg['remote_environment']: + elif self.cfg['environment_file'] or self.cfg['remote_environment']: if self.cfg['environment_file']: - env_spec = '-f ' + self.cfg['environment_file'] + env_spec = self.cfg['environment_file'] else: env_spec = self.cfg['remote_environment'] @@ -95,7 +146,8 @@ def install_step(self): install_args = "-y %s " % self.cfg['requirements'] if self.cfg['channels']: - install_args += ' '.join('-c ' + chan for chan in self.cfg['channels']) + install_args += ' '.join('-c ' + + chan for chan in self.cfg['channels']) self.log.info("Installed conda requirements") @@ -109,10 +161,13 @@ def install_step(self): def make_module_extra(self): """Add the install directory to the PATH.""" - txt = super(Conda, self).make_module_extra() - txt += self.module_generator.set_environment('CONDA_ENV', self.installdir) - txt += self.module_generator.set_environment('CONDA_PREFIX', self.installdir) - txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir) + txt = super(CondaEnhance, self).make_module_extra() + txt += self.module_generator.set_environment( + 'CONDA_ENV', self.installdir) + txt += self.module_generator.set_environment( + 'CONDA_PREFIX', self.installdir) + txt += self.module_generator.set_environment( + 'CONDA_DEFAULT_ENV', self.installdir) self.log.debug("make_module_extra added this: %s", txt) return txt diff --git a/easybuild/easyblocks/generic/condaenhance.py b/easybuild/easyblocks/generic/condaenhance.py deleted file mode 100755 index 03bb61adc96..00000000000 --- a/easybuild/easyblocks/generic/condaenhance.py +++ /dev/null @@ -1,184 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for installing software using 'conda', implemented as an easyblock. - -@author: Jillian Rowe (New York University Abu Dhabi) -@author: Kenneth Hoste (HPC-UGent) -""" - -import os -import platform - -from easybuild.easyblocks.generic.binary import Binary -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd -from easybuild.tools.modules import get_software_root -from easybuild.tools.build_log import EasyBuildError - - -class CondaEnhance(Binary): - """Support for installing software using 'conda'.""" - - @staticmethod - def extra_options(extra_vars=None): - """Extra easyconfig parameters specific to Conda easyblock.""" - extra_vars = Binary.extra_options(extra_vars) - extra_vars.update({ - 'channels': [None, "List of conda channels to pass to 'conda install'", CUSTOM], - 'environment_file': [None, "Conda environment.yml file to use with 'conda env create'", CUSTOM], - 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], - 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], - 'use_conda_lock': [False, "Whether to use conda-lock for reproducibility", CUSTOM], - 'lock_file': [None, "Path to a pre-generated conda-lock file", CUSTOM], - 'platform': [None, "Platform specified for conda-lock", CUSTOM], - }) - return extra_vars - - def extract_step(self): - """Copy sources via extract_step of parent, if any are specified.""" - if self.src: - super(CondaEnhance, self).extract_step() - - def install_step(self): - """Install software using 'conda env create' or 'conda create' & 'conda install' - (or the 'mamba', etc., equivalent).""" - if (get_software_root('anaconda2') or get_software_root('miniconda2') or - get_software_root('anaconda3') or get_software_root('miniconda3') or get_software_root('miniforge3')): - conda_cmd = 'conda' - # add independent entry for condaEnhance and miniforge (done) - elif get_software_root('mamba'): - conda_cmd = 'mamba' - elif get_software_root('micromamba'): - conda_cmd = 'micromamba' - else: - raise EasyBuildError("No conda/mamba/micromamba available.") - - def get_conda_lock_platform(): - system = platform.system().lower() - machine = platform.machine().lower() - - platforms = { - "linux": { - "x86_64": "linux-64", - "aarch64": "linux-aarch64", - "ppc64le": "linux-ppc64le" - }, - "darwin": { - "x86_64": "osx-64", - "arm64": "osx-arm64" - }, - "windows": { - "x86_64": "win-64", - "amd64": "win-64" - } - } - - return platforms.get(system, {}).get(machine) - - # initialize conda environment - if self.cfg['use_conda_lock']: - # install conda-lock - cmd = "%s install conda-lock" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) - - lock_file = self.cfg['lock_file'] - - if not lock_file: - # generate lock file - platform_name = self.cfg['platform'] - - if not platform_name: - platform_name = get_conda_lock_platform() - - cmd = "conda-lock -f %s -p %s" % ( - self.cfg['environment_file'], platform_name) - run_cmd(cmd, log_all=True, simple=True) - - # the default name for lock_file is 'conda-lock.yml' - lock_file = 'conda-lock.yml' - - # ship lock_file in installation - super(Binary, self).fetch_sources(sources=[lock_file]) - self.extract_step() - - # use lock_file to create environment - cmd = "%s conda-lock install %s --prefix %s" % (self.cfg['preinstallopts'], lock_file, self.installdir) - run_cmd(cmd, log_all=True, simple=True) - - elif self.cfg['environment_file'] or self.cfg['remote_environment']: - - if self.cfg['environment_file']: - env_spec = self.cfg['environment_file'] - else: - env_spec = self.cfg['remote_environment'] - - # use --force to ignore existing installation directory - cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, - env_spec, self.installdir) - run_cmd(cmd, log_all=True, simple=True) - - else: - - if self.cfg['requirements']: - - install_args = "-y %s " % self.cfg['requirements'] - if self.cfg['channels']: - install_args += ' '.join('-c ' + - chan for chan in self.cfg['channels']) - - self.log.info("Installed conda requirements") - - cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, - self.installdir, install_args) - run_cmd(cmd, log_all=True, simple=True) - - # clean up - cmd = "%s clean -ya" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) - - def make_module_extra(self): - """Add the install directory to the PATH.""" - txt = super(CondaEnhance, self).make_module_extra() - txt += self.module_generator.set_environment( - 'CONDA_ENV', self.installdir) - txt += self.module_generator.set_environment( - 'CONDA_PREFIX', self.installdir) - txt += self.module_generator.set_environment( - 'CONDA_DEFAULT_ENV', self.installdir) - self.log.debug("make_module_extra added this: %s", txt) - return txt - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - """ - # LD_LIBRARY_PATH issue discusses here - # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl - return { - 'PATH': ['bin', 'sbin'], - 'MANPATH': ['man', os.path.join('share', 'man')], - 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], - } From b363c0b4a0bb44a64528143a66ebc808a8564862 Mon Sep 17 00:00:00 2001 From: LeahHan Date: Mon, 5 Aug 2024 19:31:15 +1200 Subject: [PATCH 4/5] remove condalock.py, deleted unnecessary comments and fixed mistakes in conda.py --- easybuild/easyblocks/generic/conda.py | 7 +- easybuild/easyblocks/generic/condalock.py | 130 ---------------------- 2 files changed, 3 insertions(+), 134 deletions(-) delete mode 100644 easybuild/easyblocks/generic/condalock.py diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index 03bb61adc96..05f04874404 100755 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -39,7 +39,7 @@ from easybuild.tools.build_log import EasyBuildError -class CondaEnhance(Binary): +class Conda(Binary): """Support for installing software using 'conda'.""" @staticmethod @@ -60,7 +60,7 @@ def extra_options(extra_vars=None): def extract_step(self): """Copy sources via extract_step of parent, if any are specified.""" if self.src: - super(CondaEnhance, self).extract_step() + super(Conda, self).extract_step() def install_step(self): """Install software using 'conda env create' or 'conda create' & 'conda install' @@ -68,7 +68,6 @@ def install_step(self): if (get_software_root('anaconda2') or get_software_root('miniconda2') or get_software_root('anaconda3') or get_software_root('miniconda3') or get_software_root('miniforge3')): conda_cmd = 'conda' - # add independent entry for condaEnhance and miniforge (done) elif get_software_root('mamba'): conda_cmd = 'mamba' elif get_software_root('micromamba'): @@ -161,7 +160,7 @@ def get_conda_lock_platform(): def make_module_extra(self): """Add the install directory to the PATH.""" - txt = super(CondaEnhance, self).make_module_extra() + txt = super(Conda, self).make_module_extra() txt += self.module_generator.set_environment( 'CONDA_ENV', self.installdir) txt += self.module_generator.set_environment( diff --git a/easybuild/easyblocks/generic/condalock.py b/easybuild/easyblocks/generic/condalock.py deleted file mode 100644 index ca1eeca1ad5..00000000000 --- a/easybuild/easyblocks/generic/condalock.py +++ /dev/null @@ -1,130 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for installing software using 'conda', implemented as an easyblock. - -@author: Jillian Rowe (New York University Abu Dhabi) -@author: Kenneth Hoste (HPC-UGent) -""" - -import os - -from easybuild.easyblocks.generic.binary import Binary -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd -from easybuild.tools.modules import get_software_root -from easybuild.tools.build_log import EasyBuildError - - -class CondaLock(Binary): - """Support for installing software using 'conda'.""" - - @staticmethod - def extra_options(extra_vars=None): - """Extra easyconfig parameters specific to Conda easyblock.""" - extra_vars = Binary.extra_options(extra_vars) - extra_vars.update({ - 'channels': [None, "List of conda channels to pass to 'conda install'", CUSTOM], - 'environment_file': [None, "Conda environment.yml file to use with 'conda env create'", CUSTOM], - 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], - 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], - }) - return extra_vars - - def extract_step(self): - """Copy sources via extract_step of parent, if any are specified.""" - if self.src: - super(CondaLock, self).extract_step() - - def install_step(self): - """Install software using 'conda env create' or 'conda create' & 'conda install' - (or the 'mamba', etc., equivalent).""" - if (get_software_root('anaconda2') or get_software_root('miniconda2') or - get_software_root('anaconda3') or get_software_root('miniconda3')): - conda_cmd = 'conda' - elif get_software_root('miniforge3'): - conda_cmd = 'conda-lock' - elif get_software_root('mamba'): - conda_cmd = 'mamba' - elif get_software_root('micromamba'): - conda_cmd = 'micromamba' - else: - raise EasyBuildError("No conda/mamba/micromamba available.") - - # initialize conda environment - # install conda-lock package - cmd = "pip install conda-lock" - run_cmd(cmd, log_all=True, simple=True) - - if self.cfg['environment_file'] or self.cfg['remote_environment']: - - if self.cfg['environment_file']: - env_spec = self.cfg['environment_file'] - else: - env_spec = self.cfg['remote_environment'] - - # use --force to ignore existing installation directory - cmd = "%s %s install %s --prefix %s" % (self.cfg['preinstallopts'], conda_cmd, - env_spec, self.installdir) - run_cmd(cmd, log_all=True, simple=True) - - else: - - if self.cfg['requirements']: - - install_args = "-y %s " % self.cfg['requirements'] - if self.cfg['channels']: - install_args += ' '.join('-c ' + chan for chan in self.cfg['channels']) - - self.log.info("Installed conda requirements") - - cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, - self.installdir, install_args) - run_cmd(cmd, log_all=True, simple=True) - - # clean up - # cmd = "%s clean -ya" % conda_cmd - #run_cmd(cmd, log_all=True, simple=True) - - def make_module_extra(self): - """Add the install directory to the PATH.""" - txt = super(CondaLock, self).make_module_extra() - txt += self.module_generator.set_environment('CONDA_ENV', self.installdir) - txt += self.module_generator.set_environment('CONDA_PREFIX', self.installdir) - txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir) - self.log.debug("make_module_extra added this: %s", txt) - return txt - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - """ - # LD_LIBRARY_PATH issue discusses here - # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl - return { - 'PATH': ['bin', 'sbin'], - 'MANPATH': ['man', os.path.join('share', 'man')], - 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], - } From 68eb020dd3f9caa3c70d67f37d7c6ac87ebf0a31 Mon Sep 17 00:00:00 2001 From: LeahHan Date: Tue, 13 Aug 2024 15:31:47 +1200 Subject: [PATCH 5/5] add function to handle multi-platform and single-platform lock files --- easybuild/easyblocks/generic/conda.py | 120 +++++++++++++++++++------- 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index 05f04874404..2b7ecdf025e 100755 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -30,14 +30,15 @@ """ import os -import platform +import yaml + from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.run import run_cmd from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError - +import easybuild.tools.systemtools as st class Conda(Binary): """Support for installing software using 'conda'.""" @@ -52,8 +53,7 @@ def extra_options(extra_vars=None): 'remote_environment': [None, "Remote conda environment to use with 'conda env create'", CUSTOM], 'requirements': [None, "Requirements specification to pass to 'conda install'", CUSTOM], 'use_conda_lock': [False, "Whether to use conda-lock for reproducibility", CUSTOM], - 'lock_file': [None, "Path to a pre-generated conda-lock file", CUSTOM], - 'platform': [None, "Platform specified for conda-lock", CUSTOM], + 'provided_lock_file_name': [None, "Provided lock file name", CUSTOM], }) return extra_vars @@ -74,16 +74,22 @@ def install_step(self): conda_cmd = 'micromamba' else: raise EasyBuildError("No conda/mamba/micromamba available.") + + # initialize conda environment + # setuptools is just a choice, but *something* needs to be there + cmd = "%s config --add create_default_packages setuptools" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) - def get_conda_lock_platform(): - system = platform.system().lower() - machine = platform.machine().lower() + def get_system_platform_name(): + system = st.get_system_info()['os_type'].lower() + machine = st.get_system_info()['cpu_arch'].lower() platforms = { "linux": { "x86_64": "linux-64", "aarch64": "linux-aarch64", - "ppc64le": "linux-ppc64le" + "ppc64le": "linux-ppc64le", + "s390x": "linux-s390x" }, "darwin": { "x86_64": "osx-64", @@ -91,46 +97,100 @@ def get_conda_lock_platform(): }, "windows": { "x86_64": "win-64", - "amd64": "win-64" + "amd64": "win-64", + "arm64": "win-arm64", + "x86": "win-32" + }, + "zos": { + "z": "zos-z" } } return platforms.get(system, {}).get(machine) + + def check_lock_file_type(lock_file): + _, ext = os.path.splitext(lock_file) + if ext == ".lock": + return "single" + elif ext in [".yml", ".yaml"]: + return "multi" + else: + raise EasyBuildError("The provided file is not a lock file") + + def verify_lock_file_platform(lock_file, platform_name): + + file_type = check_lock_file_type(lock_file) + + if file_type == "multi": + # for a multi-platform lock file like conda-lock.yml + with open(lock_file, 'r') as f: + lock_data = yaml.safe_load(f) + return platform_name in lock_data.get('metadata', []).get('platforms',[]) + else: + # for a single-platform rendered lock file like conda-linux64.lock + with open(lock_file, 'r') as f: + for line in f: + if line.startswith("# platform:"): + return line.split(":", 1)[1].strip() == platform_name + - # initialize conda environment if self.cfg['use_conda_lock']: - # install conda-lock - cmd = "%s install conda-lock" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + lock_file = self.cfg['provided_lock_file_name'] + platform_name = get_system_platform_name() - lock_file = self.cfg['lock_file'] + # the default name for rendered lock_file is 'conda-.lock' + platform_rendered_lock_file='conda-{}.lock'.format(platform_name) if not lock_file: - # generate lock file - platform_name = self.cfg['platform'] + # install conda-lock + cmd = "%s install conda-lock" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) - if not platform_name: - platform_name = get_conda_lock_platform() + # ship environment file in installation + super(Binary, self).fetch_sources(sources=[self.cfg['environment_file']]) + self.extract_step() - cmd = "conda-lock -f %s -p %s" % ( - self.cfg['environment_file'], platform_name) + # generate lock file and render + cmd = "conda-lock -f %s -p %s && conda-lock render -p %s" % ( + self.cfg['environment_file'], platform_name, platform_name) run_cmd(cmd, log_all=True, simple=True) - # the default name for lock_file is 'conda-lock.yml' - lock_file = 'conda-lock.yml' + + else: + + lock_file_type = check_lock_file_type(lock_file) # ship lock_file in installation super(Binary, self).fetch_sources(sources=[lock_file]) self.extract_step() + + # verify that a lock file for the current platform has been provided + if not verify_lock_file_platform(lock_file,platform_name): + raise EasyBuildError("The provided lock file does not match this platform") + + + if lock_file_type == "multi": + + # install conda-lock + cmd = "%s install conda-lock" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) + + # render + cmd = "conda-lock render -p %s %s" % (platform_name, lock_file) + run_cmd(cmd, log_all=True, simple=True) + else: + + platform_rendered_lock_file=lock_file + # use lock_file to create environment - cmd = "%s conda-lock install %s --prefix %s" % (self.cfg['preinstallopts'], lock_file, self.installdir) + cmd = "%s %s create --file %s -p %s -y" % (self.cfg['preinstallopts'],conda_cmd, platform_rendered_lock_file, self.installdir) run_cmd(cmd, log_all=True, simple=True) elif self.cfg['environment_file'] or self.cfg['remote_environment']: if self.cfg['environment_file']: - env_spec = self.cfg['environment_file'] + env_spec = '-f ' + self.cfg['environment_file'] else: env_spec = self.cfg['remote_environment'] @@ -145,8 +205,7 @@ def get_conda_lock_platform(): install_args = "-y %s " % self.cfg['requirements'] if self.cfg['channels']: - install_args += ' '.join('-c ' + - chan for chan in self.cfg['channels']) + install_args += ' '.join('-c ' + chan for chan in self.cfg['channels']) self.log.info("Installed conda requirements") @@ -161,12 +220,9 @@ def get_conda_lock_platform(): def make_module_extra(self): """Add the install directory to the PATH.""" txt = super(Conda, self).make_module_extra() - txt += self.module_generator.set_environment( - 'CONDA_ENV', self.installdir) - txt += self.module_generator.set_environment( - 'CONDA_PREFIX', self.installdir) - txt += self.module_generator.set_environment( - 'CONDA_DEFAULT_ENV', self.installdir) + txt += self.module_generator.set_environment('CONDA_ENV', self.installdir) + txt += self.module_generator.set_environment('CONDA_PREFIX', self.installdir) + txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir) self.log.debug("make_module_extra added this: %s", txt) return txt