Skip to content

use python -m pip instead of pip in PythonPackage easyblock #2807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 2 commits into from
Oct 19, 2022
Merged

use python -m pip instead of pip in PythonPackage easyblock #2807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db889b6
Prefer `python -m pip` over `pip` for Python packages
Oct 18, 2022
595b3be
Make dm_reverb aware of prebuildopts
Oct 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion easybuild/easyblocks/d/dm_reverb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ def build_step(self, *args, **kwargs):
# print full compilation commands
bazel_build_opts += " --subcommands"

bazel_cmd = "bazel %s build %s %s" % (bazel_opts, bazel_build_opts, bazel_build_pkg)
bazel_cmd = "%s bazel %s build %s %s" % (self.cfg['prebuildopts'], bazel_opts, bazel_build_opts,
Copy link
Member Author

@ocaisa ocaisa Oct 18, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a fix I needed to allow me to get around EESSI/software-layer#192 which affects this package.

bazel_build_pkg)

return run_cmd(bazel_cmd, log_all=True, simple=True, log_output=True)

Expand All @@ -121,6 +122,7 @@ def install_step(self, *args, **kwargs):
'installopts': self.cfg['installopts'],
'loc': whl_path,
'prefix': self.installdir,
'python': self.python_cmd,
}

return super(EB_dm_minus_reverb, self).install_step(*args, **kwargs)
36 changes: 20 additions & 16 deletions easybuild/easyblocks/generic/pythonpackage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
# not 'easy_install' deliberately, to avoid that pkg installations listed in easy-install.pth get preference
# '.' is required at the end when using easy_install/pip in unpacked source dir
EASY_INSTALL_TARGET = "easy_install"
PIP_INSTALL_CMD = "pip install --prefix=%(prefix)s %(installopts)s %(loc)s"
PIP_INSTALL_CMD = "%(python)s -m pip install --prefix=%(prefix)s %(installopts)s %(loc)s"
SETUP_PY_INSTALL_CMD = "%(python)s setup.py %(install_target)s --prefix=%(prefix)s %(installopts)s"
UNKNOWN = 'UNKNOWN'

Expand Down Expand Up @@ -204,14 +204,14 @@ def get_pylibdirs(python_cmd):
return all_pylibdirs


def det_pip_version():
"""Determine version of currently active 'pip' command."""
def det_pip_version(python_cmd='python'):
"""Determine version of currently active 'pip' module."""

pip_version = None
log = fancylogger.getLogger('det_pip_version', fname=False)
log.info("Determining pip version...")

out, _ = run_cmd("pip --version", verbose=False, simple=False, trace=False)
out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False)

pip_version_regex = re.compile('^pip ([0-9.]+)')
res = pip_version_regex.search(out)
Expand Down Expand Up @@ -247,8 +247,8 @@ def extra_options(extra_vars=None):
"the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM],
'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM],
'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM],
'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed "
"and check for any package with an invalid (0.0.0) version.", CUSTOM],
'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are "
"installed and check for any package with an invalid (0.0.0) version.", CUSTOM],
'runtest': [True, "Run unit tests.", CUSTOM], # overrides default
'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files",
CUSTOM],
Expand All @@ -261,8 +261,8 @@ def extra_options(extra_vars=None):
# see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras
'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM],
'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM],
'use_pip_requirement': [False, "Install using 'pip install --requirement'. The sources is expected " +
"to be the requirements file.", CUSTOM],
'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " +
"expected to be the requirements file.", CUSTOM],
'zipped_egg': [False, "Install as a zipped eggs", CUSTOM],
})
# Use PYPI_SOURCE as the default for source_urls.
Expand Down Expand Up @@ -468,7 +468,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None):
using_pip = self.install_cmd.startswith(PIP_INSTALL_CMD)
if using_pip:

pip_version = det_pip_version()
pip_version = det_pip_version(python_cmd=self.python_cmd)
if pip_version:
# pip 8.x or newer required, because of --prefix option being used
if LooseVersion(pip_version) >= LooseVersion('8.0'):
Expand Down Expand Up @@ -834,8 +834,11 @@ def sanity_check_step(self, *args, **kwargs):
kwargs.update({'exts_filter': exts_filter})

if self.cfg.get('sanity_pip_check', False):
pip_version = det_pip_version()
pip_version = det_pip_version(python_cmd=python_cmd)

if pip_version:
pip_check_command = "%s -m pip check" % python_cmd

if LooseVersion(pip_version) >= LooseVersion('9.0.0'):

if not self.is_extension:
Expand All @@ -846,11 +849,11 @@ def sanity_check_step(self, *args, **kwargs):

pip_check_errors = []

pip_check_msg, ec = run_cmd("pip check", log_ok=False)
pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False)
if ec:
pip_check_errors.append('`pip check` failed:\n%s' % pip_check_msg)
pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg))
else:
self.log.info('`pip check` completed successfully')
self.log.info('`%s` completed successfully' % pip_check_command)

# Also check for a common issue where the package version shows up as 0.0.0 often caused
# by using setup.py as the installation method for a package which is released as a generic wheel
Expand All @@ -869,8 +872,8 @@ def sanity_check_step(self, *args, **kwargs):
version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package)
except StopIteration:
msg = ('Package %s in unversioned_packages was not found in the installed packages. '
'Check that the name from `pip list` is used which may be different than the '
'module name.' % unversioned_package)
'Check that the name from `python -m pip list` is used which may be different '
'than the module name.' % unversioned_package)
else:
msg = ('Package %s in unversioned_packages has a version of %s which is valid. '
'Please remove it from unversioned_packages.' % (unversioned_package, version))
Expand All @@ -894,7 +897,8 @@ def sanity_check_step(self, *args, **kwargs):
if pip_check_errors:
raise EasyBuildError('\n'.join(pip_check_errors))
else:
raise EasyBuildError("pip >= 9.0.0 is required for running 'pip check', found %s", pip_version)
raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", (pip_check_command,
pip_version))
else:
raise EasyBuildError("Failed to determine pip version!")

Expand Down
1 change: 1 addition & 0 deletions easybuild/easyblocks/t/tensorrt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ def extensions_step(self):
'installopts': self.cfg['installopts'],
'loc': whl_paths[0],
'prefix': self.installdir,
'python': self.python_cmd,
}

# Use --no-deps to prevent pip from downloading & installing
Expand Down