diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index a31b94ed9f..c93cca67aa 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: [2.7, 3.7] + python: [3.7] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 6d8506e1e9..0ee98aec99 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: [2.7, 3.7] + python: [3.7] apptainer: [1.0.0, 1.1.7] fail-fast: false steps: diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 2e20bbd4cc..3aaa3a9f70 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] fail-fast: false steps: - uses: actions/checkout@v3 @@ -91,7 +91,7 @@ jobs: pymajver=$(python -c 'import sys; print(sys.version_info[0])') pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') # check patterns in verbose output - for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .python3.\.\.\." "^>> .python3. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python3' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python3 \(.*/bin/python3\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 14d736fe80..315941b65c 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v3 @@ -30,10 +30,4 @@ jobs: - name: Run flake8 to verify PEP8-compliance of Python code run: | - # don't check py2vs3/py3.py when testing with Python 2, and vice versa - if [[ "${{ matrix.python-version }}" =~ "2." ]]; then - py_excl=py3 - else - py_excl=py2 - fi - flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py + flake8 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5475952dd2..8111dbbff7 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [2.7, 3.6] + python: [3.6] modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context @@ -46,9 +46,6 @@ jobs: module_syntax: Lua include: # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) - - python: 3.5 - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua @@ -205,10 +202,6 @@ jobs: # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # silence deprecation warning when using Python 2, since it breaks a bunch of tests - if [ "${{matrix.python}}" == '2.7' ]; then - export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2 - fi # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..df7baa0e0e 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -87,7 +87,6 @@ import weakref from easybuild.tools import LooseVersion -from easybuild.tools.py2vs3 import raise_with_traceback, string_type def _env_to_boolean(varname, default=False): @@ -214,11 +213,11 @@ class MissingLevelName(KeyError): def getLevelInt(level_name): """Given a level name, return the int value""" - if not isinstance(level_name, string_type): + if not isinstance(level_name, str): raise TypeError('Provided name %s is not a string (type %s)' % (level_name, type(level_name))) level = logging.getLevelName(level_name) - if isinstance(level, string_type): + if isinstance(level, str): raise MissingLevelName('Unknown loglevel name %s' % level_name) return level @@ -328,7 +327,7 @@ def raiseException(self, message, exception=None, catch=False): exception = self.RAISE_EXCEPTION_CLASS self.RAISE_EXCEPTION_LOG_METHOD(fullmessage) - raise_with_traceback(exception, message, tb) + raise exception(message).with_traceback(tb) # pylint: disable=unused-argument def deprecated(self, msg, cur_ver, max_ver, depth=2, exception=None, log_callback=None, *args, **kwargs): @@ -588,7 +587,7 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_ os.makedirs(directory) except Exception as ex: exc, detail, tb = sys.exc_info() - raise_with_traceback(exc, "Cannot create logdirectory %s: %s \n detail: %s" % (directory, ex, detail), tb) + raise exc("Cannot create logdirectory %s: %s \n detail: %s" % (directory, ex, detail)).with_traceback(tb) return _logToSomething( logging.handlers.RotatingFileHandler, @@ -741,7 +740,7 @@ def setLogLevel(level): """ Set a global log level for all FancyLoggers """ - if isinstance(level, string_type): + if isinstance(level, str): level = getLevelInt(level) logger = getLogger(fname=False, clsname=False) logger.setLevel(level) diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py index 6bfe91a82b..5d0205687e 100644 --- a/easybuild/base/frozendict.py +++ b/easybuild/base/frozendict.py @@ -21,10 +21,10 @@ It can be used as a drop-in replacement for dictionaries where immutability is desired. """ import operator +from collections.abc import Mapping from functools import reduce from easybuild.base import fancylogger -from easybuild.tools.py2vs3 import Mapping # minor adjustments: diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..3e9788ac44 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -31,6 +31,7 @@ * Jens Timmerman (Ghent University) """ +import configparser import copy import difflib import inspect @@ -39,13 +40,15 @@ import re import sys import textwrap +from configparser import ConfigParser from functools import reduce +from io import StringIO from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4 from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption -from easybuild.tools.py2vs3 import StringIO, configparser, ConfigParser, string_type, subprocess_popen_text +from easybuild.tools.run import subprocess_popen_text from easybuild.tools.utilities import mk_md_table, mk_rst_table, nub, shell_quote try: @@ -133,7 +136,7 @@ def get_empty_add_flex(allvalues, self=None): empty = None if isinstance(allvalues, (list, tuple)): - if isinstance(allvalues[0], string_type): + if isinstance(allvalues[0], str): empty = '' if empty is None: @@ -464,7 +467,7 @@ def is_value_a_commandline_option(self, opt, value, index=None): # --longopt=value, so no issues there either. # following checks assume that value is a string (not a store_or_None) - if not isinstance(value, string_type): + if not isinstance(value, str): return None cmdline_index = None @@ -1191,7 +1194,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non # choices nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings hlp += ' (choices: %s)' % ', '.join(nameds['choices']) - elif isinstance(extra_detail, string_type) and len(extra_detail) == 1: + elif isinstance(extra_detail, str) and len(extra_detail) == 1: args.insert(0, "-%s" % extra_detail) elif isinstance(extra_detail, (dict,)): # extract any optcomplete completer hints diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 9a2bc8a127..eaf4dc630c 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -107,7 +107,6 @@ from optparse import OptionParser, Option from pprint import pformat -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import shell_quote debugfn = None # for debugging only @@ -211,7 +210,7 @@ class FileCompleter(Completer): CALL_ARGS_OPTIONAL = ['prefix'] def __init__(self, endings=None): - if isinstance(endings, string_type): + if isinstance(endings, str): endings = [endings] elif endings is None: endings = [] @@ -282,11 +281,11 @@ class RegexCompleter(Completer): def __init__(self, regexlist, always_dirs=True): self.always_dirs = always_dirs - if isinstance(regexlist, string_type): + if isinstance(regexlist, str): regexlist = [regexlist] self.regexlist = [] for regex in regexlist: - if isinstance(regex, string_type): + if isinstance(regex, str): regex = re.compile(regex) self.regexlist.append(regex) @@ -546,7 +545,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete # File completion. if completer and (not prefix or not prefix.startswith('-')): # Call appropriate completer depending on type. - if isinstance(completer, (string_type, list, tuple)): + if isinstance(completer, (str, list, tuple)): completer = FileCompleter(completer) elif not isinstance(completer, (types.FunctionType, types.LambdaType, types.ClassType, types.ObjectType)): # TODO: what to do here? @@ -554,7 +553,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete completions = completer(**completer_kwargs) - if isinstance(completions, string_type): + if isinstance(completions, str): # is a bash command, just run it if SHELL in (BASH,): # TODO: zsh print(completions) diff --git a/easybuild/base/rest.py b/easybuild/base/rest.py index ed4c187436..4d97eb6183 100644 --- a/easybuild/base/rest.py +++ b/easybuild/base/rest.py @@ -40,9 +40,10 @@ import copy import json from functools import partial +from urllib.parse import urlencode +from urllib.request import HTTPSHandler, Request, build_opener from easybuild.base import fancylogger -from easybuild.tools.py2vs3 import HTTPSHandler, Request, build_opener, json_loads, string_type, urlencode class Client(object): @@ -180,7 +181,7 @@ def request(self, method, url, body, headers, content_type=None): else: body = conn.read() try: - pybody = json_loads(body) + pybody = json.loads(body) except ValueError: pybody = body fancylogger.getLogger().debug('reponse len: %s ', len(pybody)) @@ -203,10 +204,8 @@ def get_connection(self, method, url, body, headers): else: sep = '' - # value passed to 'data' must be a 'bytes' value (not 'str') in Python 3.x, but a string value in Python 2 - # hence, we encode the value obtained (if needed) - # this doesn't affect the value type in Python 2, and makes it a 'bytes' value in Python 3 - if isinstance(body, string_type): + # value passed to 'data' must be a 'bytes' value (not 'str') hence, we encode the value obtained (if needed) + if isinstance(body, str): body = body.encode('utf-8') request = Request(self.url + sep + url, data=body) diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 01b64d84ef..9d3cea9d3c 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -39,16 +39,10 @@ import re import sys from contextlib import contextmanager - -try: - from cStringIO import StringIO # Python 2 -except ImportError: - from io import StringIO # Python 3 +from io import StringIO from unittest import TestCase as OrigTestCase -from easybuild.tools.py2vs3 import string_type - def nicediff(txta, txtb, offset=5): """ @@ -85,7 +79,7 @@ class TestCase(OrigTestCase): def is_string(self, x): """test if the variable x is a string)""" try: - return isinstance(x, string_type) + return isinstance(x, str) except NameError: return isinstance(x, str) diff --git a/easybuild/base/wrapper.py b/easybuild/base/wrapper.py index e73c8d22c5..bc44d9c494 100644 --- a/easybuild/base/wrapper.py +++ b/easybuild/base/wrapper.py @@ -6,7 +6,24 @@ Original code by http://stackoverflow.com/users/416467/kindall from answer 4 of http://stackoverflow.com/questions/9057669/how-can-i-intercept-calls-to-pythons-magic-methods-in-new-style-classes """ -from easybuild.tools.py2vs3 import mk_wrapper_baseclass + + +# based on six's 'with_metaclass' function +# see also https://stackoverflow.com/questions/18513821/python-metaclass-understanding-the-with-metaclass +def create_base_metaclass(base_class_name, metaclass, *bases): + """Create new class with specified metaclass based on specified base class(es).""" + return metaclass(base_class_name, bases, {}) + + +def mk_wrapper_baseclass(metaclass): + + class WrapperBase(object, metaclass=metaclass): + """ + Wrapper class that provides proxy access to an instance of some internal instance. + """ + __wraps__ = None + + return WrapperBase class WrapperMeta(type): diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1d177eafb9..31974acfa8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -97,7 +97,6 @@ from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package -from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group @@ -420,7 +419,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct if source is None: raise EasyBuildError("fetch_source called with empty 'source' argument") - elif isinstance(source, string_type): + elif isinstance(source, str): filename = source elif isinstance(source, dict): # Making a copy to avoid modifying the object with pops @@ -478,7 +477,7 @@ def fetch_sources(self, sources=None, checksums=None): # Single source should be re-wrapped as a list, and checksums with it if isinstance(sources, dict): sources = [sources] - if isinstance(checksums, string_type): + if isinstance(checksums, str): checksums = [checksums] # Loop over the list of sources; list of checksums must match >= in size @@ -626,7 +625,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): # always pass source spec as dict value to fetch_source method, # mostly so we can inject stuff like source URLs - if isinstance(source, string_type): + if isinstance(source, str): source = {'filename': source} elif not isinstance(source, dict): raise EasyBuildError("Incorrect value type for source of extension %s: %s", @@ -655,7 +654,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): src_fn = ext_options.get('source_tmpl') if src_fn is None: src_fn = default_source_tmpl - elif not isinstance(src_fn, string_type): + elif not isinstance(src_fn, str): error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) @@ -727,7 +726,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): exts_sources.append(ext_src) - elif isinstance(ext, string_type): + elif isinstance(ext, str): exts_sources.append({'name': ext}) else: @@ -892,7 +891,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No url_filename = download_filename or filename - if isinstance(url, string_type): + if isinstance(url, str): if url[-1] in ['=', '/']: fullurl = "%s%s" % (url, url_filename) else: @@ -1398,7 +1397,7 @@ def make_module_extra(self, altroot=None, altversion=None): lines.append(self.module_generator.set_environment(key, value)) for (key, value) in self.cfg['modextrapaths'].items(): - if isinstance(value, string_type): + if isinstance(value, str): value = [value] elif not isinstance(value, (tuple, list)): raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple", @@ -1553,7 +1552,7 @@ def make_module_req(self): } for key, reqs in sorted(requirements.items()): - if isinstance(reqs, string_type): + if isinstance(reqs, str): self.log.warning("Hoisting string value %s into a list before iterating over it", reqs) reqs = [reqs] if self.dry_run: @@ -2432,7 +2431,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): # Single source should be re-wrapped as a list, and checksums with it if isinstance(sources, dict): sources = [sources] - if isinstance(checksums, string_type): + if isinstance(checksums, str): checksums = [checksums] if not checksums: @@ -2510,7 +2509,7 @@ def check_checksums(self): for ext in self.cfg['exts_list']: # just skip extensions for which only a name is specified # those are just there to check for things that are in the "standard library" - if not isinstance(ext, string_type): + if not isinstance(ext, str): ext_name = ext[0] # take into account that extension may be a 2-tuple with just name/version ext_opts = ext[2] if len(ext) == 3 else {} @@ -2600,7 +2599,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): if build_option('rpath_override_dirs') is not None: # make sure we have a list rpath_overrides = build_option('rpath_override_dirs') - if isinstance(rpath_overrides, string_type): + if isinstance(rpath_overrides, str): rpath_override_dirs = rpath_overrides.split(':') # Filter out any empty values rpath_override_dirs = list(filter(None, rpath_override_dirs)) @@ -2730,7 +2729,7 @@ def init_ext_instances(self): # obtain name and module path for default extention class exts_defaultclass = self.cfg['exts_defaultclass'] - if isinstance(exts_defaultclass, string_type): + if isinstance(exts_defaultclass, str): # proper way: derive module path from specified class name default_class = exts_defaultclass default_class_modpath = get_module_path(default_class, generic=True) @@ -2907,7 +2906,7 @@ def fix_shebang(self): shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang) fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang] if fix_shebang_for: - if isinstance(fix_shebang_for, string_type): + if isinstance(fix_shebang_for, str): fix_shebang_for = [fix_shebang_for] shebang = '#!%s %s' % (env_for_shebang, lang) @@ -2957,7 +2956,7 @@ def run_post_install_commands(self, commands=None): raise EasyBuildError(error_msg, commands) for cmd in commands: - if not isinstance(cmd, string_type): + if not isinstance(cmd, str): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) @@ -3368,7 +3367,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): for i, command in enumerate(commands): # set command to default. This allows for config files with # non-tuple commands - if isinstance(command, string_type): + if isinstance(command, str): self.log.debug("Using %s as sanity check command" % command) commands[i] = command else: @@ -3531,7 +3530,7 @@ def xs2str(xs): (typ, check_fn) = path_keys_and_check[key] for xs in paths[key]: - if isinstance(xs, string_type): + if isinstance(xs, str): xs = (xs,) elif not isinstance(xs, tuple): raise EasyBuildError("Unsupported type %s encountered in '%s', not a string or tuple", @@ -3905,7 +3904,7 @@ def run_step(self, step, step_methods): for step_method in step_methods: # Remove leading underscore from e.g. "_test_step" - method_name = extract_method_name(step_method).lstrip('_') + method_name = '_'.join(step_method.__code__.co_names).lstrip('_') self.log.info("Running method %s part of step %s", method_name, step) if self.dry_run: diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c0e1810021..90c9a9609d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -47,9 +47,11 @@ import os import re from contextlib import contextmanager +from collections import OrderedDict import easybuild.tools.filetools as filetools from easybuild.base import fancylogger +from easybuild.base.wrapper import create_base_metaclass from easybuild.framework.easyconfig import MANDATORY from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS, EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.default import DEFAULT_CONFIG @@ -73,7 +75,6 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name from easybuild.tools.modules import modules_tool -from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA @@ -234,7 +235,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains subtoolchain_version = None # ensure we always have a tuple of alternative subtoolchain names, which makes things easier below - if isinstance(subtoolchain_names, string_type): + if isinstance(subtoolchain_names, str): subtoolchain_names = (subtoolchain_names,) system_subtoolchain = False @@ -620,7 +621,7 @@ def update(self, key, value, allow_duplicate=True): Update an easyconfig parameter with the specified value (i.e. append to it). Note: For dictionary easyconfig parameters, 'allow_duplicate' is ignored (since it's meaningless). """ - if isinstance(value, string_type): + if isinstance(value, str): inval = [value] elif isinstance(value, (list, dict, tuple)): inval = value @@ -638,7 +639,7 @@ def update(self, key, value, allow_duplicate=True): # Grab current parameter value so we can modify it param_value = copy.deepcopy(self[key]) - if isinstance(param_value, string_type): + if isinstance(param_value, str): for item in inval: # re.search: only add value to string if it's not there yet (surrounded by whitespace) if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), param_value)): @@ -829,7 +830,7 @@ def check_deprecated(self, path): deprecated = self['deprecated'] if deprecated: - if isinstance(deprecated, string_type): + if isinstance(deprecated, str): if 'easyconfig' not in build_option('silence_deprecation_warnings'): depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated)) else: @@ -907,7 +908,7 @@ def validate_os_deps(self): not_found = [] for dep in self['osdependencies']: # make sure we have a tuple - if isinstance(dep, string_type): + if isinstance(dep, str): dep = (dep,) elif not isinstance(dep, tuple): raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)", @@ -1986,7 +1987,7 @@ def resolve_template(value, tmpl_dict): - value: some python object (supported are string, tuple/list, dict or some mix thereof) - tmpl_dict: template dictionary """ - if isinstance(value, string_type): + if isinstance(value, str): # simple escaping, making all '%foo', '%%foo', '%%%foo' post-templates values available, # but ignore a string like '%(name)s' # behaviour of strings like '%(name)s', diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 8abb697ece..e1c2dac2d0 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -41,7 +41,6 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import Section from easybuild.tools.utilities import get_subclasses -from easybuild.tools.py2vs3 import string_type # format is mandatory major.minor @@ -323,7 +322,7 @@ def parse_sections(self, toparse, current): value_type = self.VERSION_OPERATOR_VALUE_TYPES[key] # list of supported toolchains/versions # first one is default - if isinstance(value, string_type): + if isinstance(value, str): # so the split should be unnecessary # (if it's not a list already, it's just one value) # TODO this is annoying. check if we can force this in configobj diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 680e4ee77b..994fc93fed 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -49,7 +49,6 @@ from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import read_file, write_file from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import INDENT_4SPACES, quote_py_str @@ -255,7 +254,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): else: # dependencies are already dumped as strings, so they do not need to be quoted again - if isinstance(param_val, string_type) and param_name not in DEPENDENCY_PARAMETERS: + if isinstance(param_val, str) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index fe3b1a2316..c9f517122b 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -37,7 +37,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type from easybuild.tools.toolchain.utilities import search_toolchain @@ -144,7 +143,7 @@ def test(self, test_version): if not self: raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__) - if isinstance(test_version, string_type): + if isinstance(test_version, str): test_version = self._convert(test_version) elif not isinstance(test_version, EasyVersion): raise EasyBuildError("test: argument should be a string or EasyVersion (type %s)", type(test_version)) @@ -640,7 +639,7 @@ def add(self, versop_new, data=None, update=None): :param update: if versop_new already exist and has data set, try to update the existing data with the new data; instead of overriding the existing data with the new data (method used for updating is .update) """ - if isinstance(versop_new, string_type): + if isinstance(versop_new, str): versop_new = VersionOperator(versop_new) elif not isinstance(versop_new, VersionOperator): raise EasyBuildError("add: argument must be a VersionOperator instance or string: %s; type %s", diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index a9b084cc12..dd53f48890 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -40,7 +40,6 @@ from easybuild.framework.easyconfig.types import PARAMETER_TYPES, check_type_of_param_value from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file -from easybuild.tools.py2vs3 import string_type # deprecated easyconfig parameters, and their replacements @@ -160,7 +159,7 @@ def _read(self, filename=None): except IOError as err: raise EasyBuildError('Failed to obtain content with %s: %s', self.get_fn, err) - if not isinstance(self.rawcontent, string_type): + if not isinstance(self.rawcontent, str): msg = 'rawcontent is not a string: type %s, content %s' % (type(self.rawcontent), self.rawcontent) raise EasyBuildError("Unexpected result for raw content: %s", msg) diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index 0015b0c9b9..10139118e9 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -32,11 +32,11 @@ """ import re import sys +from importlib import reload from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.utilities import only_if_module_is_available try: @@ -161,7 +161,7 @@ def cmdline_easyconfigs_style_check(ecs): # if an EasyConfig instance is provided, just grab the corresponding file path if isinstance(ec, EasyConfig): path = ec.path - elif isinstance(ec, string_type): + elif isinstance(ec, str): path = ec else: raise EasyBuildError("Value of unknown type encountered in cmdline_easyconfigs_style_check: %s (type: %s)", diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 603423c20b..897f12b34d 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -38,7 +38,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type from easybuild.tools.systemtools import get_shared_lib_ext, pick_dep_version from easybuild.tools.config import build_option @@ -306,7 +305,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: raise EasyBuildError("Unexpected type for dependency: %s", dep) - if isinstance(dep_name, string_type) and dep_version: + if isinstance(dep_name, str) and dep_version: pref = name_to_prefix.get(dep_name.lower()) if pref: dep_version = pick_dep_version(dep_version) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 48dc1910b0..fdcafa1c2a 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -45,6 +45,7 @@ import re import sys import tempfile +from collections import OrderedDict from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR @@ -62,7 +63,6 @@ from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_pr from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import only_if_module_is_available, quote_str diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 41c688355b..b812f54cad 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -60,7 +60,6 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.py2vs3 import string_type from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig, search_easyconfigs from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES @@ -346,7 +345,7 @@ def __repr__(self): '"None"': 'None', } for (key, val) in tweaks.items(): - if isinstance(val, string_type) and val in special_values: + if isinstance(val, str) and val in special_values: str_val = val val = special_values[val] else: @@ -1205,7 +1204,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin tc_candidate = fetch_parameters_from_easyconfig(read_file(path), ['toolchain'])[0] if isinstance(tc_candidate, dict) and tc_candidate['name'] == SYSTEM_TOOLCHAIN_NAME: cand_paths_filtered += [path] - if isinstance(tc_candidate, string_type) and tc_candidate == TC_CONSTANT_SYSTEM: + if isinstance(tc_candidate, str) and tc_candidate == TC_CONSTANT_SYSTEM: cand_paths_filtered += [path] cand_paths = cand_paths_filtered diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 622479f480..fa9829e312 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -37,7 +37,6 @@ from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type _log = fancylogger.getLogger('easyconfig.types', fname=False) @@ -272,7 +271,7 @@ def to_toolchain_dict(spec): :param spec: a comma-separated string with two or three values, or a 2/3-element list of strings, or a dict """ # check if spec is a string or a list of two values; else, it can not be converted - if isinstance(spec, string_type): + if isinstance(spec, str): spec = spec.split(',') if isinstance(spec, (list, tuple)): @@ -314,11 +313,11 @@ def to_list_of_strings(value): res = None # if value is already of correct type, we don't need to change anything - if isinstance(value, list) and all(isinstance(s, string_type) for s in value): + if isinstance(value, list) and all(isinstance(s, str) for s in value): res = value - elif isinstance(value, string_type): + elif isinstance(value, str): res = [value] - elif isinstance(value, tuple) and all(isinstance(s, string_type) for s in value): + elif isinstance(value, tuple) and all(isinstance(s, str) for s in value): res = list(value) else: raise EasyBuildError("Don't know how to convert provided value to a list of strings: %s", value) @@ -341,7 +340,7 @@ def to_list_of_strings_and_tuples(spec): raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec)) for elem in spec: - if isinstance(elem, (string_type, tuple)): + if isinstance(elem, (str, tuple)): str_tup_list.append(elem) elif isinstance(elem, list): str_tup_list.append(tuple(elem)) @@ -366,7 +365,7 @@ def to_list_of_strings_and_tuples_and_dicts(spec): raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec)) for elem in spec: - if isinstance(elem, (string_type, tuple, dict)): + if isinstance(elem, (str, tuple, dict)): str_tup_list.append(elem) elif isinstance(elem, list): str_tup_list.append(tuple(elem)) @@ -392,17 +391,17 @@ def to_sanity_check_paths_entry(spec): raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec)) for elem in spec: - if isinstance(elem, (string_type, tuple)): + if isinstance(elem, (str, tuple)): result.append(elem) elif isinstance(elem, list): result.append(tuple(elem)) elif isinstance(elem, dict): for key, value in elem.items(): - if not isinstance(key, string_type): + if not isinstance(key, str): raise EasyBuildError("Expected keys to be of type string, got %s (%s)", key, type(key)) elif isinstance(value, list): elem[key] = tuple(value) - elif not isinstance(value, (string_type, tuple)): + elif not isinstance(value, (str, tuple)): raise EasyBuildError("Expected elements to be of type string, tuple or list, got %s (%s)", value, type(value)) result.append(elem) @@ -515,11 +514,11 @@ def to_checksums(checksums): # * a tuple with 2 elements: checksum type + checksum value # * a list of checksums (i.e. multiple checksums for a single file) # * a dict (filename to checksum mapping) - if isinstance(checksum, string_type): + if isinstance(checksum, str): res.append(checksum) elif isinstance(checksum, (list, tuple)): # 2 elements + only string/int values => a checksum tuple - if len(checksum) == 2 and all(isinstance(x, (string_type, int)) for x in checksum): + if len(checksum) == 2 and all(isinstance(x, (str, int)) for x in checksum): res.append(tuple(checksum)) else: res.append(to_checksums(checksum)) @@ -543,9 +542,9 @@ def ensure_iterable_license_specs(specs): """ if specs is None: license_specs = [None] - elif isinstance(specs, string_type): + elif isinstance(specs, str): license_specs = [specs] - elif isinstance(specs, (list, tuple)) and all(isinstance(x, string_type) for x in specs): + elif isinstance(specs, (list, tuple)) and all(isinstance(x, str) for x in specs): license_specs = list(specs) else: msg = "Unsupported type %s for easyconfig parameter 'license_file'! " % type(specs) @@ -620,25 +619,25 @@ def ensure_iterable_license_specs(specs): STRING_OR_TUPLE_DICT, STRING_OR_TUPLE_OR_DICT_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS] # easy types, that can be verified with isinstance -EASY_TYPES = [string_type, bool, dict, int, list, str, tuple] +EASY_TYPES = [str, bool, dict, int, list, str, tuple] # type checking is skipped for easyconfig parameters names not listed in PARAMETER_TYPES PARAMETER_TYPES = { 'checksums': CHECKSUMS, 'docurls': LIST_OF_STRINGS, - 'name': string_type, + 'name': str, 'osdependencies': STRING_OR_TUPLE_LIST, 'patches': STRING_OR_TUPLE_OR_DICT_LIST, 'sanity_check_paths': SANITY_CHECK_PATHS_DICT, 'toolchain': TOOLCHAIN_DICT, - 'version': string_type, + 'version': str, } # add all dependency types as dependencies for dep in DEPENDENCY_PARAMETERS: PARAMETER_TYPES[dep] = DEPENDENCIES TYPE_CONVERSION_FUNCTIONS = { - string_type: str, + str: str, float: float, int: int, str: str, diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 9296c18827..9ab411c918 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -37,7 +37,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import only_if_module_is_available try: import yaml @@ -53,7 +52,7 @@ def check_value(value, context): Check whether specified value obtained from a YAML file in specified context represents is valid. The value must be a string (not a float or an int). """ - if not isinstance(value, string_type): + if not isinstance(value, str): error_msg = '\n'.join([ "Value %(value)s (of type %(type)s) obtained for %(context)s is not valid!", "Make sure to wrap the value in single quotes (like '%(value)s') to avoid that it is interpreted " diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index ed90f15e6d..8dc4669392 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -43,7 +43,6 @@ from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir from easybuild.tools.run import check_async_cmd, run_cmd -from easybuild.tools.py2vs3 import string_type def resolve_exts_filter_template(exts_filter, ext): @@ -54,7 +53,7 @@ def resolve_exts_filter_template(exts_filter, ext): :return: (cmd, input) as a tuple of strings """ - if isinstance(exts_filter, string_type) or len(exts_filter) != 2: + if isinstance(exts_filter, str) or len(exts_filter) != 2: raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")') cmd, cmdinput = exts_filter diff --git a/easybuild/main.py b/easybuild/main.py index 319c80b5ae..f657b8c091 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -77,7 +77,6 @@ from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs -from easybuild.tools.py2vs3 import python2_is_deprecated from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state @@ -603,9 +602,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args - if 'python2' not in build_option('silence_deprecation_warnings'): - python2_is_deprecated() - global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, from_pr_list, tweaked_ecs_paths) = cfg_settings diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index cb2d6fed04..5cb9eac8f5 100755 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -29,6 +29,7 @@ import re +from urllib.request import HTTPError, URLError from easybuild.base import fancylogger from easybuild.base.generaloption import simple_option @@ -37,7 +38,6 @@ from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions -from easybuild.tools.py2vs3 import HTTPError, URLError HTTP_DELETE_OK = 204 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index e99ddb4af9..9cd2f0c88c 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -31,8 +31,8 @@ * Stijn De Weirdt (Ghent University) """ import time +from collections import OrderedDict from easybuild.tools.filetools import det_size -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.systemtools import get_system_info from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3f8acec4f5..88571ed37c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -45,11 +45,12 @@ import tempfile import time from abc import ABCMeta +from string import ascii_letters from easybuild.base import fancylogger from easybuild.base.frozendict import FrozenDictKnownKeys +from easybuild.base.wrapper import create_base_metaclass from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type try: import rich # noqa @@ -522,7 +523,7 @@ def init(options, config_options_dict): # make sure source path is a list sourcepath = tmpdict['sourcepath'] - if isinstance(sourcepath, string_type): + if isinstance(sourcepath, str): tmpdict['sourcepath'] = sourcepath.split(':') _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath'])) elif not isinstance(sourcepath, (tuple, list)): diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 48c7dd348d..8163726280 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -27,8 +27,6 @@ from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE -from easybuild.tools.py2vs3 import string_type - # imported lazily to avoid startup performance hit if it isn't used compiler = None @@ -562,11 +560,11 @@ def __getitem__(self, key): """Fetch the item and do string interpolation.""" val = dict.__getitem__(self, key) if self.main.interpolation: - if isinstance(val, string_type): + if isinstance(val, str): return self._interpolate(key, val) if isinstance(val, list): def _check(entry): - if isinstance(entry, string_type): + if isinstance(entry, str): return self._interpolate(key, entry) return entry new = [_check(entry) for entry in val] @@ -588,7 +586,7 @@ def __setitem__(self, key, value, unrepr=False): ``unrepr`` must be set when setting a value to a dictionary, without creating a new sub-section. """ - if not isinstance(key, (bytes, string_type)): + if not isinstance(key, (bytes, str)): raise ValueError('The key "%s" is not a string.' % key) # add the comment @@ -622,11 +620,11 @@ def __setitem__(self, key, value, unrepr=False): if key not in self: self.scalars.append(key) if not self.main.stringify: - if isinstance(value, string_type): + if isinstance(value, str): pass elif isinstance(value, (list, tuple)): for entry in value: - if not isinstance(entry, string_type): + if not isinstance(entry, str): raise TypeError('Value is not a string "%s".' % entry) else: raise TypeError('Value is not a string "%s".' % value) @@ -946,7 +944,7 @@ def as_bool(self, key): return val else: try: - if not isinstance(val, string_type): + if not isinstance(val, str): # TODO: Why do we raise a KeyError here? raise KeyError() else: @@ -1210,7 +1208,7 @@ def __init__(self, infile=None, options=None, configspec=None, encoding=None, self._load(infile, configspec) def _load(self, infile, configspec): - if isinstance(infile, string_type): + if isinstance(infile, str): self.filename = infile if os.path.isfile(infile): with open(infile, 'r') as fh: @@ -1431,7 +1429,7 @@ def _handle_bom(self, infile): else: infile = newline # UTF8 - don't decode - if isinstance(infile, string_type): + if isinstance(infile, str): return infile.splitlines(True) else: return infile @@ -1439,7 +1437,7 @@ def _handle_bom(self, infile): return self._decode(infile, encoding) # No BOM discovered and no encoding specified, just return - if isinstance(infile, (bytes, string_type)): + if isinstance(infile, (bytes, str)): # infile read from a file will be a single string return infile.splitlines(True) return infile @@ -1457,7 +1455,7 @@ def _decode(self, infile, encoding): if is a string, it also needs converting to a list. """ - if isinstance(infile, string_type): + if isinstance(infile, str): # can't be unicode # NOTE: Could raise a ``UnicodeDecodeError`` return infile.decode(encoding).splitlines(True) @@ -1482,7 +1480,7 @@ def _str(self, value): Used by ``stringify`` within validate, to turn non-string values into strings. """ - if not isinstance(value, string_type): + if not isinstance(value, str): return str(value) else: return value @@ -1730,7 +1728,7 @@ def _quote(self, value, multiline=True): return self._quote(value[0], multiline=False) + ',' return ', '.join([self._quote(val, multiline=False) for val in value]) - if not isinstance(value, string_type): + if not isinstance(value, str): if self.stringify: value = str(value) else: @@ -2275,7 +2273,7 @@ def reload(self): This method raises a ``ReloadError`` if the ConfigObj doesn't have a filename attribute pointing to a file. """ - if not isinstance(self.filename, string_type): + if not isinstance(self.filename, str): raise ReloadError() filename = self.filename diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index dc9cbb39a1..41ec9829c7 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -41,7 +41,6 @@ from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.filetools import read_file, remove_file, which from easybuild.tools.run import run_cmd -from easybuild.tools.py2vs3 import string_type ARCH = 'arch' # Arch Linux @@ -299,7 +298,7 @@ def resolve_template_data(self): install_os_deps = [] for osdep in osdeps: - if isinstance(osdep, string_type): + if isinstance(osdep, str): install_os_deps.append("yum install --quiet --assumeyes %s" % osdep) # tuple entry indicates multiple options elif isinstance(osdep, tuple): diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index 65be9dadf5..e01a117427 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -36,7 +36,6 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import which -from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd @@ -49,7 +48,7 @@ def det_os_deps(easyconfigs): res = set() os_deps = reduce(operator.add, [obj['ec']['osdependencies'] for obj in easyconfigs], []) for os_dep in os_deps: - if isinstance(os_dep, string_type): + if isinstance(os_dep, str): res.add(os_dep) elif isinstance(os_dep, tuple): res.update(os_dep) diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index f085590988..9a675850f9 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -35,7 +35,6 @@ from easybuild.base import fancylogger from easybuild.base.wrapper import Wrapper from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type _log = fancylogger.getLogger('tools.convert', fname=False) @@ -52,7 +51,7 @@ def __init__(self, obj): """Support the conversion of obj to something""" self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__, fname=False) self.__dict__['data'] = None - if isinstance(obj, string_type): + if isinstance(obj, str): self.data = self._from_string(obj) else: raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 1fcc6ece83..7e048d4529 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -39,7 +39,9 @@ import copy import inspect import os +from collections import OrderedDict from easybuild.tools import LooseVersion +from string import ascii_lowercase from easybuild.base import fancylogger from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories @@ -59,7 +61,6 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file from easybuild.tools.modules import modules_tool -from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index daa143b46c..86ba0f796c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -58,6 +58,8 @@ import time import zlib from functools import partial +from html.parser import HTMLParser +import urllib.request as std_urllib from easybuild.base import fancylogger from easybuild.tools import run @@ -66,7 +68,6 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -393,7 +394,7 @@ def remove(paths): :param paths: path(s) to remove """ - if isinstance(paths, string_type): + if isinstance(paths, str): paths = [paths] _log.info("Removing %d files & directories", len(paths)) @@ -1274,7 +1275,7 @@ def verify_checksum(path, checksums): # Set to None and allow to fail elsewhere checksum = None - if isinstance(checksum, string_type): + if isinstance(checksum, str): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) if len(checksum) == 64: typ = CHECKSUM_TYPE_SHA256 @@ -1318,7 +1319,7 @@ def verify_checksum(path, checksums): def is_sha256_checksum(value): """Check whether provided string is a SHA256 checksum.""" res = False - if isinstance(value, string_type): + if isinstance(value, str): if re.match('^[0-9a-f]{64}$', value): res = True _log.debug("String value '%s' has the correct format to be a SHA256 checksum", value) @@ -1490,7 +1491,7 @@ def create_patch_info(patch_spec): # string value as patch argument can be either path where patch should be applied, # or path to where a non-patch file should be copied - elif isinstance(patch_arg, string_type): + elif isinstance(patch_arg, str): if patch_spec[0].endswith('.patch'): patch_info['sourcepath'] = patch_arg # non-patch files are assumed to be files to copy @@ -1500,7 +1501,7 @@ def create_patch_info(patch_spec): raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", str(patch_spec)) - elif isinstance(patch_spec, string_type): + elif isinstance(patch_spec, str): validate_patch_spec(patch_spec) patch_info = {'name': patch_spec} elif isinstance(patch_spec, dict): @@ -1651,7 +1652,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)', on_missing_match, ', '.join(allowed_values)) - if isinstance(paths, string_type): + if isinstance(paths, str): paths = [paths] # only report when in 'dry run' mode @@ -2326,7 +2327,7 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None): # always consider $LM_LICENSE_FILE lic_env_vars = ['LM_LICENSE_FILE'] - if isinstance(custom_env_vars, string_type): + if isinstance(custom_env_vars, str): lic_env_vars.insert(0, custom_env_vars) elif custom_env_vars is not None: lic_env_vars = custom_env_vars + lic_env_vars @@ -2581,7 +2582,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, string_type): + if isinstance(paths, str): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index bc3a6b3e27..14c0772996 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -45,6 +45,8 @@ import tempfile import time from datetime import datetime, timedelta +from string import ascii_letters +from urllib.request import HTTPError, URLError, urlopen from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR @@ -57,7 +59,6 @@ from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_framework_files from easybuild.tools.filetools import det_patched_files, download_file, extract_file from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file -from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version from easybuild.tools.utilities import nub, only_if_module_is_available diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 33ff4659d4..618e83e916 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -48,7 +48,6 @@ from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import get_subclasses, quote_str @@ -215,7 +214,7 @@ def _filter_paths(self, key, paths): added_paths = self.added_paths_per_key.setdefault(key, set()) # paths can be a string - if isinstance(paths, string_type): + if isinstance(paths, str): if paths in added_paths: filtered_paths = None else: @@ -388,7 +387,7 @@ def is_loaded(self, mod_names): :param mod_names: (list of) module name(s) to check load status for """ - if isinstance(mod_names, string_type): + if isinstance(mod_names, str): res = self.IS_LOADED_TEMPLATE % mod_names else: res = [self.IS_LOADED_TEMPLATE % m for m in mod_names] @@ -416,7 +415,7 @@ def unpack_setenv_value(self, env_var_name, env_var_val): use_pushenv = False # value may be specified as a string, or as a dict for special cases - if isinstance(env_var_val, string_type): + if isinstance(env_var_val, str): value = env_var_val elif isinstance(env_var_val, dict): @@ -630,7 +629,7 @@ def _generate_extensions_list(self): for ext in exts_list: if isinstance(ext, tuple): exts_ver_list.append('%s/%s' % (ext[0], ext[1])) - elif isinstance(ext, string_type): + elif isinstance(ext, str): exts_ver_list.append(ext) return sorted(exts_ver_list, key=str.lower) @@ -788,7 +787,7 @@ def conditional_statement(self, conditions, body, negative=False, else_body=None :param cond_or: combine multiple conditions using 'or' (default is to combine with 'and') :param cond_tmpl: template for condition expression (default: '%s') """ - if isinstance(conditions, string_type): + if isinstance(conditions, str): conditions = [conditions] if cond_or: @@ -985,7 +984,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key) return '' - if isinstance(paths, string_type): + if isinstance(paths, str): self.log.debug("Wrapping %s into a list before using it to %s path %s", paths, update_type, key) paths = [paths] @@ -1233,7 +1232,7 @@ def conditional_statement(self, conditions, body, negative=False, else_body=None :param cond_or: combine multiple conditions using 'or' (default is to combine with 'and') :param cond_tmpl: template for condition expression (default: '%s') """ - if isinstance(conditions, string_type): + if isinstance(conditions, str): conditions = [conditions] if cond_or: @@ -1456,7 +1455,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key) return '' - if isinstance(paths, string_type): + if isinstance(paths, str): self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key) paths = [paths] diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 487dbac063..0da1c142e3 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -33,9 +33,9 @@ import re from easybuild.base import fancylogger +from easybuild.base.wrapper import create_base_metaclass from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import Singleton -from easybuild.tools.py2vs3 import create_base_metaclass DEVEL_MODULE_SUFFIX = '-easybuild-devel' diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 8e8f3e919e..331a24eb05 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -40,7 +40,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme -from easybuild.tools.py2vs3 import string_type from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.utilities import get_subclasses, import_available_modules @@ -64,12 +63,12 @@ def det_full_ec_version(ec): # prepend/append version prefix/suffix versionprefix = ec.get('versionprefix', '') - if not isinstance(versionprefix, string_type): + if not isinstance(versionprefix, str): raise EasyBuildError("versionprefix value should be a string, found '%s': %s (full spec: %s)", type(versionprefix).__name__, versionprefix, ec) versionsuffix = ec.get('versionsuffix', '') - if not isinstance(versionsuffix, string_type): + if not isinstance(versionsuffix, str): raise EasyBuildError("versionsuffix value should be a string, found '%s': %s (full spec: %s)", type(versionsuffix).__name__, versionsuffix, ec) @@ -94,7 +93,7 @@ def avail_module_naming_schemes(): def is_valid_module_name(mod_name): """Check whether the specified value is a valid module name.""" # module name must be a string - if not isinstance(mod_name, string_type): + if not isinstance(mod_name, str): _log.warning("Wrong type for module name %s (%s), should be a string" % (mod_name, type(mod_name))) return False # module name must be relative path diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..9037be6ea8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -50,8 +50,7 @@ from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX -from easybuild.tools.py2vs3 import subprocess_popen_text -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_cmd, subprocess_popen_text from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 100fd6a72f..49f6cc0de7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -45,6 +45,7 @@ import sys import tempfile import pwd +from collections import OrderedDict import easybuild.tools.environment as env from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog @@ -96,7 +97,6 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod -from easybuild.tools.py2vs3 import OrderedDict, string_type from easybuild.tools.robot import det_robot_path from easybuild.tools.run import run_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes @@ -1103,7 +1103,7 @@ def _ensure_abs_path(self, opt_name): opt_val = getattr(self.options, opt_name) if opt_val: - if isinstance(opt_val, string_type): + if isinstance(opt_val, str): setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val)) elif isinstance(opt_val, list): abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val] @@ -1826,7 +1826,7 @@ def parse_external_modules_metadata(cfgs): unknown_keys.setdefault(mod, []).append(key) for key in ['name', 'version']: - if isinstance(entry.get(key), string_type): + if isinstance(entry.get(key), str): entry[key] = [entry[key]] _log.debug("Transformed external module metadata value %s for %s into a single-value list: %s", key, mod, entry[key]) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 0dc5b3b362..c10b169e57 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -32,10 +32,10 @@ * Jørgen Nordmoen (University of Oslo) """ import functools +from collections import OrderedDict from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style -from easybuild.tools.py2vs3 import OrderedDict try: from rich.console import Console, Group diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 087084c687..9a695e33da 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -39,12 +39,12 @@ import pprint from easybuild.base import fancylogger +from easybuild.base.wrapper import create_base_metaclass from easybuild.tools.config import PKG_TOOL_FPM, PKG_TYPE_RPM, Singleton from easybuild.tools.config import build_option, get_package_naming_scheme, log_path from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, which from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme -from easybuild.tools.py2vs3 import create_base_metaclass from easybuild.tools.run import run_cmd from easybuild.tools.utilities import get_subclasses, import_available_modules diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py index 5d9854b39f..4bd0e47191 100644 --- a/easybuild/tools/py2vs3/__init__.py +++ b/easybuild/tools/py2vs3/__init__.py @@ -24,18 +24,16 @@ # import sys -# all functionality provided by the py2 and py3 modules is made available via the easybuild.tools.py2vs3 namespace -if sys.version_info[0] >= 3: - from easybuild.tools.py2vs3.py3 import * # noqa -else: - from easybuild.tools.py2vs3.py2 import * # noqa +from easybuild.base import fancylogger +from easybuild.base.wrapper import create_base_metaclass # noqa -# based on six's 'with_metaclass' function -# see also https://stackoverflow.com/questions/18513821/python-metaclass-understanding-the-with-metaclass -def create_base_metaclass(base_class_name, metaclass, *bases): - """Create new class with specified metaclass based on specified base class(es).""" - return metaclass(base_class_name, bases, {}) +# all functionality provided by the py3 modules is made available via the easybuild.tools.py2vs3 namespace +from easybuild.tools.py2vs3.py3 import * # noqa + + +_log = fancylogger.getLogger('py2vs3', fname=False) +_log.deprecated("Using py2vs3 is deprecated, since EasyBuild no longer runs on Python 2.", '6.0') def python2_is_deprecated(): diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py deleted file mode 100644 index bd7e164e39..0000000000 --- a/easybuild/tools/py2vs3/py2.py +++ /dev/null @@ -1,111 +0,0 @@ -# -# Copyright 2019-2023 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# -""" -Functionality to facilitate keeping code compatible with Python 2 & Python 3. - -Implementations for Python 2. - -Authors: - -* Kenneth Hoste (Ghent University) -""" -# these are not used here, but imported from here in other places -import ConfigParser as configparser # noqa -import json -import subprocess -import time -import urllib2 as std_urllib # noqa -from collections import Mapping, OrderedDict # noqa -from HTMLParser import HTMLParser # noqa -from string import letters as ascii_letters # noqa -from string import lowercase as ascii_lowercase # noqa -from StringIO import StringIO # noqa -from urllib import urlencode # noqa -from urllib2 import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa - -# Use the safe version. In Python 3.2+ this is the default already -ConfigParser = configparser.SafeConfigParser - - -# reload function (built-in in Python 2) -reload = reload - -# string type that can be used in 'isinstance' calls -string_type = basestring - -# trivial wrapper for json.loads (Python 3 version is less trivial) -json_loads = json.loads - - -def subprocess_popen_text(cmd, **kwargs): - """Call subprocess.Popen with specified named arguments.""" - kwargs.setdefault('stderr', subprocess.PIPE) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs) - - -def subprocess_terminate(proc, timeout): - """Terminate the subprocess if it hasn't finished after the given timeout""" - res = None - for pipe in (proc.stdout, proc.stderr, proc.stdin): - if pipe: - pipe.close() - while timeout > 0: - res = proc.poll() - if res is not None: - break - delay = min(timeout, 0.1) - time.sleep(delay) - timeout -= delay - if res is None: - proc.terminate() - - -def raise_with_traceback(exception_class, message, traceback): - """Raise exception of specified class with given message and traceback.""" - raise exception_class, message, traceback # noqa: E999 - - -def extract_method_name(method_func): - """Extract method name from lambda function.""" - return '_'.join(method_func.func_code.co_names) - - -def mk_wrapper_baseclass(metaclass): - - class WrapperBase(object): - """ - Wrapper class that provides proxy access to an instance of some internal instance. - """ - __metaclass__ = metaclass - __wraps__ = None - - return WrapperBase - - -def sort_looseversions(looseversions): - """Sort list of (values including) LooseVersion instances.""" - # with Python 2, we can safely use 'sorted' on LooseVersion instances - # (but we can't in Python 3, see https://bugs.python.org/issue14894) - return sorted(looseversions) diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 91a5342075..b81e549013 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -34,7 +34,6 @@ # these are not used here, but imported from here in other places import configparser # noqa import json -import subprocess import sys import urllib.request as std_urllib # noqa from collections import OrderedDict # noqa @@ -59,6 +58,9 @@ except ImportError: HAVE_DISTUTILS = False +from easybuild.base.wrapper import mk_wrapper_baseclass # noqa +from easybuild.tools.run import subprocess_popen_text, subprocess_terminate # noqa + # string type that can be used in 'isinstance' calls string_type = str @@ -75,24 +77,6 @@ def json_loads(body): return json.loads(body) -def subprocess_popen_text(cmd, **kwargs): - """Call subprocess.Popen in text mode with specified named arguments.""" - # open stdout/stderr in text mode in Popen when using Python 3 - kwargs.setdefault('stderr', subprocess.PIPE) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, **kwargs) - - -def subprocess_terminate(proc, timeout): - """Terminate the subprocess if it hasn't finished after the given timeout""" - try: - proc.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - for pipe in (proc.stdout, proc.stderr, proc.stdin): - if pipe: - pipe.close() - proc.terminate() - - def raise_with_traceback(exception_class, message, traceback): """Raise exception of specified class with given message and traceback.""" raise exception_class(message).with_traceback(traceback) @@ -103,17 +87,6 @@ def extract_method_name(method_func): return '_'.join(method_func.__code__.co_names) -def mk_wrapper_baseclass(metaclass): - - class WrapperBase(object, metaclass=metaclass): - """ - Wrapper class that provides proxy access to an instance of some internal instance. - """ - __wraps__ = None - - return WrapperBase - - def safe_cmp_looseversions(v1, v2): """Safe comparison function for two (values containing) LooseVersion instances.""" diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index cbd33ab654..743a82bc36 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -38,7 +38,6 @@ """ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import get_subclasses, import_available_modules _log = fancylogger.getLogger('repository', fname=False) @@ -159,10 +158,10 @@ def init_repository(repository, repository_path): inited_repo = None if isinstance(repository, Repository): inited_repo = repository - elif isinstance(repository, string_type): + elif isinstance(repository, str): repo = avail_repositories().get(repository) try: - if isinstance(repository_path, string_type): + if isinstance(repository_path, str): inited_repo = repo(repository_path) elif isinstance(repository_path, (tuple, list)) and len(repository_path) <= 2: inited_repo = repo(*repository_path) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c87caec419..7ce358ca89 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -50,7 +50,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option -from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import trace_msg @@ -84,7 +83,7 @@ def cache_aware_func(cmd, *args, **kwargs): # cache key is combination of command and input provided via stdin key = (cmd, kwargs.get('inp', None)) # fetch from cache if available, cache it if it's not, but only on cmd strings - if isinstance(cmd, string_type) and key in cache: + if isinstance(cmd, str) and key in cache: _log.debug("Using cached value for command '%s': %s", cmd, cache[key]) return cache[key] else: @@ -151,7 +150,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True """ cwd = os.getcwd() - if isinstance(cmd, string_type): + if isinstance(cmd, str): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) @@ -228,7 +227,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if isinstance(cmd, list): exec_cmd = None cmd.insert(0, '/usr/bin/env') - elif isinstance(cmd, string_type): + elif isinstance(cmd, str): cmd = '/usr/bin/env %s' % cmd else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) @@ -372,7 +371,7 @@ def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, re """ cwd = os.getcwd() - if not isinstance(cmd, string_type) and len(cmd) > 1: + if not isinstance(cmd, str) and len(cmd) > 1: # We use shell=True and hence we should really pass the command as a string # When using a list then every element past the first is passed to the shell itself, not the command! raise EasyBuildError("The command passed must be a string!") @@ -447,7 +446,7 @@ def process_QA(q, a_s): def check_answers_list(answers): """Make sure we have a list of answers (as strings).""" - if isinstance(answers, string_type): + if isinstance(answers, str): answers = [answers] elif not isinstance(answers, list): if cmd_log: @@ -717,7 +716,7 @@ def extract_errors_from_log(log_txt, reg_exps): actions = (IGNORE, WARN, ERROR) # promote single string value to list, since code below expects a list - if isinstance(reg_exps, string_type): + if isinstance(reg_exps, str): reg_exps = [reg_exps] re_tuples = [] @@ -770,3 +769,21 @@ def check_log_for_errors(log_txt, reg_exps): if errors: raise EasyBuildError("Found %s error(s) in command output (output: %s)", len(errors), "\n\t".join(errors)) + + +def subprocess_popen_text(cmd, **kwargs): + """Call subprocess.Popen in text mode with specified named arguments.""" + # open stdout/stderr in text mode in Popen when using Python 3 + kwargs.setdefault('stderr', subprocess.PIPE) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, **kwargs) + + +def subprocess_terminate(proc, timeout): + """Terminate the subprocess if it hasn't finished after the given timeout""" + try: + proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + for pipe in (proc.stdout, proc.stderr, proc.stdin): + if pipe: + pipe.close() + proc.terminate() diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index cafef4ba73..2d58ae1f55 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -42,9 +42,10 @@ import sys import termios import warnings +from collections import OrderedDict from ctypes.util import find_library from socket import gethostname -from easybuild.tools.py2vs3 import subprocess_popen_text +from easybuild.tools.run import subprocess_popen_text # pkg_resources is provided by the setuptools Python package, # which we really want to keep as an *optional* dependency @@ -64,7 +65,6 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import OrderedDict, string_type from easybuild.tools.run import run_cmd @@ -1024,12 +1024,12 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None) if required_patterns is None: required_regexs = [] else: - required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns] + required_regexs = [re.compile(p) if isinstance(p, str) else p for p in required_patterns] if banned_patterns is None: banned_regexs = [] else: - banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns] + banned_regexs = [re.compile(p) if isinstance(p, str) else p for p in banned_patterns] # resolve symbolic links (unless they're broken) if os.path.islink(path) and os.path.exists(path): @@ -1311,7 +1311,7 @@ def pick_dep_version(dep_version): result = None else: result = pick_system_specific_value("version", dep_version) - if not isinstance(result, string_type) and result is not False: + if not isinstance(result, str) and result is not False: typ = type(dep_version) raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 32634032fc..2eb36b6a6e 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -34,7 +34,6 @@ from easybuild.tools import systemtools from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.py2vs3 import string_type from easybuild.tools.toolchain.constants import COMPILER_VARIABLES from easybuild.tools.toolchain.toolchain import Toolchain @@ -311,7 +310,7 @@ def _set_optimal_architecture(self, default_optarch=None): (--optarch and --optarch=GENERIC still override this value) """ ec_optarch = self.options.get('optarch', False) - if isinstance(ec_optarch, string_type): + if isinstance(ec_optarch, str): if OPTARCH_MAP_CHAR in ec_optarch: error_msg = "When setting optarch in the easyconfig (found %s), " % ec_optarch error_msg += "the syntax is not allowed. " % OPTARCH_MAP_CHAR @@ -341,7 +340,7 @@ def _set_optimal_architecture(self, default_optarch=None): use_generic = False if optarch is not None: # optarch has been parsed as a simple string - if isinstance(optarch, string_type): + if isinstance(optarch, str): if optarch == OPTARCH_GENERIC: use_generic = True else: diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 06dcf9ee5d..6b2c00c95d 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -39,7 +39,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type class ToolchainOptions(dict): @@ -105,7 +104,7 @@ def option(self, name, templatedict=None): 'value': value, }) - if isinstance(res, string_type): + if isinstance(res, str): # allow for template res = self.options_map[name] % templatedict elif isinstance(res, (list, tuple,)): diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index b879a4005c..1b33dd24f2 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -34,12 +34,11 @@ import os import re import sys -from string import digits +from string import ascii_letters, digits from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from easybuild.tools.py2vs3 import ascii_letters, string_type _log = fancylogger.getLogger('tools.utilities') @@ -72,7 +71,7 @@ def quote_str(val, escape_newline=False, prefer_single_quotes=False, escape_back :param tcl: Boolean for whether we are quoting for Tcl syntax """ - if isinstance(val, string_type): + if isinstance(val, str): # escape backslashes if escape_backslash: val = val.replace('\\', '\\\\') @@ -164,7 +163,7 @@ def only_if_module_is_available(modnames, pkgname=None, url=None): if pkgname and url is None: url = 'https://pypi.python.org/pypi/%s' % pkgname - if isinstance(modnames, string_type): + if isinstance(modnames, str): modnames = (modnames,) def wrap(orig): diff --git a/eb b/eb index d214d82b34..402bb87d0a 100755 --- a/eb +++ b/eb @@ -32,6 +32,7 @@ # @author: Kenneth Hoste (Ghent University) # @author: Pieter De Baets (Ghent University) # @author: Jens Timmerman (Ghent University) +# @author: Simon Branford (University of Birmingham) keyboard_interrupt() { echo "Keyboard interrupt!" @@ -40,9 +41,8 @@ keyboard_interrupt() { trap keyboard_interrupt SIGINT -# Python 2.6+ or 3.5+ required -REQ_MIN_PY2VER=6 -REQ_MIN_PY3VER=5 +# Python 3.6+ required +REQ_MIN_PY3VER=6 EASYBUILD_MAIN='easybuild.main' @@ -60,7 +60,7 @@ PYTHON= # - EB_INSTALLPYTHON is set when EasyBuild is installed as a module (by EasyBuild). It is set to the PYTHON # used during that installation (for example, you could override PYTHON using EB_PYTHON at installation # time, this variable preserves that choice). -for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python' 'python3' 'python2'; do +for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python3' 'python'; do # Only consider non-empty values, i.e. continue if e.g. $EB_PYTHON is not set [ -n "${python_cmd}" ] || continue @@ -76,10 +76,7 @@ for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python' 'python3' 'pytho pyver_maj=$(echo "${pyver}" | cut -f1 -d'.') pyver_min=$(echo "${pyver}" | cut -f2 -d'.') - if [ "${pyver_maj}" -eq 2 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY2VER}" ]; then - verbose "'${python_cmd}' version: ${pyver}, which matches Python 2 version requirement (>= 2.${REQ_MIN_PY2VER})" - PYTHON="${python_cmd}" - elif [ "${pyver_maj}" -eq 3 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY3VER}" ]; then + if [ "${pyver_maj}" -eq 3 ] && [ "${pyver_min}" -ge "${REQ_MIN_PY3VER}" ]; then verbose "'${python_cmd}' version: ${pyver}, which matches Python 3 version requirement (>= 3.${REQ_MIN_PY3VER})" PYTHON="${python_cmd}" fi @@ -106,7 +103,7 @@ done if [ -z "${PYTHON}" ]; then echo -n "ERROR: No compatible 'python' command found via \$PATH " >&2 - echo "(EasyBuild requires Python 2.${REQ_MIN_PY2VER}+ or 3.${REQ_MIN_PY3VER}+)" >&2 + echo "(EasyBuild requires Python 3.${REQ_MIN_PY3VER}+)" >&2 exit 1 else verbose "Selected Python command: ${python_cmd} ($(command -v "${python_cmd}"))" diff --git a/requirements.txt b/requirements.txt index 85a9df78e7..699cc90372 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,16 +3,8 @@ keyring keyrings.alt -# GitPython 3.1.15 deprecates Python 3.5 -GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' -GitPython; python_version >= '3.6' or python_version <= '3.0' - -# autopep8 -# stick to older autopep8 with Python 2.7, since autopep8 1.7.0 requires pycodestyle>=2.9.1 (which is Python 3 only) -autopep8<1.7.0; python_version < '3.0' -autopep8; python_version >= '3.0' - -# PyYAML +GitPython +autopep8 PyYAML # optional Python packages for EasyBuild @@ -20,18 +12,11 @@ PyYAML # flake8 is a superset of pycodestyle flake8 -# 2.6.7 uses invalid Python 2 syntax -GC3Pie!=2.6.7; python_version < '3.0' -GC3Pie; python_version >= '3.0' and python_version < '3.11' +GC3Pie; python_version < '3.11' python-graph-dot python-hglib requests archspec -# cryptography 3.4.0 no longer supports Python 2.7 -cryptography==3.3.2; python_version == '2.7' -cryptography; python_version >= '3.5' and python_version < '3.11' - -# rich is only supported for Python 3.6+ -rich; python_version >= '3.6' +rich diff --git a/setup.py b/setup.py index d45eccf949..c7b496867a 100644 --- a/setup.py +++ b/setup.py @@ -110,8 +110,6 @@ def find_rel_test(): "Intended Audience :: System Administrators", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index f8db1927d8..56666f3f50 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -35,7 +35,7 @@ import easybuild.tools.asyncprocess as p from easybuild.tools.asyncprocess import Popen -from easybuild.tools.py2vs3 import subprocess_terminate +from easybuild.tools.run import subprocess_terminate class AsyncProcessTest(EnhancedTestCase): diff --git a/test/framework/config.py b/test/framework/config.py index 4b42672cfc..ce5eb2148e 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -33,6 +33,7 @@ import shutil import sys import tempfile +from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -46,7 +47,6 @@ from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX -from easybuild.tools.py2vs3 import reload class EasyBuildConfigTest(EnhancedTestCase): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 018c3e293a..d92e0e9d3d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -52,7 +52,6 @@ from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild -from easybuild.tools.py2vs3 import string_type class EasyBlockTest(EnhancedTestCase): @@ -431,7 +430,7 @@ def test_make_module_req(self): write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar') write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar') for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): - if isinstance(path, string_type): + if isinstance(path, str): path = (path, ) os.mkdir(os.path.join(eb.installdir, *path)) # this is not a path that should be picked up @@ -1241,7 +1240,7 @@ def test_make_module_step(self): self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (key, vals) in modextrapaths.items(): - if isinstance(vals, string_type): + if isinstance(vals, str): vals = [vals] for val in vals: if get_module_syntax() == 'Tcl': @@ -2463,7 +2462,7 @@ def test_checksum_step(self): # make sure that test easyconfig file indeed doesn't contain any checksums (either top-level or for extensions) self.assertEqual(ec_json['ec']['checksums'], []) for ext in ec_json['ec']['exts_list']: - if isinstance(ext, string_type): + if isinstance(ext, str): continue elif isinstance(ext, tuple): self.assertEqual(ext[2].get('checksums', []), []) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f09a0896a6..7bf72fc57b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -38,7 +38,9 @@ import sys import tempfile import textwrap +from collections import OrderedDict from easybuild.tools import LooseVersion +from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -72,7 +74,6 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata -from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 9a17d8f0a5..638137ec56 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -39,7 +39,6 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file -from easybuild.tools.py2vs3 import string_type TESTDIRBASE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') @@ -198,11 +197,11 @@ def test_easyconfig_constants(self): # make sure both keys and values are of appropriate types for constant_name in constants: - self.assertIsInstance(constant_name, string_type, "Constant name %s is a string" % constant_name) + self.assertIsInstance(constant_name, str, "Constant name %s is a string" % constant_name) val = constants[constant_name] fail_msg = "The constant %s should have an acceptable type, found %s (%s)" % (constant_name, type(val), str(val)) - self.assertIsInstance(val, (string_type, dict, tuple), fail_msg) + self.assertIsInstance(val, (str, dict, tuple), fail_msg) # check a couple of randomly picked constant values self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..5b50296e4c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -40,15 +40,16 @@ import sys import tempfile import time +from io import StringIO from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner +from urllib import request from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import StringIO, std_urllib class FileToolsTest(EnhancedTestCase): @@ -404,7 +405,7 @@ def test_det_file_size(self): # also try with actual HTTP header try: - fh = std_urllib.urlopen(test_url) + fh = request.urlopen(test_url) self.assertEqual(ft.det_file_size(fh.info()), expected_size) fh.close() @@ -416,7 +417,7 @@ def test_det_file_size(self): res.close() except ImportError: pass - except std_urllib.URLError: + except request.URLError: print("Skipping online test for det_file_size (working offline)") def test_download_file(self): @@ -437,8 +438,8 @@ def test_download_file(self): # install broken proxy handler for opening local files # this should make urlopen use this broken proxy for downloading from a file:// URL - proxy_handler = std_urllib.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) - std_urllib.install_opener(std_urllib.build_opener(proxy_handler)) + proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) + request.install_opener(request.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file @@ -448,7 +449,7 @@ def test_download_file(self): ft.write_file(target_location, '') # restore a working file handler, and retest download of local file - std_urllib.install_opener(std_urllib.build_opener(std_urllib.FileHandler())) + request.install_opener(request.build_opener(request.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") @@ -465,10 +466,10 @@ def test_download_file(self): target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') url = 'https://raw.githubusercontent.com/easybuilders/easybuild-framework/master/README.rst' try: - std_urllib.urlopen(url) + request.urlopen(url) res = ft.download_file(fn, url, target_location) self.assertEqual(res, target_location, "download with specified timeout works") - except std_urllib.URLError: + except request.URLError: print("Skipping timeout test in test_download_file (working offline)") # also test behaviour of download_file under --dry-run diff --git a/test/framework/github.py b/test/framework/github.py index 9df53736e4..6efb5f2e2c 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -36,9 +36,11 @@ import sys import textwrap import unittest +from string import ascii_letters from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from time import gmtime from unittest import TextTestRunner +from urllib.request import HTTPError, URLError import easybuild.tools.testing from easybuild.base.rest import RestClient @@ -52,7 +54,6 @@ from easybuild.tools.github import VALID_CLOSE_PR_REASONS from easybuild.tools.github import is_patch_for, pick_default_branch from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state -from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh try: diff --git a/test/framework/license.py b/test/framework/license.py index 80c741908f..8b231a8346 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -33,7 +33,6 @@ from unittest import TextTestRunner from easybuild.framework.easyconfig.licenses import License, LicenseVeryRestrictive, what_licenses -from easybuild.tools.py2vs3 import string_type class LicenseTest(EnhancedTestCase): @@ -62,7 +61,7 @@ def test_licenses(self): """Test format of available licenses.""" lics = what_licenses() for lic in lics: - self.assertIsInstance(lic, string_type) + self.assertIsInstance(lic, str) self.assertTrue(lic.startswith('License')) self.assertTrue(issubclass(lics[lic], License)) diff --git a/test/framework/options.py b/test/framework/options.py index 27928265ec..ae8676332c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -36,9 +36,9 @@ import sys import tempfile import textwrap -import warnings -from easybuild.tools import LooseVersion +from importlib import reload from unittest import TextTestRunner +from urllib.request import URLError import easybuild.main import easybuild.tools.build_log @@ -63,7 +63,6 @@ from easybuild.tools.modules import Lmod from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color -from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import HAVE_ARCHSPEC @@ -6343,50 +6342,6 @@ def test_installdir(self): eb = EasyBlock(EasyConfig(toy_ec)) self.assertTrue(eb.installdir.endswith('/software/Core/toy/0.0')) - def test_sort_looseversions(self): - """Test sort_looseversions function.""" - # Test twice: With the standard distutils LooseVersion (when available) and with our class - # Note that our class directly allows sorting but should also work with sort_loosversions - for use_distutils in (True, False): - if use_distutils: - try: - from distutils.version import LooseVersion as version_class - except ImportError: - continue - else: - version_class = LooseVersion - - with warnings.catch_warnings(): - if use_distutils: - warnings.simplefilter("ignore", category=DeprecationWarning) - ver1 = version_class('1.2.3') - ver2 = version_class('4.5.6') - ver3 = version_class('1.2.3dev') - ver4 = version_class('system') - ver5 = version_class('rc3') - ver6 = version_class('v1802') - - # some versions are included multiple times on purpose, - # to also test comparison between equal LooseVersion instances - input = [ver3, ver5, ver1, ver2, ver4, ver6, ver3, ver4, ver1] - expected = [ver1, ver1, ver3, ver3, ver2, ver5, ver4, ver4, ver6] - self.assertEqual(sort_looseversions(input), expected) - if not use_distutils: - self.assertEqual(sorted(input), expected) - - # also test on list of tuples consisting of a LooseVersion instance + a string - # (as in the list_software_* functions) - suff1 = '' - suff2 = '-foo' - suff3 = '-bar' - input = [(ver3, suff1), (ver5, suff3), (ver1, suff2), (ver2, suff3), (ver4, suff1), - (ver6, suff2), (ver3, suff3), (ver4, suff3), (ver1, suff1)] - expected = [(ver1, suff1), (ver1, suff2), (ver3, suff1), (ver3, suff3), (ver2, suff3), - (ver5, suff3), (ver4, suff1), (ver4, suff3), (ver6, suff2)] - self.assertEqual(sort_looseversions(input), expected) - if not use_distutils: - self.assertEqual(sorted(input), expected) - def test_cuda_compute_capabilities(self): """Test --cuda-compute-capabilities configuration option.""" args = ['--cuda-compute-capabilities=3.5,6.2,7.0', '--show-config'] diff --git a/test/framework/run.py b/test/framework/run.py index 48a0f75fd2..f1b6f1973f 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -49,9 +49,8 @@ from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.filetools import adjust_permissions, read_file, write_file from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process -from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa +from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN -from easybuild.tools.py2vs3 import subprocess_terminate class RunTest(EnhancedTestCase): diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 0841e3b76f..6f851862a8 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -40,7 +40,6 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file -from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN @@ -422,7 +421,7 @@ def test_avail_core_count_darwin(self): def test_cpu_model_native(self): """Test getting CPU model.""" cpu_model = get_cpu_model() - self.assertIsInstance(cpu_model, string_type) + self.assertIsInstance(cpu_model, str) def test_cpu_model_linux(self): """Test getting CPU model (mocked for Linux).""" @@ -497,7 +496,7 @@ def test_cpu_features_native(self): cpu_feat = get_cpu_features() self.assertIsInstance(cpu_feat, list) self.assertTrue(len(cpu_feat) >= 0) - self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat)) + self.assertTrue(all(isinstance(x, str) for x in cpu_feat)) def test_cpu_features_linux(self): """Test getting CPU features (mocked for Linux).""" @@ -581,7 +580,7 @@ def test_cpu_architecture(self): def test_cpu_arch_name_native(self): """Test getting CPU arch name.""" arch_name = get_cpu_arch_name() - self.assertIsInstance(arch_name, string_type) + self.assertIsInstance(arch_name, str) def test_cpu_arch_name(self): """Test getting CPU arch name.""" @@ -715,12 +714,12 @@ def test_shared_lib_ext_darwin(self): def test_platform_name_native(self): """Test getting platform name.""" platform_name_nover = get_platform_name() - self.assertIsInstance(platform_name_nover, string_type) + self.assertIsInstance(platform_name_nover, str) len_nover = len(platform_name_nover.split('-')) self.assertTrue(len_nover >= 3) platform_name_ver = get_platform_name(withversion=True) - self.assertIsInstance(platform_name_ver, string_type) + self.assertIsInstance(platform_name_ver, str) len_ver = len(platform_name_ver.split('-')) self.assertTrue(platform_name_ver.startswith(platform_name_ver)) self.assertTrue(len_ver >= len_nover) @@ -740,12 +739,12 @@ def test_platform_name_darwin(self): def test_os_name(self): """Test getting OS name.""" os_name = get_os_name() - self.assertTrue(isinstance(os_name, string_type) or os_name == UNKNOWN) + self.assertTrue(isinstance(os_name, str) or os_name == UNKNOWN) def test_os_version(self): """Test getting OS version.""" os_version = get_os_version() - self.assertTrue(isinstance(os_version, string_type) or os_version == UNKNOWN) + self.assertTrue(isinstance(os_version, str) or os_version == UNKNOWN) # make sure that bug fixed in https://github.com/easybuilders/easybuild-framework/issues/3952 # does not surface again, by mocking what's needed to make get_os_version fall into SLES-specific path @@ -766,7 +765,7 @@ def test_gcc_version_native(self): """Test getting gcc version.""" gcc_version = get_gcc_version() if gcc_version is not None: - self.assertIsInstance(gcc_version, string_type) + self.assertIsInstance(gcc_version, str) def test_gcc_version_linux(self): """Test getting gcc version (mocked for Linux).""" @@ -783,7 +782,7 @@ def test_gcc_version_darwin(self): def test_glibc_version_native(self): """Test getting glibc version.""" glibc_version = get_glibc_version() - self.assertTrue(isinstance(glibc_version, string_type) or glibc_version == UNKNOWN) + self.assertTrue(isinstance(glibc_version, str) or glibc_version == UNKNOWN) def test_glibc_version_linux(self): """Test getting glibc version (mocked for Linux).""" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c36431c45..e39bcd3016 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -50,7 +50,6 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir from easybuild.tools.filetools import read_file, symlink, write_file, which -from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template @@ -521,7 +520,7 @@ def test_get_variable_libs_list(self): ldflags = tc.get_variable('LDFLAGS', typ=list) self.assertIsInstance(ldflags, list) if len(ldflags) > 0: - self.assertIsInstance(ldflags[0], string_type) + self.assertIsInstance(ldflags[0], str) def test_validate_pass_by_value(self): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index be1421ec85..bdf244e22c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -41,6 +41,7 @@ import tempfile import textwrap from easybuild.tools import LooseVersion +from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -56,7 +57,6 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -734,7 +734,7 @@ def test_toy_group_check(self): for group in [group_name, (group_name, "Hey, you're not in the '%s' group!" % group_name)]: - if isinstance(group, string_type): + if isinstance(group, str): write_file(test_ec, read_file(toy_ec) + "\ngroup = '%s'\n" % group) else: write_file(test_ec, read_file(toy_ec) + "\ngroup = %s\n" % str(group)) diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 83f1ef41ac..1eb2f19fd3 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -41,7 +41,6 @@ from easybuild.framework.easyconfig.types import to_list_of_strings_and_tuples_and_dicts from easybuild.framework.easyconfig.types import to_sanity_check_paths_dict, to_toolchain_dict from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.py2vs3 import string_type class TypeCheckingTest(EnhancedTestCase): @@ -248,9 +247,9 @@ def test_check_type_of_param_value_patches(self): def test_convert_value_type(self): """Test convert_value_type function.""" # to string - self.assertEqual(convert_value_type(100, string_type), '100') + self.assertEqual(convert_value_type(100, str), '100') self.assertEqual(convert_value_type((100,), str), '(100,)') - self.assertEqual(convert_value_type([100], string_type), '[100]') + self.assertEqual(convert_value_type([100], str), '[100]') self.assertEqual(convert_value_type(None, str), 'None') # to int/float @@ -269,7 +268,7 @@ def test_convert_value_type(self): self.assertEqual(convert_value_type((), LIST_OF_STRINGS), []) # idempotency - self.assertEqual(convert_value_type('foo', string_type), 'foo') + self.assertEqual(convert_value_type('foo', str), 'foo') self.assertEqual(convert_value_type('foo', str), 'foo') self.assertEqual(convert_value_type(100, int), 100) self.assertEqual(convert_value_type(1.6, float), 1.6) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 37a9b29318..b32a5fd06a 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -37,6 +37,7 @@ import tempfile import unittest from contextlib import contextmanager +from importlib import reload from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -54,7 +55,6 @@ from easybuild.tools.filetools import copy_dir, mkdir, read_file, which from easybuild.tools.modules import curr_module_paths, modules_tool, reset_module_caches from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir -from easybuild.tools.py2vs3 import reload # make sure tests are robust against any non-default configuration settings;