diff --git a/.travis.yml b/.travis.yml index 7f3b0a0e..130ff720 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python python: - "2.6" - "2.7" + - "3.5" + - "3.6" + script: # 'hpcugent' is a mandatory remote for setup.py to work (because of get_name_url) - git remote add hpcugent https://github.com/hpcugent/vsc-install.git diff --git a/README.md b/README.md index c2ebf3da..6266ca4d 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,6 @@ except (ExceptionOne, ExceptionTwo) ... turning off these errors ------------------------- - If in any of these cases you think: yes, I really needed to do this, I'm monkeypatching things, I'm adding extra functionality that does indeed have an extra(default) paramenter, etc, etc you can let pylint know to ignore this error in this one specific block of code diff --git a/lib/vsc/install/commontest.py b/lib/vsc/install/commontest.py index b2533b75..d3309c61 100644 --- a/lib/vsc/install/commontest.py +++ b/lib/vsc/install/commontest.py @@ -33,6 +33,7 @@ @author: Stijn De Weirdt (Ghent University) """ + import logging import optparse import os @@ -47,7 +48,7 @@ # No prospector in py26 or earlier # Also not enforced on installation -HAS_PROTECTOR = False +HAS_PROSPECTOR = False Prospector = None ProspectorConfig = None @@ -57,12 +58,17 @@ _old_basicconfig = logging.basicConfig from prospector.run import Prospector from prospector.config import ProspectorConfig - HAS_PROTECTOR = True + HAS_PROSPECTOR = True # restore in case pyroma is missing (see https://github.com/landscapeio/prospector/pull/156) logging.basicConfig = _old_basicconfig except ImportError: pass +# Prospector doesn't have support for 3.5 / 3.6 +# https://github.com/PyCQA/prospector/issues/233 +if sys.version_info >= (3, 5): + HAS_PROSPECTOR = False + class CommonTest(TestCase): """ @@ -95,7 +101,6 @@ class CommonTest(TestCase): 'undefined', 'no-value-for-parameter', 'dangerous-default-value', - 'redefined-builtin', 'bare-except', 'E713', # not 'c' in d: -> 'c' not in d: 'arguments-differ', @@ -124,9 +129,10 @@ class CommonTest(TestCase): 'old-ne-operator', # don't use <> as not equal operator, use != 'backtick', # don't use `variable` to turn a variable in a string, use the str() function 'old-raise-syntax', # sed when the alternate raise syntax raise foo, bar is used instead of raise foo(bar) . + 'redefined-builtin', # once we get ready to really move to python3 - # 'print-statement', # use print() and from future import __print__ instead of print - # 'metaclass-assignment', # __metaclass__ doesn't exist anymore in python3 + 'print-statement', # use print() and from future import __print__ instead of print + 'metaclass-assignment', # __metaclass__ doesn't exist anymore in python3 ] # Prospector commandline options (positional path is added automatically) @@ -189,7 +195,7 @@ def test_prospector(self): """Run prospector.run.main, but apply white/blacklists to the results""" orig_expand_default = optparse.HelpFormatter.expand_default - if not HAS_PROTECTOR: + if not HAS_PROSPECTOR: if sys.version_info < (2, 7): log.info('No protector tests are ran on py26 or older.') else: diff --git a/lib/vsc/install/headers.py b/lib/vsc/install/headers.py index 1d7efe36..3e0e3e10 100644 --- a/lib/vsc/install/headers.py +++ b/lib/vsc/install/headers.py @@ -93,7 +93,8 @@ def get_header(filename, script=False): if not os.path.isfile(filename): raise Exception('get_header filename %s not found' % filename) - txt = open(filename).read() + with open(filename) as fh: + txt = fh.read() blocks = HEADER_REGEXP.split(txt) if len(blocks) == 1: @@ -162,10 +163,9 @@ def begin_end_from_header(header): def _write(filename, content): """Simple wrapper around open().write for unittesting""" - fh = open(filename, 'w') - fh.write(content) - fh.close() - + with open(filename, 'w') as fh: + fh.write(content) + def check_header(filename, script=False, write=False): """ @@ -226,7 +226,8 @@ def check_header(filename, script=False, write=False): if write and changed: log.info('write enabled and different header. Going to modify file %s' % filename) - wholetext = open(filename).read() + with open(filename) as fh: + wholetext = fh.read() newtext = '' if shebang is not None: newtext += shebang + "\n" diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index 7700a518..69f6afbe 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -30,14 +30,21 @@ @author: Stijn De Weirdt (Ghent University) @author: Andy Georges (Ghent University) """ -import __builtin__ + +from __future__ import print_function +import sys + +if sys.version_info < (3, 0): + import __builtin__ +else: + import builtins as __builtin__ # make builtins accessible via same way as in Python 3 + import glob import hashlib import inspect import json import os import shutil -import sys import re import setuptools.command.test @@ -109,7 +116,7 @@ def _log(self, level, msg, args): try: return self._orig_log(self, level, newmsg, args) except Exception: - print newmsg % args + print(newmsg % args) log.Log = NewLog log._global_log = NewLog() @@ -148,7 +155,7 @@ def _log(self, level, msg, args): RELOAD_VSC_MODS = False -VERSION = '0.10.32' +VERSION = '0.11.00' log.info('This is (based on) vsc.install.shared_setup %s' % VERSION) @@ -361,13 +368,15 @@ def get_name_url(self, filename=None, version=None, license_name=None): if len(res) != 3: raise Exception("Cannot determine name, url and download url from filename %s: got %s" % (filename, res)) else: + keepers = {} for name, value in res.items(): if value is None: log.info('Removing None %s' % name) - res.pop(name) + else: + keepers[name] = value - log.info('get_name_url returns %s' % res) - return res + log.info('get_name_url returns %s' % keepers) + return keepers def rel_gitignore(self, paths, base_dir=None): """ @@ -1047,7 +1056,7 @@ def finalize_options(self): def _print(self, cmd): """Print is evil, cmd is list""" - print ' '.join(cmd) + print(' '.join(cmd)) def git_tag(self): """Tag the version in git""" @@ -1182,8 +1191,12 @@ def sanitize(name): name starts with 'vsc' and name does not start with python- """ - if isinstance(name, basestring): + if isinstance(name, (list, tuple)): + klass = _fvs('sanitize') + return ",".join([klass.sanitize(r) for r in name]) + + else: if os.environ.get('VSC_RPM_PYTHON', 'NOT_ONE') == '1': # hardcoded prefix map for pydep, rpmname in PYTHON_BDIST_RPM_PREFIX_MAP.items(): @@ -1197,14 +1210,19 @@ def sanitize(name): if p_p: name = 'python-%s' % name return name - else: - klass = _fvs('sanitize') - return ",".join([klass.sanitize(r) for r in name]) + + + @staticmethod def get_md5sum(filename): """Use this function to compute the md5sum in the KNOWN_LICENSES hash""" - return hashlib.md5(open(filename).read()).hexdigest() + hasher = hashlib.md5() + with open(filename, "rb") as fh: + for chunk in iter(lambda: fh.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + def get_license(self, license_name=None): """ @@ -1254,11 +1272,15 @@ def parse_target(self, target, urltemplate=None): # update the cmdclass with ones from vsc_setup_klass # cannot do this in one go, when SHARED_TARGET is defined, vsc_setup doesn't exist yet - for name, klass in new_target['cmdclass'].items(): + keepers = new_target['cmdclass'].copy() + for name in new_target['cmdclass']: + klass = new_target['cmdclass'][name] try: - new_target['cmdclass'][name] = getattr(vsc_setup_klass, klass.__name__) + keepers[name] = getattr(vsc_setup_klass, klass.__name__) except AttributeError: - del new_target['cmdclass'][name] + del keepers[name] + log.info("Not including new_target['cmdclass']['%s']" % name) + new_target['cmdclass'] = keepers # prepare classifiers classifiers = new_target.setdefault('classifiers', []) @@ -1385,7 +1407,7 @@ def parse_target(self, target, urltemplate=None): dep, depversion])] log.debug("New target = %s" % (new_target)) - print new_target + print(new_target) return new_target @staticmethod @@ -1410,8 +1432,8 @@ def build_setup_cfg_for_bdist_rpm(target): try: setup_cfg = open('setup.cfg', 'w') # and truncate - except (IOError, OSError), err: - print "Cannot create setup.cfg for target %s: %s" % (target['name'], err) + except (IOError, OSError) as err: + print("Cannot create setup.cfg for target %s: %s" % (target['name'], err)) sys.exit(1) klass = _fvs('build_setup_cfg_for_bdist_rpm') @@ -1481,7 +1503,6 @@ def action_target(self, target, setupfn=None, extra_sdist=None, urltemplate=None self.prepare_rpm(target) x = self.parse_target(target, urltemplate) - setupfn(**x) diff --git a/lib/vsc/install/testing.py b/lib/vsc/install/testing.py index 207e3e0e..6fca3402 100644 --- a/lib/vsc/install/testing.py +++ b/lib/vsc/install/testing.py @@ -36,7 +36,11 @@ import re import sys -from cStringIO import StringIO +try: + from cStringIO import StringIO # Python 2 +except ImportError: + from io import StringIO # Python 3 + from unittest import TestCase as OrigTestCase from vsc.install.headers import nicediff @@ -51,9 +55,18 @@ class TestCase(OrigTestCase): ASSERT_MAX_DIFF = 100 DIFF_OFFSET = 5 # lines of text around changes + def is_string(self, x): + """test if the variable x is a string)""" + try: + return isinstance(x, basestring) + except NameError: + return isinstance(x, str) + + # pylint: disable=arguments-differ def assertEqual(self, a, b, msg=None): """Make assertEqual always print useful messages""" + try: super(TestCase, self).assertEqual(a, b) except AssertionError as e: @@ -62,11 +75,11 @@ def assertEqual(self, a, b, msg=None): else: msg = "%s: %s" % (msg, e) - if isinstance(a, basestring): + if self.is_string(a): txta = a else: txta = pprint.pformat(a) - if isinstance(b, basestring): + if self.is_string(b): txtb = b else: txtb = pprint.pformat(b) @@ -120,9 +133,9 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs): str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()] str_args = ', '.join(map(str, args) + str_kwargs) self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args)) - except error, err: + except error as err: msg = self.convert_exception_to_str(err) - if isinstance(regex, basestring): + if self.is_string(regex): regex = re.compile(regex) self.assertTrue(regex.search(msg), "Pattern '%s' is found in '%s'" % (regex.pattern, msg)) diff --git a/test/headers.py b/test/headers.py index e22ad5d2..648a3497 100644 --- a/test/headers.py +++ b/test/headers.py @@ -111,8 +111,8 @@ def test_gen_license_header(self): } for license in KNOWN_LICENSES.keys(): res_fn = os.path.join(self.setup.REPO_TEST_DIR, 'headers', license) - result = open(res_fn).read() - + with open(res_fn) as fh: + result = fh.read() gen_txt = gen_license_header(license, **data) self.assertEqual(gen_txt, result, msg='generated header for license %s as expected' % license) log.info('generated license header %s' % license) diff --git a/test/shared_setup.py b/test/shared_setup.py index 3ea88aaa..dc3e70e1 100644 --- a/test/shared_setup.py +++ b/test/shared_setup.py @@ -24,6 +24,9 @@ # along with vsc-install. If not, see . # """Test shared_setup""" + +from __future__ import print_function + import os import re @@ -110,8 +113,8 @@ def test_rel_gitignore(self): base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './testdata') try: self.setup.rel_gitignore(['testdata'], base_dir=base_dir) - except Exception as e: - self.assertTrue('.pyc' in e.message, msg=e) + except Exception as err: + self.assertTrue('.pyc' in str(err)) else: self.assertTrue(False, 'rel_gitignore should have raised an exception, but did not!') # it should not fail if base_dir does not contain a .git folder @@ -127,14 +130,13 @@ def test_action_target(self): """Test action_target function, mostly w.r.t. backward compatibility.""" def fake_setup(*args, **kwargs): """Fake setup function to test action_target with.""" - print 'args: ', args - print 'kwargs:', kwargs + print('args: %s' % str(args)) + print('kwargs: %s' % kwargs) self.mock_stdout(True) action_target({'name': 'vsc-test', 'version': '1.0.0'}, setupfn=fake_setup) txt = self.get_stdout() self.mock_stdout(False) - self.assertTrue(re.search(r"^args:\s*\(\)", txt, re.M)) self.assertTrue(re.search(r"^kwargs:\s*\{.*'name':\s*'vsc-test'", txt, re.M)) @@ -150,6 +152,7 @@ def test_prepare_rpm(self): """ Test the prepare rpm function especially in effect to generating correct package list wrt excluded_pkgs_rpm + we assume the order of the lists doesn't matter (and sort to compare) """ package = { 'name': 'vsc-test', @@ -161,7 +164,8 @@ def test_prepare_rpm(self): libdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './testdata') setup.REPO_LIB_DIR = libdir setup.prepare_rpm(package) - self.assertEqual(setup.SHARED_TARGET['packages'], ['vsc', 'vsc.test']) + + self.assertEqual(sorted(setup.SHARED_TARGET['packages']), ['vsc', 'vsc.test']) package = { 'name': 'vsc-test', 'excluded_pkgs_rpm': ['vsc', 'vsc.test'], @@ -170,7 +174,8 @@ def test_prepare_rpm(self): setup = vsc_setup() setup.REPO_LIB_DIR = libdir setup.prepare_rpm(package) - self.assertEqual(setup.SHARED_TARGET['packages'], ['vsc', 'vsc.test']) + + self.assertEqual(sorted(setup.SHARED_TARGET['packages']), ['vsc', 'vsc.test']) def test_parse_target(self): """Test for parse target"""