minimal tweaks to update for python3

.travis.yml

@@ -2,6 +2,9 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.5"
+  - "3.6"
+
 script:
   # 'hpcugent' is a mandatory remote for setup.py to work (because of get_name_url)
   - git remote add hpcugent https://github.com/hpcugent/vsc-install.git

README.md

@@ -219,7 +219,6 @@ except (ExceptionOne, ExceptionTwo) ...
 turning off these errors
 -------------------------
 
-
 If in any of these cases you think: yes, I really needed to do this,
 I'm monkeypatching things, I'm adding extra functionality that does indeed have an extra(default) paramenter, etc, etc
 you can let pylint know to ignore this error in this one specific block of code

lib/vsc/install/commontest.py

@@ -33,6 +33,7 @@
 
 @author: Stijn De Weirdt (Ghent University)
 """
+
 import logging
 import optparse
 import os
@@ -47,7 +48,7 @@
 
 # No prospector in py26 or earlier
 # Also not enforced on installation
-HAS_PROTECTOR = False
+HAS_PROSPECTOR = False
 Prospector = None
 ProspectorConfig = None
 
@@ -57,12 +58,17 @@
         _old_basicconfig = logging.basicConfig
         from prospector.run import Prospector
         from prospector.config import ProspectorConfig
-        HAS_PROTECTOR = True
+        HAS_PROSPECTOR = True
         # restore in case pyroma is missing (see https://github.com/landscapeio/prospector/pull/156)
         logging.basicConfig = _old_basicconfig
     except ImportError:
         pass
 
+# Prospector doesn't have support for 3.5 / 3.6
+# https://github.com/PyCQA/prospector/issues/233
+if sys.version_info >= (3, 5):
+    HAS_PROSPECTOR = False
+
 
 class CommonTest(TestCase):
     """
@@ -95,7 +101,6 @@ class CommonTest(TestCase):
         'undefined',
         'no-value-for-parameter',
         'dangerous-default-value',
-        'redefined-builtin',
         'bare-except',
         'E713',  # not 'c' in d: -> 'c' not in d:
         'arguments-differ',
@@ -124,9 +129,10 @@ class CommonTest(TestCase):
         'old-ne-operator',  # don't use <> as not equal operator, use !=
         'backtick',  # don't use `variable` to turn a variable in a string, use the str() function
         'old-raise-syntax',  # sed when the alternate raise syntax raise foo, bar is used instead of raise foo(bar) .
+        'redefined-builtin',
         # once we get ready to really move to python3
-        # 'print-statement',  # use print() and from future import __print__ instead of print
-        # 'metaclass-assignment',  # __metaclass__ doesn't exist anymore in python3
+        'print-statement',  # use print() and from future import __print__ instead of print
+        'metaclass-assignment',  # __metaclass__ doesn't exist anymore in python3
     ]
 
     # Prospector commandline options (positional path is added automatically)
@@ -189,7 +195,7 @@ def test_prospector(self):
         """Run prospector.run.main, but apply white/blacklists to the results"""
         orig_expand_default = optparse.HelpFormatter.expand_default
 
-        if not HAS_PROTECTOR:
+        if not HAS_PROSPECTOR:
             if sys.version_info < (2, 7):
                 log.info('No protector tests are ran on py26 or older.')
             else:

lib/vsc/install/headers.py

@@ -93,7 +93,8 @@ def get_header(filename, script=False):
     if not os.path.isfile(filename):
         raise Exception('get_header filename %s not found' % filename)
 
-    txt = open(filename).read()
+    with open(filename) as fh:
+        txt = fh.read()
 
     blocks = HEADER_REGEXP.split(txt)
     if len(blocks) == 1:
@@ -162,10 +163,9 @@ def begin_end_from_header(header):
 
 def _write(filename, content):
     """Simple wrapper around open().write for unittesting"""
-    fh = open(filename, 'w')
-    fh.write(content)
-    fh.close()
-
+    with open(filename, 'w') as fh:
+        fh.write(content)
+
 
 def check_header(filename, script=False, write=False):
     """
@@ -226,7 +226,8 @@ def check_header(filename, script=False, write=False):
 
     if write and changed:
         log.info('write enabled and different header. Going to modify file %s' % filename)
-        wholetext = open(filename).read()
+        with open(filename) as fh:
+            wholetext = fh.read()
         newtext = ''
         if shebang is not None:
             newtext += shebang + "\n"

lib/vsc/install/shared_setup.py

@@ -30,14 +30,21 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Andy Georges (Ghent University)
 """
-import __builtin__
+
+from __future__ import print_function
+import sys
+
+if sys.version_info < (3, 0):
+    import __builtin__
+else:
+    import builtins as __builtin__  # make builtins accessible via same way as in Python 3
+
 import glob
 import hashlib
 import inspect
 import json
 import os
 import shutil
-import sys
 import re
 
 import setuptools.command.test
@@ -109,7 +116,7 @@ def _log(self, level, msg, args):
             try:
                 return self._orig_log(self, level, newmsg, args)
             except Exception:
-                print newmsg % args
+                print(newmsg % args)
 
     log.Log = NewLog
     log._global_log = NewLog()
@@ -148,7 +155,7 @@ def _log(self, level, msg, args):
 
 RELOAD_VSC_MODS = False
 
-VERSION = '0.10.32'
+VERSION = '0.11.00'
 
 log.info('This is (based on) vsc.install.shared_setup %s' % VERSION)
 
@@ -361,13 +368,15 @@ def get_name_url(self, filename=None, version=None, license_name=None):
         if len(res) != 3:
             raise Exception("Cannot determine name, url and download url from filename %s: got %s" % (filename, res))
         else:
+            keepers = {}
             for name, value in res.items():
                 if value is None:
                     log.info('Removing None %s' % name)
-                    res.pop(name)
+                else:
+                    keepers[name] = value
 
-            log.info('get_name_url returns %s' % res)
-            return res
+            log.info('get_name_url returns %s' % keepers)
+            return keepers
 
     def rel_gitignore(self, paths, base_dir=None):
         """
@@ -1047,7 +1056,7 @@ def finalize_options(self):
 
         def _print(self, cmd):
             """Print is evil, cmd is list"""
-            print ' '.join(cmd)
+            print(' '.join(cmd))
 
         def git_tag(self):
             """Tag the version in git"""
@@ -1182,8 +1191,12 @@ def sanitize(name):
                 name starts with 'vsc'
                 and name does not start with python-
         """
-        if isinstance(name, basestring):
 
+        if isinstance(name, (list, tuple)):
+            klass = _fvs('sanitize')
+            return ",".join([klass.sanitize(r) for r in name])
+
+        else:
             if os.environ.get('VSC_RPM_PYTHON', 'NOT_ONE') == '1':
                 # hardcoded prefix map
                 for pydep, rpmname in PYTHON_BDIST_RPM_PREFIX_MAP.items():
@@ -1197,14 +1210,19 @@ def sanitize(name):
                 if p_p:
                     name = 'python-%s' % name
             return name
-        else:
-            klass = _fvs('sanitize')
-            return ",".join([klass.sanitize(r) for r in name])
+
+
+
 
     @staticmethod
     def get_md5sum(filename):
         """Use this function to compute the md5sum in the KNOWN_LICENSES hash"""
-        return hashlib.md5(open(filename).read()).hexdigest()
+        hasher = hashlib.md5()
+        with open(filename, "rb") as fh:
+            for chunk in iter(lambda: fh.read(4096), b""):
+                hasher.update(chunk)
+        return hasher.hexdigest()
+
 
     def get_license(self, license_name=None):
         """
@@ -1254,11 +1272,15 @@ def parse_target(self, target, urltemplate=None):
 
         # update the cmdclass with ones from vsc_setup_klass
         # cannot do this in one go, when SHARED_TARGET is defined, vsc_setup doesn't exist yet
-        for name, klass in new_target['cmdclass'].items():
+        keepers = new_target['cmdclass'].copy()
+        for name in new_target['cmdclass']:
+            klass = new_target['cmdclass'][name]
             try:
-                new_target['cmdclass'][name] = getattr(vsc_setup_klass, klass.__name__)
+                keepers[name] = getattr(vsc_setup_klass, klass.__name__)
             except AttributeError:
-                del new_target['cmdclass'][name]
+                del keepers[name]
+                log.info("Not including new_target['cmdclass']['%s']" % name)
+        new_target['cmdclass'] = keepers
 
         # prepare classifiers
         classifiers = new_target.setdefault('classifiers', [])
@@ -1385,7 +1407,7 @@ def parse_target(self, target, urltemplate=None):
                                                            dep, depversion])]
 
         log.debug("New target = %s" % (new_target))
-        print new_target
+        print(new_target)
         return new_target
 
     @staticmethod
@@ -1410,8 +1432,8 @@ def build_setup_cfg_for_bdist_rpm(target):
 
         try:
             setup_cfg = open('setup.cfg', 'w')  # and truncate
-        except (IOError, OSError), err:
-            print "Cannot create setup.cfg for target %s: %s" % (target['name'], err)
+        except (IOError, OSError) as err:
+            print("Cannot create setup.cfg for target %s: %s" % (target['name'], err))
             sys.exit(1)
 
         klass = _fvs('build_setup_cfg_for_bdist_rpm')
@@ -1481,7 +1503,6 @@ def action_target(self, target, setupfn=None, extra_sdist=None, urltemplate=None
 
         self.prepare_rpm(target)
         x = self.parse_target(target, urltemplate)
-
         setupfn(**x)
 
 

lib/vsc/install/testing.py

@@ -36,7 +36,11 @@
 import re
 import sys
 
-from cStringIO import StringIO
+try:
+    from cStringIO import StringIO  # Python 2
+except ImportError:
+    from io import StringIO  # Python 3
+
 from unittest import TestCase as OrigTestCase
 from vsc.install.headers import nicediff
 
@@ -51,9 +55,18 @@ class TestCase(OrigTestCase):
     ASSERT_MAX_DIFF = 100
     DIFF_OFFSET = 5 # lines of text around changes
 
+    def is_string(self, x):
+        """test if the variable x is a string)"""
+        try:
+            return isinstance(x, basestring)
+        except NameError:
+            return isinstance(x, str)
+
+
     # pylint: disable=arguments-differ
     def assertEqual(self, a, b, msg=None):
         """Make assertEqual always print useful messages"""
+
         try:
             super(TestCase, self).assertEqual(a, b)
         except AssertionError as e:
@@ -62,11 +75,11 @@ def assertEqual(self, a, b, msg=None):
             else:
                 msg = "%s: %s" % (msg, e)
 
-            if isinstance(a, basestring):
+            if self.is_string(a):
                 txta = a
             else:
                 txta = pprint.pformat(a)
-            if isinstance(b, basestring):
+            if self.is_string(b):
                 txtb = b
             else:
                 txtb = pprint.pformat(b)
@@ -120,9 +133,9 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs):
             str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()]
             str_args = ', '.join(map(str, args) + str_kwargs)
             self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args))
-        except error, err:
+        except error as err:
             msg = self.convert_exception_to_str(err)
-            if isinstance(regex, basestring):
+            if self.is_string(regex):
                 regex = re.compile(regex)
             self.assertTrue(regex.search(msg), "Pattern '%s' is found in '%s'" % (regex.pattern, msg))
 

test/headers.py

@@ -111,8 +111,8 @@ def test_gen_license_header(self):
         }
         for license in KNOWN_LICENSES.keys():
             res_fn = os.path.join(self.setup.REPO_TEST_DIR, 'headers', license)
-            result = open(res_fn).read()
-
+            with open(res_fn) as fh:
+                result = fh.read()
             gen_txt = gen_license_header(license, **data)
             self.assertEqual(gen_txt, result, msg='generated header for license %s as expected' % license)
             log.info('generated license header %s' % license)