minimal tweaks to update for python3

.travis.yml

@@ -2,6 +2,9 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.5"
+  - "3.6"
+
 script:
   # 'hpcugent' is a mandatory remote for setup.py to work (because of get_name_url)
   - git remote add hpcugent https://github.com/hpcugent/vsc-install.git

README.md

@@ -219,7 +219,6 @@ except (ExceptionOne, ExceptionTwo) ...
 turning off these errors
 -------------------------
 
-
 If in any of these cases you think: yes, I really needed to do this,
 I'm monkeypatching things, I'm adding extra functionality that does indeed have an extra(default) paramenter, etc, etc
 you can let pylint know to ignore this error in this one specific block of code

lib/vsc/install/commontest.py

@@ -33,6 +33,7 @@
 
 @author: Stijn De Weirdt (Ghent University)
 """
+
 import logging
 import optparse
 import os
@@ -63,6 +64,11 @@
     except ImportError:
         pass
 
+# Prospector doesn't have support for 3.5 / 3.6
+# https://github.com/PyCQA/prospector/issues/233
+if sys.version_info >= (3, 5):
+    HAS_PROTECTOR = False
+
 
 class CommonTest(TestCase):
     """
@@ -87,15 +93,16 @@ class CommonTest(TestCase):
     #   Whitelist: if match, fail test
     PROSPECTOR_BLACKLIST = [
         # 'wrong-import-position',  # not sure about this, these usually have a good reason
+        "Redefining built-in 'basestring'",  # basestring is defined when running on top of Python 3.x
         'Locally disabling',  # shows up when you locally disable a warning, this is the point
         'Useless suppression',  # shows up when you locally disable/suppress a warning, this is the point
+        'redefined-builtin',
     ]
     # to dissable any of these warnings in a block, you can do things like add a comment # pylint: disable=C0321
     PROSPECTOR_WHITELIST = [
         'undefined',
         'no-value-for-parameter',
         'dangerous-default-value',
-        'redefined-builtin',
         'bare-except',
         'E713',  # not 'c' in d: -> 'c' not in d:
         'arguments-differ',

lib/vsc/install/headers.py

@@ -93,7 +93,8 @@ def get_header(filename, script=False):
     if not os.path.isfile(filename):
         raise Exception('get_header filename %s not found' % filename)
 
-    txt = open(filename).read()
+    with open(filename) as fh:
+        txt = fh.read()
 
     blocks = HEADER_REGEXP.split(txt)
     if len(blocks) == 1:
@@ -162,10 +163,9 @@ def begin_end_from_header(header):
 
 def _write(filename, content):
     """Simple wrapper around open().write for unittesting"""
-    fh = open(filename, 'w')
-    fh.write(content)
-    fh.close()
-
+    with open(filename, 'w') as fh:
+        fh.write(content)
+
 
 def check_header(filename, script=False, write=False):
     """
@@ -226,7 +226,8 @@ def check_header(filename, script=False, write=False):
 
     if write and changed:
         log.info('write enabled and different header. Going to modify file %s' % filename)
-        wholetext = open(filename).read()
+        with open(filename) as fh:
+            wholetext = fh.read()
         newtext = ''
         if shebang is not None:
             newtext += shebang + "\n"

lib/vsc/install/shared_setup.py

@@ -30,14 +30,21 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Andy Georges (Ghent University)
 """
-import __builtin__
+
+import sys
+
+if sys.version_info < (3, 0):
+    import __builtin__
+else:
+    basestring = (bytes, str)    # make sure 'basestring' (a builtin in Python 2.x) is defined when using Python 3
+    import builtins as __builtin__  # make builtins accessible via same way as in Python 3
+
 import glob
 import hashlib
 import inspect
 import json
 import os
 import shutil
-import sys
 import re
 
 import setuptools.command.test
@@ -109,7 +116,7 @@ def _log(self, level, msg, args):
             try:
                 return self._orig_log(self, level, newmsg, args)
             except Exception:
-                print newmsg % args
+                print(newmsg % args)
 
     log.Log = NewLog
     log._global_log = NewLog()
@@ -148,7 +155,7 @@ def _log(self, level, msg, args):
 
 RELOAD_VSC_MODS = False
 
-VERSION = '0.10.32'
+VERSION = '0.11.00'
 
 log.info('This is (based on) vsc.install.shared_setup %s' % VERSION)
 
@@ -361,13 +368,15 @@ def get_name_url(self, filename=None, version=None, license_name=None):
         if len(res) != 3:
             raise Exception("Cannot determine name, url and download url from filename %s: got %s" % (filename, res))
         else:
+            keepers = {}
             for name, value in res.items():
                 if value is None:
                     log.info('Removing None %s' % name)
-                    res.pop(name)
+                else:
+                    keepers[name] = value
 
-            log.info('get_name_url returns %s' % res)
-            return res
+            log.info('get_name_url returns %s' % keepers)
+            return keepers
 
     def rel_gitignore(self, paths, base_dir=None):
         """
@@ -1047,7 +1056,7 @@ def finalize_options(self):
 
         def _print(self, cmd):
             """Print is evil, cmd is list"""
-            print ' '.join(cmd)
+            print(' '.join(cmd))
 
         def git_tag(self):
             """Tag the version in git"""
@@ -1201,10 +1210,17 @@ def sanitize(name):
             klass = _fvs('sanitize')
             return ",".join([klass.sanitize(r) for r in name])
 
+
+
     @staticmethod
     def get_md5sum(filename):
         """Use this function to compute the md5sum in the KNOWN_LICENSES hash"""
-        return hashlib.md5(open(filename).read()).hexdigest()
+        hasher = hashlib.md5()
+        with open(filename, "rb") as fh:
+            for chunk in iter(lambda: fh.read(4096), b""):
+                hasher.update(chunk)
+        return hasher.hexdigest()
+
 
     def get_license(self, license_name=None):
         """
@@ -1254,11 +1270,15 @@ def parse_target(self, target, urltemplate=None):
 
         # update the cmdclass with ones from vsc_setup_klass
         # cannot do this in one go, when SHARED_TARGET is defined, vsc_setup doesn't exist yet
-        for name, klass in new_target['cmdclass'].items():
+        keepers = new_target['cmdclass'].copy()
+        for name in new_target['cmdclass']:
+            klass = new_target['cmdclass'][name]
             try:
-                new_target['cmdclass'][name] = getattr(vsc_setup_klass, klass.__name__)
+                keepers[name] = getattr(vsc_setup_klass, klass.__name__)
             except AttributeError:
-                del new_target['cmdclass'][name]
+                del keepers[name]
+                log.info("Not including new_target['cmdclass']['%s']" % name)
+        new_target['cmdclass'] = keepers
 
         # prepare classifiers
         classifiers = new_target.setdefault('classifiers', [])
@@ -1385,7 +1405,7 @@ def parse_target(self, target, urltemplate=None):
                                                            dep, depversion])]
 
         log.debug("New target = %s" % (new_target))
-        print new_target
+        print(new_target)
         return new_target
 
     @staticmethod
@@ -1410,8 +1430,8 @@ def build_setup_cfg_for_bdist_rpm(target):
 
         try:
             setup_cfg = open('setup.cfg', 'w')  # and truncate
-        except (IOError, OSError), err:
-            print "Cannot create setup.cfg for target %s: %s" % (target['name'], err)
+        except (IOError, OSError) as err:
+            print("Cannot create setup.cfg for target %s: %s" % (target['name'], err))
             sys.exit(1)
 
         klass = _fvs('build_setup_cfg_for_bdist_rpm')
@@ -1481,7 +1501,6 @@ def action_target(self, target, setupfn=None, extra_sdist=None, urltemplate=None
 
         self.prepare_rpm(target)
         x = self.parse_target(target, urltemplate)
-
         setupfn(**x)
 
 

lib/vsc/install/testing.py

@@ -36,7 +36,16 @@
 import re
 import sys
 
-from cStringIO import StringIO
+try:
+    basestring  # Python 2
+except NameError:
+    basestring = (bytes, str)  # Python 3
+
+try:
+    from cStringIO import StringIO  # Python 2
+except ImportError:
+    from io import StringIO  # Python 3
+
 from unittest import TestCase as OrigTestCase
 from vsc.install.headers import nicediff
 
@@ -54,6 +63,7 @@ class TestCase(OrigTestCase):
     # pylint: disable=arguments-differ
     def assertEqual(self, a, b, msg=None):
         """Make assertEqual always print useful messages"""
+
         try:
             super(TestCase, self).assertEqual(a, b)
         except AssertionError as e:
@@ -120,7 +130,7 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs):
             str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()]
             str_args = ', '.join(map(str, args) + str_kwargs)
             self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args))
-        except error, err:
+        except error as err:
             msg = self.convert_exception_to_str(err)
             if isinstance(regex, basestring):
                 regex = re.compile(regex)

test/headers.py

@@ -111,8 +111,8 @@ def test_gen_license_header(self):
         }
         for license in KNOWN_LICENSES.keys():
             res_fn = os.path.join(self.setup.REPO_TEST_DIR, 'headers', license)
-            result = open(res_fn).read()
-
+            with open(res_fn) as fh:
+                result = fh.read()
             gen_txt = gen_license_header(license, **data)
             self.assertEqual(gen_txt, result, msg='generated header for license %s as expected' % license)
             log.info('generated license header %s' % license)

test/shared_setup.py

@@ -110,8 +110,8 @@ def test_rel_gitignore(self):
         base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './testdata')
         try:
             self.setup.rel_gitignore(['testdata'], base_dir=base_dir)
-        except Exception as e:
-            self.assertTrue('.pyc' in e.message, msg=e)
+        except Exception as err:
+            self.assertTrue('.pyc' in str(err))
         else:
             self.assertTrue(False, 'rel_gitignore should have raised an exception, but did not!')
         # it should not fail if base_dir does not contain a .git folder
@@ -127,14 +127,13 @@ def test_action_target(self):
         """Test action_target function, mostly w.r.t. backward compatibility."""
         def fake_setup(*args, **kwargs):
             """Fake setup function to test action_target with."""
-            print 'args: ', args
-            print 'kwargs:', kwargs
+            print('args: %s' % str(args))
+            print('kwargs: %s' % kwargs)
 
         self.mock_stdout(True)
         action_target({'name': 'vsc-test', 'version': '1.0.0'}, setupfn=fake_setup)
         txt = self.get_stdout()
         self.mock_stdout(False)
-
         self.assertTrue(re.search(r"^args:\s*\(\)", txt, re.M))
         self.assertTrue(re.search(r"^kwargs:\s*\{.*'name':\s*'vsc-test'", txt, re.M))
 
@@ -150,6 +149,7 @@ def test_prepare_rpm(self):
         """
         Test the prepare rpm function
         especially in effect to generating correct package list wrt excluded_pkgs_rpm
+        we assume the order of the lists doesn't matter (and sort to compare)
         """
         package = {
             'name': 'vsc-test',
@@ -161,7 +161,8 @@ def test_prepare_rpm(self):
         libdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './testdata')
         setup.REPO_LIB_DIR = libdir
         setup.prepare_rpm(package)
-        self.assertEqual(setup.SHARED_TARGET['packages'], ['vsc', 'vsc.test'])
+
+        self.assertEqual(sorted(setup.SHARED_TARGET['packages']), ['vsc', 'vsc.test'])
         package = {
             'name': 'vsc-test',
             'excluded_pkgs_rpm': ['vsc', 'vsc.test'],
@@ -170,7 +171,8 @@ def test_prepare_rpm(self):
         setup = vsc_setup()
         setup.REPO_LIB_DIR = libdir
         setup.prepare_rpm(package)
-        self.assertEqual(setup.SHARED_TARGET['packages'], ['vsc', 'vsc.test'])
+
+        self.assertEqual(sorted(setup.SHARED_TARGET['packages']), ['vsc', 'vsc.test'])
 
     def test_parse_target(self):
         """Test for parse target"""