Add new "python_versions" attribute to the pack metadata file

contrib/packs/tests/test_action_download.py

@@ -241,41 +241,141 @@ def test_download_pack_stackstorm_version_identifier_check(self):
         action = self.get_action_instance()
 
         # Version is satisfied
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.0.0'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.0.0'
 
         result = action.run(packs=['test3'], abs_repo_base=self.repo_base)
         self.assertEqual(result['test3'], 'Success.')
 
         # Pack requires a version which is not satisfied by current StackStorm version
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.2.0'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.2.0'
         expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
                         'current version is "2.2.0"')
         self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'],
                                 abs_repo_base=self.repo_base)
 
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '2.3.0'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '2.3.0'
         expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
                         'current version is "2.3.0"')
         self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'],
                                 abs_repo_base=self.repo_base)
 
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.9'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.9'
         expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
                         'current version is "1.5.9"')
         self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'],
                                 abs_repo_base=self.repo_base)
 
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0'
         expected_msg = ('Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
                         'current version is "1.5.0"')
         self.assertRaisesRegexp(ValueError, expected_msg, action.run, packs=['test3'],
                                 abs_repo_base=self.repo_base)
 
         # Version is not met, but force=true parameter is provided
-        st2common.util.pack_management.CURRENT_STACKSTROM_VERSION = '1.5.0'
+        st2common.util.pack_management.CURRENT_STACKSTORM_VERSION = '1.5.0'
         result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=True)
         self.assertEqual(result['test3'], 'Success.')
 
+    def test_download_pack_python_version_check(self):
+        action = self.get_action_instance()
+
+        # No python_versions attribute specified in the metadata file
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': []
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.11'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
+            self.assertEqual(result['test3'], 'Success.')
+
+        # Pack works with Python 2.x installation is running 2.7
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['2']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
+            self.assertEqual(result['test3'], 'Success.')
+
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.12'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
+            self.assertEqual(result['test3'], 'Success.')
+
+        # Pack works with Python 2.x installation is running 3.5
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['2']
+            }
+
+            st2common.util.pack_management.six.PY2 = False
+            st2common.util.pack_management.six.PY3 = True
+
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.5.2'
+
+            expected_msg = (r'Pack "test3" requires Python 2.x, but current Python version is '
+                            '"3.5.2"')
+            self.assertRaisesRegexp(ValueError, expected_msg, action.run,
+                                    packs=['test3'], abs_repo_base=self.repo_base, force=False)
+
+        # Pack works with Python 3.x installation is running 2.7
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['3']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.2'
+
+            expected_msg = (r'Pack "test3" requires Python 3.x, but current Python version is '
+                            '"2.7.2"')
+            self.assertRaisesRegexp(ValueError, expected_msg, action.run,
+                                    packs=['test3'], abs_repo_base=self.repo_base, force=False)
+
+        # Pack works with Python 2.x and 3.x installation is running 2.7 and 3.6.1
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['2', '3']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
+            self.assertEqual(result['test3'], 'Success.')
+
+            st2common.util.pack_management.six.PY2 = False
+            st2common.util.pack_management.six.PY3 = True
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '3.6.1'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
+            self.assertEqual(result['test3'], 'Success.')
+
     def test_resolve_urls(self):
         url = eval_repo_url(
             "https://github.com/StackStorm-Exchange/stackstorm-test")

st2common/st2common/models/api/pack.py

@@ -100,6 +100,22 @@ class PackAPI(BaseAPI):
                                '">=1.8.0, <2.2.0"',
                 'pattern': ST2_VERSION_REGEX,
             },
+            'python_versions': {
+                'type': 'array',
+                'description': ('Major Python versions supported by this pack. E.g. '
+                                '"2" for Python 2.7.x and "3" for Python 3.6.x'),
+                'items': {
+                    'type': 'string',
+                    'enum': [
+                        '2',
+                        '3'
+                    ]
+                },
+                'minItems': 1,
+                'maxItems': 2,
+                'uniqueItems': True,
+                'additionalItems': True
+            },
             'author': {
                 'type': 'string',
                 'description': 'Pack author or authors.',
@@ -144,7 +160,10 @@ class PackAPI(BaseAPI):
                 'description': 'Location of the pack on disk in st2 system.',
                 'required': False
             }
-        }
+        },
+        # NOTE: We add this here explicitly so we can gracefuly add new attributs to pack.yaml
+        # without breaking existing installations
+        'additionalProperties': True
     }
 
     def __init__(self, **values):
@@ -189,6 +208,7 @@ def to_model(cls, pack):
         version = str(pack.version)
 
         stackstorm_version = getattr(pack, 'stackstorm_version', None)
+        python_versions = getattr(pack, 'python_versions', [])
         author = pack.author
         email = pack.email
         contributors = getattr(pack, 'contributors', [])
@@ -200,7 +220,8 @@ def to_model(cls, pack):
         model = cls.model(ref=ref, name=name, description=description, keywords=keywords,
                           version=version, author=author, email=email, contributors=contributors,
                           files=files, dependencies=dependencies, system=system,
-                          stackstorm_version=stackstorm_version, path=pack_dir)
+                          stackstorm_version=stackstorm_version, path=pack_dir,
+                          python_versions=python_versions)
         return model
 
 

st2common/st2common/models/db/pack.py

@@ -47,6 +47,7 @@ class PackDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin,
     keywords = me.ListField(field=me.StringField())
     version = me.StringField(regex=PACK_VERSION_REGEX, required=True)
     stackstorm_version = me.StringField(regex=ST2_VERSION_REGEX)
+    python_versions = me.ListField(field=me.StringField())
     author = me.StringField(required=True)
     email = me.EmailField()
     contributors = me.ListField(field=me.StringField())

st2common/st2common/util/pack_management.py

@@ -26,6 +26,7 @@
 import stat
 import re
 
+import six
 from git.repo import Repo
 from gitdb.exc import BadName, BadObject
 from lockfile import LockFile
@@ -42,6 +43,7 @@
 from st2common.util.green import shell
 from st2common.util.versioning import complex_semver_match
 from st2common.util.versioning import get_stackstorm_version
+from st2common.util.versioning import get_python_version
 
 __all__ = [
     'download_pack',
@@ -58,7 +60,8 @@
 LOG = logging.getLogger(__name__)
 
 CONFIG_FILE = 'config.yaml'
-CURRENT_STACKSTROM_VERSION = get_stackstorm_version()
+CURRENT_STACKSTORM_VERSION = get_stackstorm_version()
+CURRENT_PYTHON_VERSION = get_python_version()
 
 
 def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False,
@@ -375,17 +378,33 @@ def verify_pack_version(pack_dir):
     pack_metadata = get_pack_metadata(pack_dir=pack_dir)
     pack_name = pack_metadata.get('name', None)
     required_stackstorm_version = pack_metadata.get('stackstorm_version', None)
+    supported_python_versions = pack_metadata.get('python_versions', None)
 
-    # If stackstorm_version attribute is speficied, verify that the pack works with currently
+    # If stackstorm_version attribute is specified, verify that the pack works with currently
     # running version of StackStorm
     if required_stackstorm_version:
-        if not complex_semver_match(CURRENT_STACKSTROM_VERSION, required_stackstorm_version):
-            msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". ' %
-                   (pack_name, required_stackstorm_version, CURRENT_STACKSTROM_VERSION),
-                   'You can override this restriction by providing the "force" flag, but ',
-                   'the pack is not guaranteed to work.')
+        if not complex_semver_match(CURRENT_STACKSTORM_VERSION, required_stackstorm_version):
+            msg = ('Pack "%s" requires StackStorm "%s", but current version is "%s". '
+                   'You can override this restriction by providing the "force" flag, but '
+                   'the pack is not guaranteed to work.' %
+                   (pack_name, required_stackstorm_version, CURRENT_STACKSTORM_VERSION))
             raise ValueError(msg)
 
+    if supported_python_versions:
+        if set(supported_python_versions) == set(['2']) and not six.PY2:
+            msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". '
+                   'You can override this restriction by providing the "force" flag, but '
+                   'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION))
+            raise ValueError(msg)
+        elif set(supported_python_versions) == set(['3']) and not six.PY3:
+            msg = ('Pack "%s" requires Python 3.x, but current Python version is "%s". '
+                   'You can override this restriction by providing the "force" flag, but '
+                   'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION))
+            raise ValueError(msg)
+        else:
+            # Pack support Python 2.x and 3.x so no check is needed
+            pass
+
     return True
 
 

st2common/st2common/util/versioning.py

@@ -18,12 +18,16 @@
 """
 
 from __future__ import absolute_import
+
+import sys
+
 import semver
 
 from st2common import __version__ as stackstorm_version
 
 __all__ = [
     'get_stackstorm_version',
+    'get_python_version',
 
     'complex_semver_match'
 ]
@@ -41,13 +45,27 @@ def get_stackstorm_version():
     return stackstorm_version
 
 
+def get_python_version():
+    """
+    Return Python version used by this installation.
+    """
+    version_info = sys.version_info
+    return '%s.%s.%s' % (version_info.major, version_info.minor, version_info.micro)
+
+
 def complex_semver_match(version, version_specifier):
     """
     Custom semver match function which also supports complex semver specifiers
     such as >=1.6, <2.0, etc.
 
+    NOTE: This function also supports special "all" version specifier. When "all"
+    is specified, any version provided will be considered valid.
+
     :rtype: ``bool``
     """
+    if version_specifier == 'all':
+        return True
+
     split_version_specifier = version_specifier.split(',')
 
     if len(split_version_specifier) == 1:

st2common/tests/unit/test_resource_registrar.py

@@ -43,6 +43,8 @@
 PACK_PATH_14 = os.path.join(get_fixtures_base_path(), 'packs/dummy_pack_14')
 PACK_PATH_17 = os.path.join(get_fixtures_base_path(), 'packs_invalid/dummy_pack_17')
 PACK_PATH_18 = os.path.join(get_fixtures_base_path(), 'packs_invalid/dummy_pack_18')
+PACK_PATH_20 = os.path.join(get_fixtures_base_path(), 'packs/dummy_pack_20')
+PACK_PATH_21 = os.path.join(get_fixtures_base_path(), 'packs/dummy_pack_21')
 
 
 class ResourceRegistrarTestCase(CleanDbTestCase):
@@ -89,6 +91,23 @@ def test_register_packs(self):
         for excluded_file in excluded_files:
             self.assertTrue(excluded_file not in pack_db.files)
 
+    def test_register_pack_arbitrary_properties_are_allowed(self):
+        # Test registering a pack which has "arbitrary" properties in pack.yaml
+        # We support this use-case (ignore properties which are not defined on the PackAPI model)
+        # so we can add new attributes in a new version without breaking existing installations.
+        registrar = ResourceRegistrar(use_pack_cache=False)
+        registrar._pack_loader.get_packs = mock.Mock()
+        registrar._pack_loader.get_packs.return_value = {
+            'dummy_pack_20': PACK_PATH_20,
+        }
+        packs_base_paths = content_utils.get_packs_base_paths()
+        registrar.register_packs(base_dirs=packs_base_paths)
+
+        # Ref is provided
+        pack_db = Pack.get_by_name('dummy_pack_20')
+        self.assertEqual(pack_db.ref, 'dummy_pack_20_ref')
+        self.assertEqual(len(pack_db.contributors), 0)
+
     def test_register_pack_pack_ref(self):
         # Verify DB is empty
         pack_dbs = Pack.get_all()
@@ -162,6 +181,7 @@ def test_register_pack_pack_stackstorm_version_and_future_parameters(self):
         self.assertEqual(pack_db.dependencies, ['core=0.2.0'])
         self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')
         self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}})
+        self.assertEqual(pack_db.python_versions, ['2', '3'])
 
         # Note: We only store parameters which are defined in the schema, all other custom user
         # defined attributes are ignored
@@ -192,3 +212,13 @@ def test_register_pack_invalid_config_schema_invalid_attribute(self):
         expected_msg = r'Additional properties are not allowed \(\'invalid\' was unexpected\)'
         self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
                                 base_dirs=packs_base_paths)
+
+    def test_register_pack_invalid_python_versions_attribute(self):
+        registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
+        registrar._pack_loader.get_packs = mock.Mock()
+        registrar._pack_loader.get_packs.return_value = {'dummy_pack_21': PACK_PATH_21}
+        packs_base_paths = content_utils.get_packs_base_paths()
+
+        expected_msg = r"'4' is not one of \['2', '3'\]"
+        self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
+                                base_dirs=packs_base_paths)

st2common/tests/unit/test_versioning_utils.py

@@ -29,6 +29,11 @@ def test_complex_semver_match(self):
         self.assertTrue(complex_semver_match('2.1.0', '>=1.6.0, <2.2.0'))
         self.assertTrue(complex_semver_match('2.1.9', '>=1.6.0, <2.2.0'))
 
+        self.assertTrue(complex_semver_match('1.6.0', 'all'))
+        self.assertTrue(complex_semver_match('1.6.1', 'all'))
+        self.assertTrue(complex_semver_match('2.0.0', 'all'))
+        self.assertTrue(complex_semver_match('2.1.0', 'all'))
+
         self.assertTrue(complex_semver_match('1.6.0', '>=1.6.0'))
         self.assertTrue(complex_semver_match('1.6.1', '>=1.6.0'))
         self.assertTrue(complex_semver_match('2.1.0', '>=1.6.0'))