Add support for overridding whether resources are enabled

CHANGELOG.rst

@@ -11,6 +11,10 @@ Added
 
   Contributed by @khushboobhatia01
 
+* Added support to override enabled parameter of resources. #5506
+
+  Contributed by Amanda McGuinness (@amanda11 Intive)
+
 Fixed
 ~~~~~
 

st2common/st2common/bootstrap/actionsregistrar.py

@@ -142,6 +142,9 @@ def _register_action(self, pack, action):
         )
         content["metadata_file"] = metadata_file
 
+        # Pass override information
+        self._override_loader.override(pack, "actions", content)
+
         action_api = ActionAPI(**content)
 
         try:

st2common/st2common/bootstrap/aliasesregistrar.py

@@ -144,6 +144,9 @@ def _get_action_alias_db(
         else:
             content["metadata_file"] = metadata_file
 
+        # Pass override information
+        self._override_loader.override(pack, "aliases", content)
+
         action_alias_api = ActionAliasAPI(**content)
         action_alias_api.validate()
         action_alias_db = ActionAliasAPI.to_model(action_alias_api)

st2common/st2common/bootstrap/base.py

@@ -22,6 +22,7 @@
 from st2common import log as logging
 from st2common.constants.pack import CONFIG_SCHEMA_FILE_NAME
 from st2common.content.loader import MetaLoader
+from st2common.content.loader import OverrideLoader
 from st2common.content.loader import ContentPackLoader
 from st2common.models.api.pack import PackAPI
 from st2common.models.api.pack import ConfigSchemaAPI
@@ -68,6 +69,7 @@ def __init__(
         self._fail_on_failure = fail_on_failure
 
         self._meta_loader = MetaLoader()
+        self._override_loader = OverrideLoader()
         self._pack_loader = ContentPackLoader()
 
         # Maps runner name -> RunnerTypeDB

st2common/st2common/bootstrap/rulesregistrar.py

@@ -128,6 +128,9 @@ def _register_rules_from_pack(self, pack, rules):
                 )
                 content["metadata_file"] = metadata_file
 
+                # Pass override information
+                self._override_loader.override(pack, "rules", content)
+
                 rule_api = RuleAPI(**content)
                 rule_api.validate()
                 rule_db = RuleAPI.to_model(rule_api)

st2common/st2common/bootstrap/sensorsregistrar.py

@@ -167,6 +167,9 @@ def _register_sensor_from_pack(self, pack, sensor):
         )
         content["metadata_file"] = metadata_file
 
+        # Pass override information
+        self._override_loader.override(pack, "sensors", content)
+
         sensors_dir = os.path.dirname(sensor_metadata_file_path)
         sensor_file_path = os.path.join(sensors_dir, entry_point)
         artifact_uri = "file://%s" % (sensor_file_path)

st2common/st2common/content/loader.py

@@ -20,6 +20,7 @@
 from yaml.parser import ParserError
 import six
 
+from oslo_config import cfg
 from st2common import log as logging
 from st2common.constants.meta import ALLOWED_EXTS
 from st2common.constants.meta import PARSER_FUNCS
@@ -28,7 +29,7 @@
 if six.PY2:
     from io import open
 
-__all__ = ["ContentPackLoader", "MetaLoader"]
+__all__ = ["ContentPackLoader", "MetaLoader", "OverrideLoader"]
 
 LOG = logging.getLogger(__name__)
 
@@ -268,3 +269,124 @@ def _load(self, parser_func, file_path):
             except ParserError:
                 LOG.exception("Failed loading content from %s.", file_path)
                 raise
+
+
+class OverrideLoader(object):
+    """
+    Class for loading pack override data
+    """
+
+    # Mapping of permitted override types to resource name
+    ALLOWED_OVERRIDE_TYPES = {
+        "sensors": "class_name",
+        "actions": "name",
+        "rules": "name",
+        "aliases": "name",
+    }
+
+    ALLOWED_OVERRIDE_NAMES = [
+        "enabled",
+    ]
+
+    def override(self, pack_name, type, content):
+
+        """
+        Loads override content for pack, and updates content
+
+        :param pack_name: Name of pack
+        :type pack_name: ``str``
+        :param type: Type of resource loading
+        :type type: ``str``
+        :param content: Content as loaded from meta information
+        :type content: ``object``
+
+        """
+
+        if type not in self.ALLOWED_OVERRIDE_TYPES.keys():
+            raise ValueError(
+                f"Invalid override type of {type} attempted for pack {pack_name}"
+            )
+
+        override_dir = os.path.join(cfg.CONF.system.base_path, "configs/overrides")
+        # Apply global overrides
+        global_file = os.path.join(override_dir, "global.yaml")
+        self._apply_override_file(global_file, pack_name, type, content, True)
+
+        # Apply pack overrides
+        override_file = os.path.join(override_dir, f"{pack_name}.yaml")
+        self._apply_override_file(override_file, pack_name, type, content, False)
+
+        return content
+
+    def _apply_override_file(
+        self, override_file, pack_name, type, content, global_file
+    ):
+
+        """
+        Loads override content from override file
+
+        :param override_file: Override filename
+        :type override_file: ``str``
+        :param pack_name: Name of pack
+        :type pack_name: ``str``
+        :param type: Type of resource loading
+        :type type: ``str``
+        :param content: Content as loaded from meta information
+        :type content: ``object``
+        :param global_file: Whether global file
+        :type global_file: ``bool``
+        """
+
+        if not os.path.exists(override_file):
+            # No override file for pack
+            LOG.info(f"No override file {override_file} found")
+            return content
+
+        # Read override file
+        file_name, file_ext = os.path.splitext(override_file)
+        overrides = self._load(PARSER_FUNCS[file_ext], override_file)
+        # Apply overrides
+        if type in overrides.keys():
+            type_override = overrides[type]
+            name = content[self.ALLOWED_OVERRIDE_TYPES[type]]
+            if "defaults" in type_override.keys():
+                for key in type_override["defaults"].keys():
+                    if key in self.ALLOWED_OVERRIDE_NAMES:
+                        content[key] = type_override["defaults"][key]
+                        LOG.info(
+                            f"Overridden {type} {pack_name}.{name} {key} to default value of {content[key]} from {override_file}"
+                        )
+                    else:
+                        raise ValueError(
+                            f"Override attempted with invalid default key {key} in pack {pack_name}"
+                        )
+
+            if global_file:
+                # No exceptions required in global content file
+                return content
+
+            if "exceptions" in type_override.keys():
+                if name in type_override["exceptions"]:
+                    for key in type_override["exceptions"][name].keys():
+                        if key in self.ALLOWED_OVERRIDE_NAMES:
+                            content[key] = type_override["exceptions"][name][key]
+                            LOG.info(
+                                f"Overridden {type} {pack_name}.{name} {key} to exception value of {content[key]} from {override_file}"
+                            )
+                        else:
+                            raise ValueError(
+                                f"Override attempted with invalid exceptions key {key} in pack {pack_name}"
+                            )
+
+        return content
+
+    def _load(self, parser_func, file_path):
+        with open(file_path, "r", encoding="utf-8") as fd:
+            try:
+                return parser_func(fd)
+            except ValueError:
+                LOG.exception("Failed loading content from %s.", file_path)
+                raise
+            except ParserError:
+                LOG.exception("Failed loading content from %s.", file_path)
+                raise

st2common/tests/resources/configs/overrides/global.yaml

@@ -0,0 +1,4 @@
+---
+sensors:
+    defaults:
+        enabled: false

st2common/tests/resources/configs/overrides/overpack1.yaml

@@ -0,0 +1,7 @@
+---
+actions:
+  defaults:
+    enabled: False
+  exceptions:
+    action2:
+      enabled: True

st2common/tests/resources/configs/overrides/overpack2.yaml

@@ -0,0 +1,11 @@
+---
+actions:
+  defaults:
+    enabled: False
+    rubbish: False
+  exceptions:
+    action2:
+      enabled: True
+sensors:
+  defaults:
+    enabled: True

st2common/tests/resources/configs/overrides/overpack3.yaml

@@ -0,0 +1,11 @@
+---
+actions:
+  defaults:
+    enabled: False
+  exceptions:
+    action2:
+      rubbish: True
+sensors:
+  exceptions:
+    sensor1:
+      enabled: True

st2common/tests/resources/configs/overrides/overpack4.yaml

@@ -0,0 +1,4 @@
+---
+actions:
+  defaults:
+    enabled: False

st2common/tests/unit/test_content_loader.py

@@ -15,6 +15,7 @@
 
 from __future__ import absolute_import
 
+from oslo_config import cfg
 import os
 
 import unittest2
@@ -31,8 +32,10 @@
 from mock import Mock
 
 from st2common.content.loader import ContentPackLoader
+from st2common.content.loader import OverrideLoader
 from st2common.content.loader import LOG
 from st2common.constants.meta import yaml_safe_load
+from st2tests import config
 
 CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
 RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources"))
@@ -114,6 +117,104 @@ def test_get_content_from_pack_no_sensors(self):
         )
         self.assertEqual(result, None)
 
+    def test_get_override_action_from_default(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action1", "enabled": True}
+        loader.override("overpack1", "actions", content)
+        self.assertFalse(content["enabled"])
+        content = {"name": "action1", "enabled": False}
+        loader.override("overpack1", "actions", content)
+        self.assertFalse(content["enabled"])
+
+    def test_get_override_action_from_exception(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action2", "enabled": True}
+        loader.override("overpack1", "actions", content)
+        self.assertTrue(content["enabled"])
+        content = {"name": "action2", "enabled": False}
+        loader.override("overpack1", "actions", content)
+        self.assertTrue(content["enabled"])
+
+    def test_get_override_action_from_default_no_exceptions(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action1", "enabled": True}
+        loader.override("overpack4", "actions", content)
+        self.assertFalse(content["enabled"])
+        content = {"name": "action2", "enabled": True}
+        loader.override("overpack4", "actions", content)
+        self.assertFalse(content["enabled"])
+
+    def test_get_override_action_from_global_default_no_exceptions(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"class_name": "sensor1", "enabled": True}
+        loader.override("overpack1", "sensors", content)
+        self.assertFalse(content["enabled"])
+
+    def test_get_override_action_from_global_overridden_by_pack(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"class_name": "sensor1", "enabled": True}
+        loader.override("overpack2", "sensors", content)
+        self.assertTrue(content["enabled"])
+
+    def test_get_override_action_from_global_overridden_by_pack_exception(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"class_name": "sensor1", "enabled": True}
+        loader.override("overpack3", "sensors", content)
+        self.assertTrue(content["enabled"])
+
+    def test_get_override_invalid_type(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action2", "enabled": True}
+        self.assertRaises(
+            ValueError,
+            loader.override,
+            pack_name="overpack1",
+            type="wrongtype",
+            content=content,
+        )
+
+    def test_get_override_invalid_default_key(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action1", "enabled": True}
+        self.assertRaises(
+            ValueError,
+            loader.override,
+            pack_name="overpack2",
+            type="actions",
+            content=content,
+        )
+
+    def test_get_override_invalid_exceptions_key(self):
+        config.parse_args()
+        cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system")
+        loader = OverrideLoader()
+        content = {"name": "action1", "enabled": True}
+        loader.override("overpack1", "actions", content)
+        content = {"name": "action2", "enabled": True}
+        self.assertRaises(
+            ValueError,
+            loader.override,
+            pack_name="overpack3",
+            type="actions",
+            content=content,
+        )
+
 
 class YamlLoaderTestCase(unittest2.TestCase):
     def test_yaml_safe_load(self):