From 5eaf09ed909abd66d6e8c3251b64bb04aae51b16 Mon Sep 17 00:00:00 2001 From: ghooo Date: Mon, 29 Mar 2021 12:39:57 -0700 Subject: [PATCH 01/32] sekelton --- config/generic_update.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 config/generic_update.py diff --git a/config/generic_update.py b/config/generic_update.py new file mode 100644 index 0000000000..faaa8d2885 --- /dev/null +++ b/config/generic_update.py @@ -0,0 +1,27 @@ +class JsonChange: + pass + +class PatchOrderer: + def order(patch): + pass + +class ChangeApplier: + def apply(change): + pass + +class PatchApplier: + def apply(self, patch, format, verbose, dry_run): + pass + +class ConfigReplacer: + def replace(self, full_json, format, verbose, dry_run): + pass + +class FileSystemRollbacker: + def checkpoint(self, checkpoint_name, format, verbose, dry_run): + pass + def rollback(self, checkpoint_name, format, verbose, dry_run): + pass + + + From 8305b6717129592599b7b695c7b3c9a807f5d4ff Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 30 Mar 2021 09:57:27 -0700 Subject: [PATCH 02/32] impl --- config/generic_update.py | 319 ++++++++++++++++++++++++++++++++++- tests/generic_update_test.py | 10 ++ 2 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 tests/generic_update_test.py diff --git a/config/generic_update.py b/config/generic_update.py index faaa8d2885..4b48766fa3 100644 --- a/config/generic_update.py +++ b/config/generic_update.py @@ -1,27 +1,332 @@ +import jsonpatch +import sonic_yang +import os +load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') +from sonic_cfggen import deep_update, FormatConverter + class JsonChange: - pass + def __init__(self, patch): + self.patch = patch + + def apply(self, current_json): + return self.patch.apply(current_json) + + def __str__(self): + return f"{self.patch}" + +class ConfigLocker(): + def acquire_lock(self): + # TODO: Implement ConfigLocker + pass + + def release_lock(self): + # TODO: Implement ConfigLocker + pass + +class NoOpConfigLocker(ConfigLocker): + def acquire_lock(self): + pass + def release_lock(self): + pass class PatchOrderer: def order(patch): + # TODO: Implement patch orderer pass class ChangeApplier: def apply(change): + # TODO: Implement change applier pass +YANG_DIR = "/usr/local/yang-models" +class ConfigWrapper: + def get_current_config(self): + config_db_json = __get_config_db_as_json() + return __convert_config_db_to_yang(config_db_json) + + def validate_config(self, yang_as_json): + config_db_as_json = __convert_yang_to_config_db(yang_as_json) + config_as_yang + + sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + sy.loadYangModel() + sy.loadData(config_db_as_json) + + try: + sy.validate_data_tree() + return True + except Exception as ex: + return False + + def get_config_db_as_json(self): + db_kwargs = dict(); data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect() + deep_update(data, FormatConverter.db_to_output(configdb.get_config())) + return FormatConverter.to_serialized(data) + + def convert_config_db_to_yang(self, config_db_as_json): + sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + sy.loadYangModel() + + yang_as_json = dict() + sy._xlateConfigDBtoYang(config_db_as_json, yang_as_json) + + return yang_as_json + + def convert_yang_to_config_db(self, yang_as_json): + sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + sy.loadYangModel() + + config_db_as_json = dict() + sy.xlateJson = yang_as_json + sy.revXlateJson = config_db_as_json + sy._revXlateYangtoConfigDB(config_db_as_json, yang_as_json) + + return yang_as_json + class PatchApplier: + def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): + self.patchorderer = patchorderer + self.changeapplier = changeapplier + self.configlocker = configlocker + def apply(self, patch, format, verbose, dry_run): - pass + if format == "ConfigDB": + yang_patch = __convert_config_db_patch_to_yang_patch(patch) + elif format == "YANG": + yang_patch = patch + else: + raise AttributeError(f"format argument value '{format}', the supported values are 'ConfigDB' or 'YANG'") + + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + __acquire_config_lock() + old_config = __get_current_config() + target_config = __simulate_patch(patch, old_config) + + if not(__validate(target_config)): + raise Exception(f"The given patch is not valid") + + changes = __order_patch(patch) + for change in changes: + __apply_change(change) + + new_config = __get_current_config() + if not(__verify_same_json(target_config, new_config)): + raise Exception(f"After applying patch to config, there is still some parts not updated") + __release_config_lock() + + def __convert_config_db_patch_to_yang_patch(patch): + current_config_db = self.configwrapper.get_config_db_as_json() + target_config_db = self.__simulate_patch(patch, current_config_db) + + current_yang = self.configwrapper.convert_config_db_to_yang(current_config_db) + target_yang = self.configwrapper.convert_config_db_to_yang(target_config_db) + + return __generate_patch(current_yang, target_yang) + + def __acquire_config_lock(): + self.configlocker.acquire_lock() + + def __release_config_lock(): + self.configlocker.release_lock() + + def __get_current_config(): + self.configwrapper.get_current_config() + + def __simulate_patch(patch, jsonconfig): + return patch.apply(jsonconfig) + + def __validate(target_config): + return self.configwrapper.validate_config(target_config) + + def __order_patch(patch): + return self.patchorderer.order(patch) + + def __apply_change(change): + return self.changeapplier.apply(change) + + def __generate_patch(current, target): + return jsonpatch.make_patch(current, target) + + def __verify_same_json(expected_json, actual_json): + return jsonpatch.make_patch(expected_json, actual_json) == [] class ConfigReplacer: + def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): + # NOTE: patch-applier receives a NoOpConfigLocker + self.patchapplier = PatchApplier(patchorder, changeapplier, NoOpConfigLocker(), configwrapper) + self.configwrapper = configwrapper + self.configlocker = configlocker + def replace(self, full_json, format, verbose, dry_run): - pass + if format == "ConfigDB": + yang_as_json = convert_config_db_to_yang(full_json) + elif format == "YANG": + yang_as_json = full_json + else: + raise AttributeError(f"format argument value '{format}', the supported values are 'ConfigDB' or 'YANG'") + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + __acquire_config_lock() + if not(__validate(target_config)): + raise Exception(f"The given target config is not valid") + + old_config = __get_current_config() + patch = __generate_patch(old_config, target_config) + + __apply_patch(patch, verbose, dry_run) + + new_config = __get_current_config() + if not(__verify_same_json(target_config, new_config)): + raise Exception(f"After applying patch to config, there is still some parts not updated") + + __release_config_lock() + + def __acquire_config_lock(): + self.configlocker.acquire_lock() + + def __release_config_lock(): + self.configlocker.release_lock() + + def __validate(target_config): + return self.configwrapper.validate_config(target_config) + + def __get_current_config(): + self.configwrapper.get_current_config() + + def __generate_patch(current, target): + return jsonpatch.make_patch(current, target) + + def __apply_patch(patch, verbose, dry_run): + self.patchapplier.apply(patch, verbose, dry_run) + + def __verify_same_json(expected_json, actual_json): + return jsonpatch.make_patch(expected_json, actual_json) == [] + +CHECKPOINTS_DIR = "/etc/sonic/checkpoints" +CHECKPOINT_EXT = ".cp.json" class FileSystemRollbacker: - def checkpoint(self, checkpoint_name, format, verbose, dry_run): - pass - def rollback(self, checkpoint_name, format, verbose, dry_run): - pass + def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): + # NOTE: config-replacer receives a NoOpConfigLocker + self.configreplacer = PatchApplier(patchorder, changeapplier, NoOpConfigLocker(), configwrapper) + self.configlocker = configlocker + + def rollback(self, checkpoint_name, verbose, dry_run): + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + __acquire_config_lock() + target_config = __get_checkpoint_content(checkpoint_name) + + __config_replace(target_config) + + __release_lock() + + def checkpoint(self, checkpoint_name, verbose, dry_run): + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + __acquire_config_lock() + current_config = _get_current_config() + + __save_checkpoint_content(current_config) + + __release_lock() + + def list_checkpoints(self, verbose, dry_run): + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + if not(__checkpoints_dir_exist()): + return [] + + return __get_checkpoint_names() + + def delete_checkpoint(self, checkpoint_name, verbose, dry_run): + if verbose: + # TODO: Implement verbose logging + raise NotImplementedError + + if dry_run: + # TODO: Implement dry-run + raise NotImplementedError + + if not(__check_checkpoint_exists()): + raise Exception("Checkpoint does not exist") + + __delete_checkpoint(checkpoint_name) + + def __acquire_config_lock(): + self.configlocker.acquire_lock() + + def __release_config_lock(): + self.configlocker.release_lock() + + def __get_current_config(): + self.configwrapper.get_current_config() + + def __ensure_checkpoints_dir_exists(self): + os.makedirs(CHECKPOINTS_DIR, exist_ok=True) + + def __save_checkpoint_content(name, content): + __ensure_checkpoints_dir_exists() + path = __get_checkpoint_full_path(name) + with open(path, "w") as fh: + fh.write(text) + + def __get_checkpoint_content(name): + path = __get_checkpoint_full_path(name) + with open(path) as fh: + text = fh.read() + return json.loads(text) + + def __get_checkpoint_full_path(name) + return os.path.join(CHECKPOINTS_DIR, name, CHECKPOINT_EXT) + + def __config_replace(target_config): + self.configreplacer.replace(target_config) + + def __get_checkpoint_names(): + return [f for f in listdir(CHECKPOINTS_DIR) if f.endswith(CHECKPOINT_EXT)] + def __checkpoints_dir_exist(): + os.path.isdir(CHECKPOINTS_DIR) + def __check_checkpoint_exists(name): + path = __get_checkpoint_full_path(name) + return os.path.isfile(path) + def __delete_checkpoint(name): + path = __get_checkpoint_full_path(name) + return os.remove(path) diff --git a/tests/generic_update_test.py b/tests/generic_update_test.py new file mode 100644 index 0000000000..e2afc8f2e2 --- /dev/null +++ b/tests/generic_update_test.py @@ -0,0 +1,10 @@ +import unittest +import os +from imp import load_source +load_source('generic_update', \ + os.path.join(os.path.dirname(__file__), '..', 'config', 'generic_update.py')) +import generic_update + +# TODO: Add unit-tests +class Test(unittest.TestCase): + pass From 0175885ed2f2cdc713f56cf4961dff81e800afd5 Mon Sep 17 00:00:00 2001 From: ghooo Date: Mon, 5 Apr 2021 15:29:00 -0700 Subject: [PATCH 03/32] adding unit-tests and replying to comments --- config/generic_update.py | 550 ++++++----- setup.py | 3 +- tests/generic_update_test.py | 1742 +++++++++++++++++++++++++++++++++- 3 files changed, 2069 insertions(+), 226 deletions(-) diff --git a/config/generic_update.py b/config/generic_update.py index 4b48766fa3..cb24bf41fd 100644 --- a/config/generic_update.py +++ b/config/generic_update.py @@ -1,32 +1,25 @@ +import json import jsonpatch import sonic_yang import os +import copy +from enum import Enum +from swsssdk import ConfigDBConnector +from imp import load_source load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter class JsonChange: - def __init__(self, patch): - self.patch = patch + # TODO: Implement JsonChange + pass - def apply(self, current_json): - return self.patch.apply(current_json) - - def __str__(self): - return f"{self.patch}" - -class ConfigLocker(): +class ConfigLock(): def acquire_lock(self): - # TODO: Implement ConfigLocker - pass - - def release_lock(self): - # TODO: Implement ConfigLocker + # TODO: Implement ConfigLock pass -class NoOpConfigLocker(ConfigLocker): - def acquire_lock(self): - pass def release_lock(self): + # TODO: Implement ConfigLock pass class PatchOrderer: @@ -41,292 +34,407 @@ def apply(change): YANG_DIR = "/usr/local/yang-models" class ConfigWrapper: - def get_current_config(self): - config_db_json = __get_config_db_as_json() - return __convert_config_db_to_yang(config_db_json) - - def validate_config(self, yang_as_json): - config_db_as_json = __convert_yang_to_config_db(yang_as_json) - config_as_yang - - sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) - sy.loadYangModel() - sy.loadData(config_db_as_json) - - try: - sy.validate_data_tree() - return True - except Exception as ex: - return False + def __init__(self, default_config_db_connector = None, yang_dir = YANG_DIR): + self.default_config_db_connector = default_config_db_connector + self.yang_dir = YANG_DIR def get_config_db_as_json(self): - db_kwargs = dict(); data = dict() - configdb = ConfigDBConnector(**db_kwargs) - configdb.connect() + configdb = self.__create_and_connect_config_db() + data = dict() deep_update(data, FormatConverter.db_to_output(configdb.get_config())) return FormatConverter.to_serialized(data) - def convert_config_db_to_yang(self, config_db_as_json): - sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + def get_sonic_yang_as_json(self): + config_db_json = self.get_config_db_as_json() + return self.convert_config_db_to_sonic_yang(config_db_json) + + def convert_config_db_to_sonic_yang(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) sy.loadYangModel() - yang_as_json = dict() - sy._xlateConfigDBtoYang(config_db_as_json, yang_as_json) + cropped_config_db_as_json = self.crop_tables_without_yang(config_db_as_json) - return yang_as_json + sonic_yang_as_json = dict() - def convert_yang_to_config_db(self, yang_as_json): - sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + sy._xlateConfigDBtoYang(cropped_config_db_as_json, sonic_yang_as_json) + + return sonic_yang_as_json + + def convert_sonic_yang_to_config_db(self, sonic_yang_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) sy.loadYangModel() + # replace container of the format 'module:table' with just 'table' + new_sonic_yang_json = {} + for module_top in sonic_yang_as_json.keys(): + new_sonic_yang_json[module_top] = {} + for container in sonic_yang_as_json[module_top].keys(): + table = container.split(':')[1] + new_sonic_yang_json[module_top][table] = sonic_yang_as_json[module_top][container] + config_db_as_json = dict() - sy.xlateJson = yang_as_json + sy.xlateJson = new_sonic_yang_json sy.revXlateJson = config_db_as_json - sy._revXlateYangtoConfigDB(config_db_as_json, yang_as_json) + sy._revXlateYangtoConfigDB(new_sonic_yang_json, config_db_as_json) - return yang_as_json + return config_db_as_json -class PatchApplier: - def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): - self.patchorderer = patchorderer - self.changeapplier = changeapplier - self.configlocker = configlocker + def validate_sonic_yang_config(self, sonic_yang_as_json): + config_db_as_json = self.convert_sonic_yang_to_config_db(sonic_yang_as_json) - def apply(self, patch, format, verbose, dry_run): - if format == "ConfigDB": - yang_patch = __convert_config_db_patch_to_yang_patch(patch) - elif format == "YANG": - yang_patch = patch - else: - raise AttributeError(f"format argument value '{format}', the supported values are 'ConfigDB' or 'YANG'") + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + try: + sy.loadData(config_db_as_json) - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError + sy.validate_data_tree() + return True + except Exception as ex: + return False - __acquire_config_lock() - old_config = __get_current_config() - target_config = __simulate_patch(patch, old_config) + def crop_tables_without_yang(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() - if not(__validate(target_config)): - raise Exception(f"The given patch is not valid") + sy.jIn = copy.deepcopy(config_db_as_json) - changes = __order_patch(patch) - for change in changes: - __apply_change(change) + sy.tablesWithOutYang = dict() - new_config = __get_current_config() - if not(__verify_same_json(target_config, new_config)): - raise Exception(f"After applying patch to config, there is still some parts not updated") - __release_config_lock() + sy._cropConfigDB() - def __convert_config_db_patch_to_yang_patch(patch): - current_config_db = self.configwrapper.get_config_db_as_json() - target_config_db = self.__simulate_patch(patch, current_config_db) + return sy.jIn + + def __create_and_connect_config_db(self): + if self.default_config_db_connector != None: + return self.default_config_db_connector + + db_kwargs = dict() + data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect() + return configdb + +class DryRunConfigWrapper(ConfigWrapper): + # TODO: implement DryRunConfigWrapper + # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. + pass - current_yang = self.configwrapper.convert_config_db_to_yang(current_config_db) - target_yang = self.configwrapper.convert_config_db_to_yang(target_config_db) +class PatchWrapper(): + def __init__(self, configwrapper = ConfigWrapper()): + self.configwrapper = configwrapper + + def validate_config_db_patch(self, patch): + config_db = {} + for operation in patch: + tokens = operation['path'].split('/')[1:] + if len(tokens) == 0: # Modifying whole config_db + tables_dict = {table_name: {} for table_name in operation['value']} + config_db.update(tables_dict) + elif not tokens[0]: # Not empty + raise Exception("Table name in patch cannot be empty") + else: + config_db[tokens[0]] = {} - return __generate_patch(current_yang, target_yang) + cropped_config_db = self.configwrapper.crop_tables_without_yang(config_db) - def __acquire_config_lock(): - self.configlocker.acquire_lock() + # valid if no tables dropped during cropping + return len(cropped_config_db.keys()) == len(config_db.keys()) - def __release_config_lock(): - self.configlocker.release_lock() + def verify_same_json(self, expected, actual): + # patch will be [] if no diff, [] evaluates to False + return not jsonpatch.make_patch(expected, actual) - def __get_current_config(): - self.configwrapper.get_current_config() + def generate_patch(self, current, target): + return jsonpatch.make_patch(current, target) - def __simulate_patch(patch, jsonconfig): + def simulate_patch(self, patch, jsonconfig): return patch.apply(jsonconfig) - def __validate(target_config): - return self.configwrapper.validate_config(target_config) + def convert_config_db_patch_to_sonic_yang_patch(self, patch): + if not(self.validate_config_db_patch(patch)): + raise Exception(f"Given patch is not valid") - def __order_patch(patch): - return self.patchorderer.order(patch) + current_config_db = self.configwrapper.get_config_db_as_json() + target_config_db = self.simulate_patch(patch, current_config_db) - def __apply_change(change): - return self.changeapplier.apply(change) + current_yang = self.configwrapper.convert_config_db_to_sonic_yang(current_config_db) + target_yang = self.configwrapper.convert_config_db_to_sonic_yang(target_config_db) - def __generate_patch(current, target): - return jsonpatch.make_patch(current, target) + return self.generate_patch(current_yang, target_yang) - def __verify_same_json(expected_json, actual_json): - return jsonpatch.make_patch(expected_json, actual_json) == [] +class ConfigFormat(Enum): + SONICYANG = 1 + CONFIGDB = 2 -class ConfigReplacer: - def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): - # NOTE: patch-applier receives a NoOpConfigLocker - self.patchapplier = PatchApplier(patchorder, changeapplier, NoOpConfigLocker(), configwrapper) +class PatchApplier: + def __init__( \ + self, \ + patchorderer = PatchOrderer(), \ + changeapplier = ChangeApplier(), \ + configwrapper = ConfigWrapper(), \ + patchwrapper = PatchWrapper()): + self.patchorderer = patchorderer + self.changeapplier = changeapplier self.configwrapper = configwrapper - self.configlocker = configlocker + self.patchwrapper = patchwrapper - def replace(self, full_json, format, verbose, dry_run): - if format == "ConfigDB": - yang_as_json = convert_config_db_to_yang(full_json) - elif format == "YANG": - yang_as_json = full_json - else: - raise AttributeError(f"format argument value '{format}', the supported values are 'ConfigDB' or 'YANG'") + def apply(self, patch): + # 1. Get old config as SONiC Yang + old_config = self.configwrapper.get_sonic_yang_as_json() - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + # 2. Generate target config + target_config = self.patchwrapper.simulate_patch(patch, old_config) - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError + # 3. Validate target config + if not(self.configwrapper.validate_sonic_yang_config(target_config)): + raise Exception(f"The given patch is not valid") + + # 4. Generate list of changes to apply + changes = self.patchorderer.order(patch) + + # 5. Apply changes in order + for change in changes: + self.changeapplier.apply(change) + + # 6. Validate config updated successfully + new_config = self.configwrapper.get_sonic_yang_as_json() + if not(self.patchwrapper.verify_same_json(target_config, new_config)): + raise Exception(f"After applying patch to config, there are still some parts not updated") + +class ConfigReplacer: + def __init__(self, patchapplier = PatchApplier(), configwrapper = ConfigWrapper(), patchwrapper = PatchWrapper()): + self.patchapplier = patchapplier + self.configwrapper = configwrapper + self.patchwrapper = patchwrapper - __acquire_config_lock() - if not(__validate(target_config)): + def replace(self, target_config): + if not(self.configwrapper.validate_sonic_yang_config(target_config)): raise Exception(f"The given target config is not valid") - old_config = __get_current_config() - patch = __generate_patch(old_config, target_config) + old_config = self.configwrapper.get_sonic_yang_as_json() + patch = self.patchwrapper.generate_patch(old_config, target_config) - __apply_patch(patch, verbose, dry_run) + self.patchapplier.apply(patch) - new_config = __get_current_config() - if not(__verify_same_json(target_config, new_config)): + new_config = self.configwrapper.get_sonic_yang_as_json() + if not(self.patchwrapper.verify_same_json(target_config, new_config)): raise Exception(f"After applying patch to config, there is still some parts not updated") - __release_config_lock() +CHECKPOINTS_DIR = "/etc/sonic/checkpoints" +CHECKPOINT_EXT = ".cp.json" +class FileSystemConfigRollbacker: + def __init__( \ + self, \ + checkpoints_dir = CHECKPOINTS_DIR, \ + config_replacer = ConfigReplacer(), \ + configwrapper = ConfigWrapper()): + self.checkpoints_dir = checkpoints_dir + self.config_replacer = config_replacer + self.configwrapper = configwrapper - def __acquire_config_lock(): - self.configlocker.acquire_lock() + def rollback(self, checkpoint_name): + if not self.__check_checkpoint_exists(checkpoint_name): + raise Exception(f"Checkpoint '{checkpoint_name}' does not exist") - def __release_config_lock(): - self.configlocker.release_lock() + target_config = self.__get_checkpoint_content(checkpoint_name) - def __validate(target_config): - return self.configwrapper.validate_config(target_config) + self.config_replacer.replace(target_config) - def __get_current_config(): - self.configwrapper.get_current_config() + def checkpoint(self, checkpoint_name): + json_content = self.configwrapper.get_sonic_yang_as_json() - def __generate_patch(current, target): - return jsonpatch.make_patch(current, target) + path = self.__get_checkpoint_full_path(checkpoint_name) - def __apply_patch(patch, verbose, dry_run): - self.patchapplier.apply(patch, verbose, dry_run) + self.__ensure_checkpoints_dir_exists() - def __verify_same_json(expected_json, actual_json): - return jsonpatch.make_patch(expected_json, actual_json) == [] + self.__save_json_file(path, json_content) -CHECKPOINTS_DIR = "/etc/sonic/checkpoints" -CHECKPOINT_EXT = ".cp.json" -class FileSystemRollbacker: - def __init__(self, patchorderer = PatchOrderer(), changeapplier = ChangeApplier(), configlocker = ConfigLocker(), configwrapper = ConfigWrapper()): - # NOTE: config-replacer receives a NoOpConfigLocker - self.configreplacer = PatchApplier(patchorder, changeapplier, NoOpConfigLocker(), configwrapper) - self.configlocker = configlocker + def list_checkpoints(self): + if not self.__checkpoints_dir_exist(): + return [] - def rollback(self, checkpoint_name, verbose, dry_run): - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + return self.__get_checkpoint_names() - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError + def delete_checkpoint(self, checkpoint_name): + if not self.__check_checkpoint_exists(checkpoint_name): + raise Exception("Checkpoint does not exist") - __acquire_config_lock() - target_config = __get_checkpoint_content(checkpoint_name) + self.__delete_checkpoint(checkpoint_name) - __config_replace(target_config) + def __ensure_checkpoints_dir_exists(self): + os.makedirs(self.checkpoints_dir, exist_ok=True) - __release_lock() + def __save_json_file(self, path, json_content): + with open(path, "w") as fh: + fh.write(json.dumps(json_content)) - def checkpoint(self, checkpoint_name, verbose, dry_run): - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + def __get_checkpoint_content(self, checkpoint_name): + path = self.__get_checkpoint_full_path(checkpoint_name) + with open(path) as fh: + text = fh.read() + return json.loads(text) - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError + def __get_checkpoint_full_path(self, name): + return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") - __acquire_config_lock() - current_config = _get_current_config() + def __get_checkpoint_names(self): + return [f[:-len(CHECKPOINT_EXT)] for f in os.listdir(self.checkpoints_dir) if f.endswith(CHECKPOINT_EXT)] - __save_checkpoint_content(current_config) + def __checkpoints_dir_exist(self): + return os.path.isdir(self.checkpoints_dir) - __release_lock() + def __check_checkpoint_exists(self, name): + path = self.__get_checkpoint_full_path(name) + return os.path.isfile(path) - def list_checkpoints(self, verbose, dry_run): - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + def __delete_checkpoint(self, name): + path = self.__get_checkpoint_full_path(name) + return os.remove(path) - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError +class GenericUpdateFactory: + def create_patch_applier(self, config_format, verbose, dry_run): + self.init_verbose_logging(verbose) - if not(__checkpoints_dir_exist()): - return [] + configwrapper = self.get_config_wrapper(dry_run) - return __get_checkpoint_names() + patch_applier = PatchApplier(configwrapper=configwrapper) - def delete_checkpoint(self, checkpoint_name, verbose, dry_run): - if verbose: - # TODO: Implement verbose logging - raise NotImplementedError + patchwrapper = PatchWrapper(configwrapper) - if dry_run: - # TODO: Implement dry-run - raise NotImplementedError + if config_format == ConfigFormat.CONFIGDB: + patch_applier = \ + ConfigDbDecorator( \ + decorated_patch_applier = patch_applier, patchwrapper=patchwrapper, configwrapper=configwrapper) + elif config_format == ConfigFormat.SONICYANG: + pass + else: + raise ValueError(f"config-format '{config_format}' is not supported") - if not(__check_checkpoint_exists()): - raise Exception("Checkpoint does not exist") + if not dry_run: + patch_applier = ConfigLockDecorator(decorated_patch_applier = patch_applier) - __delete_checkpoint(checkpoint_name) + return patch_applier - def __acquire_config_lock(): - self.configlocker.acquire_lock() + def create_config_replacer(self, config_format, verbose, dry_run): + self.init_verbose_logging(verbose) - def __release_config_lock(): - self.configlocker.release_lock() + configwrapper = self.get_config_wrapper(dry_run) - def __get_current_config(): - self.configwrapper.get_current_config() + patchapplier = PatchApplier(configwrapper=configwrapper) - def __ensure_checkpoints_dir_exists(self): - os.makedirs(CHECKPOINTS_DIR, exist_ok=True) + patchwrapper = PatchWrapper(configwrapper) - def __save_checkpoint_content(name, content): - __ensure_checkpoints_dir_exists() - path = __get_checkpoint_full_path(name) - with open(path, "w") as fh: - fh.write(text) + config_replacer = ConfigReplacer(patchapplier=patchapplier, configwrapper=configwrapper) + if config_format == ConfigFormat.CONFIGDB: + config_replacer = \ + ConfigDbDecorator( \ + decorated_config_replacer = config_replacer, patchwrapper=patchwrapper, configwrapper=configwrapper) + elif config_format == ConfigFormat.SONICYANG: + pass + else: + raise ValueError(f"config-format '{config_format}' is not supported") - def __get_checkpoint_content(name): - path = __get_checkpoint_full_path(name) - with open(path) as fh: - text = fh.read() - return json.loads(text) + if not dry_run: + config_replacer = ConfigLockDecorator(decorated_config_replacer = config_replacer) - def __get_checkpoint_full_path(name) - return os.path.join(CHECKPOINTS_DIR, name, CHECKPOINT_EXT) + return config_replacer - def __config_replace(target_config): - self.configreplacer.replace(target_config) + def create_config_rollbacker(self, verbose, dry_run): + self.init_verbose_logging(verbose) - def __get_checkpoint_names(): - return [f for f in listdir(CHECKPOINTS_DIR) if f.endswith(CHECKPOINT_EXT)] + configwrapper = self.get_config_wrapper(dry_run) - def __checkpoints_dir_exist(): - os.path.isdir(CHECKPOINTS_DIR) + patchapplier = PatchApplier(configwrapper=configwrapper) + config_replacer = ConfigReplacer(configwrapper=configwrapper, patchapplier=patchapplier) + config_rollbacker = FileSystemConfigRollbacker(configwrapper = configwrapper, config_replacer = config_replacer) - def __check_checkpoint_exists(name): - path = __get_checkpoint_full_path(name) - return os.path.isfile(path) + if not dry_run: + config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker = config_rollbacker) - def __delete_checkpoint(name): - path = __get_checkpoint_full_path(name) - return os.remove(path) + return config_rollbacker + + def init_verbose_logging(self, verbose): + # TODO: implement verbose logging + # Usually logs have levels such as: error, warning, info, debug. + # By default all log levels should show up to the user, except debug. + # By allowing verbose logging, debug msgs will also be shown to the user. + pass + + def get_config_wrapper(self, dry_run): + if dry_run: + return DryRunConfigWrapper() + else: + return ConfigWrapper() + +class GenericUpdater: + def __init__(self, generic_update_factory = GenericUpdateFactory()): + self.generic_update_factory = generic_update_factory + + def apply_patch(self, patch, config_format, verbose, dry_run): + patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run) + patch_applier.apply(patch) + + def replace(self, target_config, config_format, verbose, dry_run): + config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run) + config_replacer.replace(target_config) + + def rollback(self, checkpoint_name, verbose, dry_run): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + config_rollbacker.rollback(checkpoint_name) + + def checkpoint(self, checkpoint_name, verbose, dry_run): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + config_rollbacker.checkpoint(checkpoint_name) + + def delete_checkpoint(self, checkpoint_name, verbose, dry_run): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + config_rollbacker.delete_checkpoint(checkpoint_name) + + def list_checkpoints(self, verbose, dry_run): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + return config_rollbacker.list_checkpoints() + +class ConfigDbDecorator(PatchApplier, ConfigReplacer): + def __init__(self, patchwrapper, configwrapper, decorated_patch_applier=None, decorated_config_replacer=None): + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.patchwrapper = patchwrapper + self.configwrapper = configwrapper + + def apply(self, patch): + yang_patch = self.patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + self.decorated_patch_applier.apply(yang_patch) + + def replace(self, target_config): + yang_target_config = self.configwrapper.convert_config_db_to_sonic_yang(target_config) + self.decorated_config_replacer.replace(yang_target_config) + +class ConfigLockDecorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): + def __init__( \ + self, \ + decorated_patch_applier=None, \ + decorated_config_replacer=None, \ + decorated_config_rollbacker=None, \ + configlock = ConfigLock()): + self.configlock = configlock + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.decorated_config_rollbacker = decorated_config_rollbacker + + def apply(self, patch): + self.execute_write_action(self.decorated_patch_applier.apply, patch) + + def replace(self, target_config): + self.execute_write_action(self.decorated_config_replacer.replace, target_config) + + def rollback(self, checkpoint_name): + self.execute_write_action(self.decorated_config_rollbacker.rollback, checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.execute_write_action(self.decorated_config_rollbacker.checkpoint, checkpoint_name) + + def execute_write_action(self, action, *args): + self.configlock.acquire_lock() + action(*args) + self.configlock.release_lock() diff --git a/setup.py b/setup.py index 02a8d53e38..4b05f6570c 100644 --- a/setup.py +++ b/setup.py @@ -169,7 +169,8 @@ 'sonic-yang-mgmt', 'swsssdk>=2.0.1', 'tabulate==0.8.2', - 'xmltodict==0.12.0' + 'xmltodict==0.12.0', + 'jsonpatch==1.32.0', ], setup_requires= [ 'pytest-runner', diff --git a/tests/generic_update_test.py b/tests/generic_update_test.py index e2afc8f2e2..bfe6de48e0 100644 --- a/tests/generic_update_test.py +++ b/tests/generic_update_test.py @@ -1,10 +1,1744 @@ -import unittest +import json +import jsonpatch import os +import shutil +import unittest from imp import load_source +from unittest.mock import Mock, call load_source('generic_update', \ os.path.join(os.path.dirname(__file__), '..', 'config', 'generic_update.py')) import generic_update -# TODO: Add unit-tests -class Test(unittest.TestCase): - pass +class MockSideEffectDict: + def __init__(self, map): + self.map = map + + def side_effect_func(self, *args): + l = [str(arg) for arg in args] + key = tuple(l) + value = self.map.get(key) + if value == None: + raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") + + return value + +def create_side_effect_dict(map): + return MockSideEffectDict(map).side_effect_func + +class TestConfigWrapper(unittest.TestCase): + def test_ctor__default_values_set(self): + configwrapper = generic_update.ConfigWrapper() + + self.assertEqual(None, configwrapper.default_config_db_connector) + self.assertEqual("/usr/local/yang-models", generic_update.YANG_DIR) + + def test_get_config_db_as_json__returns_config_db_as_json(self): + # Arrange + config_db_connector_mock = self.__get_config_db_connector_mock(CONFIG_DB_AS_DICT) + configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + expected = CONFIG_DB_AS_JSON + + # Act + actual = configwrapper.get_config_db_as_json() + + # Assert + self.assertDictEqual(expected, actual) + + def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): + # Arrange + config_db_connector_mock = self.__get_config_db_connector_mock(CONFIG_DB_AS_DICT) + configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + expected = SONIC_YANG_AS_JSON + + # Act + actual = configwrapper.get_sonic_yang_as_json() + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__empty_config_db__returns_empty_sonic_yang(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = {} + + # Act + actual = configwrapper.convert_config_db_to_sonic_yang({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__non_empty_config_db__returns_sonic_yang_as_json(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = SONIC_YANG_AS_JSON + + # Act + actual = configwrapper.convert_config_db_to_sonic_yang(CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__empty_sonic_yang__returns_empty_config_db(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = {} + + # Act + actual = configwrapper.convert_sonic_yang_to_config_db({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_db_as_json(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = configwrapper.convert_sonic_yang_to_config_db(SONIC_YANG_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_validate_sonic_yang_config__valid_config__returns_true(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = True + + # Act + actual = configwrapper.validate_sonic_yang_config(SONIC_YANG_AS_JSON) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_sonic_yang_config__invvalid_config__returns_false(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = False + + # Act + actual = configwrapper.validate_sonic_yang_config(SONIC_YANG_AS_JSON_INVALID) + + # Assert + self.assertEqual(expected, actual) + + def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): + # Arrange + configwrapper = generic_update.ConfigWrapper() + expected = CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = configwrapper.crop_tables_without_yang(CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def __get_config_db_connector_mock(self, config_db_as_dict): + mock_connector = Mock() + mock_connector.get_config.return_value = config_db_as_dict + return mock_connector + +class TestPatchWrapper(unittest.TestCase): + def test_validate_config_db_patch__table_without_yang_model__returns_false(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + expected = False + + # Act + actual = patchwrapper.validate_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_config_db_patch__table_with_yang_model__returns_true(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] + expected = True + + # Act + actual = patchwrapper.validate_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__failure(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + + # Act and Assert + self.assertRaises(Exception, patchwrapper.convert_config_db_patch_to_sonic_yang_patch, patch) + + def test_same_patch__no_diff__returns_true(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + + # Act and Assert + self.assertTrue(patchwrapper.verify_same_json(CONFIG_DB_AS_JSON, CONFIG_DB_AS_JSON)) + + def test_same_patch__diff__returns_false(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + + # Act and Assert + self.assertFalse(patchwrapper.verify_same_json(CONFIG_DB_AS_JSON, CROPPED_CONFIG_DB_AS_JSON)) + + def test_generate_patch__no_diff__empty_patch(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + + # Act + patch = patchwrapper.generate_patch(CONFIG_DB_AS_JSON, CONFIG_DB_AS_JSON) + + # Assert + self.assertFalse(patch) + + def test_simulate_patch__empty_patch__no_changes(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + patch = jsonpatch.JsonPatch([]) + expected = CONFIG_DB_AS_JSON + + # Act + actual = patchwrapper.simulate_patch(patch, CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_simulate_patch__non_empty_patch__changes_applied(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + patch = SINGLE_OPERATION_CONFIG_DB_PATCH + expected = SINGLE_OPERATION_CONFIG_DB_PATCH.apply(CONFIG_DB_AS_JSON) + + # Act + actual = patchwrapper.simulate_patch(patch, CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_generate_patch__diff__non_empty_patch(self): + # Arrange + patchwrapper = generic_update.PatchWrapper() + after_update_json = SINGLE_OPERATION_CONFIG_DB_PATCH.apply(CONFIG_DB_AS_JSON) + expected = SINGLE_OPERATION_CONFIG_DB_PATCH + + # Act + actual = patchwrapper.generate_patch(CONFIG_DB_AS_JSON, after_update_json) + + # Assert + self.assertTrue(actual) + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): + # Arrange + configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) + patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) + patch = jsonpatch.JsonPatch([]) + expected = jsonpatch.JsonPatch([]) + + # Act + actual = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): + # Arrange + configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) + patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) + patch = SINGLE_OPERATION_CONFIG_DB_PATCH + expected = SINGLE_OPERATION_SONIC_YANG_PATCH + + # Act + actual = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): + # Arrange + configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) + patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) + config_db_patch = MULTI_OPERATION_CONFIG_DB_PATCH + + # Act + sonic_yang_patch = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(config_db_patch) + + # Assert + self.__assert_same_patch(config_db_patch, sonic_yang_patch, configwrapper, patchwrapper) + + def __assert_same_patch(self, config_db_patch, sonic_yang_patch, configwrapper, patchwrapper): + sonic_yang = configwrapper.get_sonic_yang_as_json() + config_db = configwrapper.get_config_db_as_json() + + after_update_sonic_yang = patchwrapper.simulate_patch(sonic_yang_patch, sonic_yang) + after_update_config_db = patchwrapper.simulate_patch(config_db_patch, config_db) + + after_update_config_db_as_sonic_yang = \ + configwrapper.convert_config_db_to_sonic_yang(after_update_config_db) + + self.assertTrue(patchwrapper.verify_same_json(after_update_sonic_yang, after_update_config_db_as_sonic_yang)) + + def __get_configwrapper_mock(self, config_db_as_dict): + config_db_connector_mock = self.__get_config_db_connector_mock(config_db_as_dict) + configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + return configwrapper + + def __get_config_db_connector_mock(self, config_db_as_dict): + mock_connector = Mock() + mock_connector.get_config.return_value = config_db_as_dict + return mock_connector + +class TestPatchApplier(unittest.TestCase): + def test_apply__invalid_sonic_yang__failure(self): + # Arrange + patchapplier = self.__create_patch_applier(valid_sonic_yang=False) + + # Act and assert + self.assertRaises(Exception, patchapplier.apply, MULTI_OPERATION_SONIC_YANG_PATCH) + + def test_apply__json_not_fully_updated__failure(self): + # Arrange + patchapplier = self.__create_patch_applier(verified_same_config=False) + + # Act and assert + self.assertRaises(Exception, patchapplier.apply, MULTI_OPERATION_SONIC_YANG_PATCH) + + def test_apply__no_errors__update_successful(self): + # Arrange + changes = [Mock(), Mock()] + patchapplier = self.__create_patch_applier(changes) + + # Act + patchapplier.apply(MULTI_OPERATION_SONIC_YANG_PATCH) + + # Assert + patchapplier.configwrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + patchapplier.patchwrapper.simulate_patch.assert_has_calls( \ + [call(MULTI_OPERATION_SONIC_YANG_PATCH, SONIC_YANG_AS_JSON)]) + patchapplier.configwrapper.validate_sonic_yang_config.assert_has_calls([call(SONIC_YANG_AFTER_MULTI_PATCH)]) + patchapplier.patchorderer.order.assert_has_calls([call(MULTI_OPERATION_SONIC_YANG_PATCH)]) + patchapplier.changeapplier.apply.assert_has_calls([call(changes[0]), call(changes[1])]) + patchapplier.patchwrapper.verify_same_json.assert_has_calls( \ + [call(SONIC_YANG_AFTER_MULTI_PATCH, SONIC_YANG_AFTER_MULTI_PATCH)]) + + def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + configwrapper = Mock() + configwrapper.get_sonic_yang_as_json.side_effect = [SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH] + configwrapper.validate_sonic_yang_config.side_effect = \ + create_side_effect_dict({(str(SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) + + patchwrapper = Mock() + patchwrapper.simulate_patch.side_effect = \ + create_side_effect_dict( \ + {(str(MULTI_OPERATION_SONIC_YANG_PATCH), str(SONIC_YANG_AS_JSON)): SONIC_YANG_AFTER_MULTI_PATCH}) + patchwrapper.verify_same_json.side_effect = \ + create_side_effect_dict( \ + {(str(SONIC_YANG_AFTER_MULTI_PATCH), str(SONIC_YANG_AFTER_MULTI_PATCH)): verified_same_config}) + + changes = [Mock(), Mock()] if not changes else changes + patchorderer = Mock() + patchorderer.order.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) + + changeapplier = Mock() + changeapplier.apply.side_effect = create_side_effect_dict({(str(changes[0]),): 0, (str(changes[1]),): 0}) + + return generic_update.PatchApplier(patchorderer, changeapplier, configwrapper, patchwrapper) + +class TestConfigReplacer(unittest.TestCase): + def test_replace__invalid_sonic_yang__failure(self): + # Arrange + configreplacer = self.__create_config_replacer(valid_sonic_yang=False) + + # Act and assert + self.assertRaises(Exception, configreplacer.replace, SONIC_YANG_AFTER_MULTI_PATCH) + + def test_replace__json_not_fully_updated__failure(self): + # Arrange + configreplacer = self.__create_config_replacer(verified_same_config=False) + + # Act and assert + self.assertRaises(Exception, configreplacer.replace, SONIC_YANG_AFTER_MULTI_PATCH) + + def test_replace__no_errors__update_successful(self): + # Arrange + configreplacer = self.__create_config_replacer() + + # Act + configreplacer.replace(SONIC_YANG_AFTER_MULTI_PATCH) + + # Assert + configreplacer.configwrapper.validate_sonic_yang_config.assert_has_calls([call(SONIC_YANG_AFTER_MULTI_PATCH)]) + configreplacer.configwrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + configreplacer.patchwrapper.generate_patch.assert_has_calls( \ + [call(SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH)]) + configreplacer.patchapplier.apply.assert_has_calls([call(MULTI_OPERATION_SONIC_YANG_PATCH)]) + configreplacer.patchwrapper.verify_same_json.assert_has_calls( \ + [call(SONIC_YANG_AFTER_MULTI_PATCH, SONIC_YANG_AFTER_MULTI_PATCH)]) + + def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + configwrapper = Mock() + configwrapper.validate_sonic_yang_config.side_effect = \ + create_side_effect_dict({(str(SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) + configwrapper.get_sonic_yang_as_json.side_effect = [SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH] + + patchwrapper = Mock() + patchwrapper.generate_patch.side_effect = \ + create_side_effect_dict( \ + {(str(SONIC_YANG_AS_JSON), str(SONIC_YANG_AFTER_MULTI_PATCH)): MULTI_OPERATION_SONIC_YANG_PATCH}) + patchwrapper.verify_same_json.side_effect = \ + create_side_effect_dict( \ + {(str(SONIC_YANG_AFTER_MULTI_PATCH), str(SONIC_YANG_AFTER_MULTI_PATCH)): verified_same_config}) + + changes = [Mock(), Mock()] if not changes else changes + patchorderer = Mock() + patchorderer.order.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) + + patchapplier = Mock() + patchapplier.apply.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): 0}) + + return generic_update.ConfigReplacer(patchapplier, configwrapper, patchwrapper) + +class TestFileSystemConfigRollbacker(unittest.TestCase): + def setUp(self): + self.checkpoints_dir = os.path.join(os.getcwd(),"checkpoints") + self.checkpoint_ext = ".cp.json" + self.any_checkpoint_name = "anycheckpoint" + self.any_other_checkpoint_name = "anyothercheckpoint" + self.any_config = {} + self.clean_up() + + def tearDown(self): + self.clean_up() + + def test_rollback__checkpoint_does_not_exist__failure(self): + # Arrange + rollbacker = self.create_rollbacker() + + # Act and assert + self.assertRaises(Exception, rollbacker.rollback, "NonExistingCheckpoint") + + def test_rollback__no_errors__success(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + + # Act + rollbacker.rollback(self.any_checkpoint_name) + + # Assert + rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) + + def test_checkpoint__checkpoints_dir_does_not_exist__checkpoint_created(self): + # Arrange + rollbacker = self.create_rollbacker() + self.assertFalse(os.path.isdir(self.checkpoints_dir)) + + # Act + rollbacker.checkpoint(self.any_checkpoint_name) + + # Assert + self.assertTrue(os.path.isdir(self.checkpoints_dir)) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + def test_checkpoint__checkpoints_dir_exists__checkpoint_created(self): + # Arrange + self.create_checkpoints_dir() + rollbacker = self.create_rollbacker() + + # Act + rollbacker.checkpoint(self.any_checkpoint_name) + + # Assert + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + def test_list_checkpoints__checkpoints_dir_does_not_exist__empty_list(self): + # Arrange + rollbacker = self.create_rollbacker() + self.assertFalse(os.path.isdir(self.checkpoints_dir)) + expected = [] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + + def test_list_checkpoints__checkpoints_dir_exist_but_no_files__empty_list(self): + # Arrange + self.create_checkpoints_dir() + rollbacker = self.create_rollbacker() + expected = [] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_list_checkpoints__checkpoints_dir_has_multiple_files__multiple_files(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + self.add_checkpoint(self.any_other_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + expected = [self.any_checkpoint_name, self.any_other_checkpoint_name] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_list_checkpoints__checkpoints_names_have_special_characters__multiple_files(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint("check.point1", self.any_config) + self.add_checkpoint(".checkpoint2", self.any_config) + self.add_checkpoint("checkpoint3.", self.any_config) + rollbacker = self.create_rollbacker() + expected = ["check.point1", ".checkpoint2", "checkpoint3."] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_delete_checkpoint__checkpoint_does_not_exist__failure(self): + # Arrange + rollbacker = self.create_rollbacker() + + # Act and assert + self.assertRaises(Exception, rollbacker.delete_checkpoint, self.any_checkpoint_name) + + def test_delete_checkpoint__checkpoint_exist__success(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + + # Act + rollbacker.delete_checkpoint(self.any_checkpoint_name) + + # Assert + self.assertFalse(self.check_checkpoint_exists(self.any_checkpoint_name)) + + def test_multiple_operations(self): + rollbacker = self.create_rollbacker() + + self.assertListEqual([], rollbacker.list_checkpoints()) + + rollbacker.checkpoint(self.any_checkpoint_name) + self.assertListEqual([self.any_checkpoint_name], rollbacker.list_checkpoints()) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + rollbacker.rollback(self.any_checkpoint_name) + rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) + + rollbacker.checkpoint(self.any_other_checkpoint_name) + self.assertListEqual([self.any_checkpoint_name, self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_other_checkpoint_name)) + + rollbacker.delete_checkpoint(self.any_checkpoint_name) + self.assertListEqual([self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + + rollbacker.delete_checkpoint(self.any_other_checkpoint_name) + self.assertListEqual([], rollbacker.list_checkpoints()) + + def clean_up(self): + if os.path.isdir(self.checkpoints_dir): + shutil.rmtree(self.checkpoints_dir) + + def create_checkpoints_dir(self): + os.makedirs(self.checkpoints_dir) + + def add_checkpoint(self, name, json_content): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + with open(path, "w") as fh: + fh.write(json.dumps(json_content)) + + def get_checkpoint(self, name): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + with open(path) as fh: + text = fh.read() + return json.loads(text) + + def check_checkpoint_exists(self, name): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + return os.path.isfile(path) + + def create_rollbacker(self): + replacer = Mock() + replacer.replace.side_effect = create_side_effect_dict({(str(self.any_config),): 0}) + + configwrapper = Mock() + configwrapper.get_sonic_yang_as_json.return_value = self.any_config + + return generic_update.FileSystemConfigRollbacker( \ + checkpoints_dir=self.checkpoints_dir, \ + config_replacer=replacer, \ + configwrapper=configwrapper) + +class TestGenericUpdateFactory(unittest.TestCase): + def setUp(self): + self.any_verbose=True + self.any_dry_run=True + + def test_create_patch_applier__invalid_config_format__failure(self): + # Arrange + factory = generic_update.GenericUpdateFactory() + + # Act and assert + self.assertRaises( \ + ValueError, factory.create_patch_applier, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) + + def test_create_patch_applier__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}}, + { + "config_format": { + generic_update.ConfigFormat.SONICYANG: None, + generic_update.ConfigFormat.CONFIGDB: generic_update.ConfigDbDecorator + } + }, + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_patch_applier) + + def test_create_config_replacer__invalid_config_format__failure(self): + # Arrange + factory = generic_update.GenericUpdateFactory() + + # Act and assert + self.assertRaises( \ + ValueError, factory.create_config_replacer, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) + + def test_create_config_replacer__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}}, + { + "config_format": { + generic_update.ConfigFormat.SONICYANG: None, + generic_update.ConfigFormat.CONFIGDB: generic_update.ConfigDbDecorator + } + }, + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_replacer) + + def test_create_config_rollbacker__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}} + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_rollbacker) + + def recursively_test_create_func(self, options, cur_option, params, expected_decorators, create_func): + if cur_option == len(options): + create_func(params, expected_decorators) + return + + param = list(options[cur_option].keys())[0] + for key in options[cur_option][param]: + params[param] = key + decorator = options[cur_option][param][key] + if decorator != None: + expected_decorators.append(decorator) + self.recursively_test_create_func(options, cur_option+1, params, expected_decorators, create_func) + if decorator != None: + expected_decorators.pop() + + def validate_create_patch_applier(self, params, expected_decorators): + factory = generic_update.GenericUpdateFactory() + patch_applier = factory.create_patch_applier(params["config_format"], params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(patch_applier, decorator_type) + + patch_applier = patch_applier.decorated_patch_applier + + self.assertIsInstance(patch_applier, generic_update.PatchApplier) + if params["dry_run"]: + self.assertIsInstance(patch_applier.configwrapper, generic_update.DryRunConfigWrapper) + else: + self.assertIsInstance(patch_applier.configwrapper, generic_update.ConfigWrapper) + + def validate_create_config_replacer(self, params, expected_decorators): + factory = generic_update.GenericUpdateFactory() + config_replacer = factory.create_config_replacer(params["config_format"], params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(config_replacer, decorator_type) + + config_replacer = config_replacer.decorated_config_replacer + + self.assertIsInstance(config_replacer, generic_update.ConfigReplacer) + if params["dry_run"]: + self.assertIsInstance(config_replacer.configwrapper, generic_update.DryRunConfigWrapper) + self.assertIsInstance(config_replacer.patchapplier.configwrapper, generic_update.DryRunConfigWrapper) + else: + self.assertIsInstance(config_replacer.configwrapper, generic_update.ConfigWrapper) + self.assertIsInstance(config_replacer.patchapplier.configwrapper, generic_update.ConfigWrapper) + + def validate_create_config_rollbacker(self, params, expected_decorators): + factory = generic_update.GenericUpdateFactory() + config_rollbacker = factory.create_config_rollbacker(params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(config_rollbacker, decorator_type) + + config_rollbacker = config_rollbacker.decorated_config_rollbacker + + self.assertIsInstance(config_rollbacker, generic_update.FileSystemConfigRollbacker) + if params["dry_run"]: + self.assertIsInstance(config_rollbacker.configwrapper, generic_update.DryRunConfigWrapper) + self.assertIsInstance(config_rollbacker.config_replacer.configwrapper, generic_update.DryRunConfigWrapper) + self.assertIsInstance( \ + config_rollbacker.config_replacer.patchapplier.configwrapper, generic_update.DryRunConfigWrapper) + else: + self.assertIsInstance(config_rollbacker.configwrapper, generic_update.ConfigWrapper) + self.assertIsInstance(config_rollbacker.config_replacer.configwrapper, generic_update.ConfigWrapper) + self.assertIsInstance( \ + config_rollbacker.config_replacer.patchapplier.configwrapper, generic_update.ConfigWrapper) + +class TestGenericUpdater(unittest.TestCase): + def setUp(self): + self.any_checkpoint_name = "anycheckpoint" + self.any_other_checkpoint_name = "anyothercheckpoint" + self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] + self.any_config_format = generic_update.ConfigFormat.SONICYANG + self.any_verbose = True + self.any_dry_run = True + + def test_apply_patch__creates_applier_and_apply(self): + # Arrange + patch_applier = Mock() + patch_applier.apply.side_effect = create_side_effect_dict({(str(SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) + + factory = Mock() + factory.create_patch_applier.side_effect = \ + create_side_effect_dict( \ + {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): patch_applier}) + + generic_updater = generic_update.GenericUpdater(factory) + + # Act + generic_updater.apply_patch( \ + SINGLE_OPERATION_SONIC_YANG_PATCH, self.any_config_format, self.any_verbose, self.any_dry_run) + + # Assert + patch_applier.apply.assert_has_calls([call(SINGLE_OPERATION_SONIC_YANG_PATCH)]) + + def test_replace__creates_replacer_and_replace(self): + # Arrange + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) + + factory = Mock() + factory.create_config_replacer.side_effect = \ + create_side_effect_dict( \ + {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): config_replacer}) + + generic_updater = generic_update.GenericUpdater(factory) + + # Act + generic_updater.replace(SONIC_YANG_AS_JSON, self.any_config_format, self.any_verbose, self.any_dry_run) + + # Assert + config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) + + def test_rollback__creates_rollbacker_and_rollback(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = generic_update.GenericUpdater(factory) + + # Act + generic_updater.rollback(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_checkpoint__creates_rollbacker_and_checkpoint(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = generic_update.GenericUpdater(factory) + + # Act + generic_updater.checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_delete_checkpoint__creates_rollbacker_and_deletes_checkpoint(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.delete_checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = generic_update.GenericUpdater(factory) + + # Act + generic_updater.delete_checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.delete_checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = generic_update.GenericUpdater(factory) + + expected = self.any_checkpoints_list + + # Act + actual = generic_updater.list_checkpoints(self.any_verbose, self.any_dry_run) + + # Assert + self.assertListEqual(expected, actual) + +class TestConfigDbDecorator(unittest.TestCase): + def test_apply__converts_to_yang_and_calls_decorated_class(self): + # Arrange + config_db_decorator = self.__create_config_db_decorator() + + # Act + config_db_decorator.apply(CONFIG_DB_AS_JSON) + + # Assert + config_db_decorator.patchwrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( \ + [call(CONFIG_DB_AS_JSON)]) + config_db_decorator.decorated_patch_applier.apply.assert_has_calls([call(SONIC_YANG_AS_JSON)]) + + def test_replace__converts_to_yang_and_calls_decorated_class(self): + # Arrange + config_db_decorator = self.__create_config_db_decorator() + + # Act + config_db_decorator.replace(CONFIG_DB_AS_JSON) + + # Assert + config_db_decorator.configwrapper.convert_config_db_to_sonic_yang.assert_has_calls([call(CONFIG_DB_AS_JSON)]) + config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) + + def __create_config_db_decorator(self): + patchapplier = Mock() + patchapplier.apply.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) + + patchwrapper = Mock() + patchwrapper.convert_config_db_patch_to_sonic_yang_patch.side_effect = \ + create_side_effect_dict({(str(CONFIG_DB_AS_JSON),): SONIC_YANG_AS_JSON}) + + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) + + configwrapper = Mock() + configwrapper.convert_config_db_to_sonic_yang.side_effect = \ + create_side_effect_dict({(str(CONFIG_DB_AS_JSON),): SONIC_YANG_AS_JSON}) + + return generic_update.ConfigDbDecorator( \ + decorated_patch_applier=patchapplier, \ + decorated_config_replacer=config_replacer, \ + patchwrapper=patchwrapper, \ + configwrapper=configwrapper) + +class TestConfigLockDecorator(unittest.TestCase): + def setUp(self): + self.any_checkpoint_name = "anycheckpoint" + + def test_apply__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.apply(SINGLE_OPERATION_SONIC_YANG_PATCH) + + # Assert + config_lock_decorator.configlock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_patch_applier.apply.assert_has_calls([call(SINGLE_OPERATION_SONIC_YANG_PATCH)]) + config_lock_decorator.configlock.release_lock.assert_called_once() + + def test_replace__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.replace(SONIC_YANG_AS_JSON) + + # Assert + config_lock_decorator.configlock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) + config_lock_decorator.configlock.release_lock.assert_called_once() + + def test_rollback__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.rollback(self.any_checkpoint_name) + + # Assert + config_lock_decorator.configlock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) + config_lock_decorator.configlock.release_lock.assert_called_once() + + def test_checkpoint__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.checkpoint(self.any_checkpoint_name) + + # Assert + config_lock_decorator.configlock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + config_lock_decorator.configlock.release_lock.assert_called_once() + + def __create_config_lock_decorator(self): + configlock = Mock() + + patchapplier = Mock() + patchapplier.apply.side_effect = create_side_effect_dict({(str(SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) + + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) + + config_rollbacker = Mock() + config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + return generic_update.ConfigLockDecorator( \ + configlock=configlock, \ + decorated_patch_applier=patchapplier, \ + decorated_config_replacer=config_replacer, \ + decorated_config_rollbacker=config_rollbacker) + +#### resources #### +CONFIG_DB_AS_DICT = { + "VLAN_MEMBER": { + ("Vlan1000", "Ethernet0"): { + "tagging_mode": "untagged" + }, + ("Vlan1000", "Ethernet4"): { + "tagging_mode": "untagged" + }, + ("Vlan1000", "Ethernet8"): { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + }, + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "wred_red_enable": "true", + "ecn": "ecn_all", + "green_max_threshold": "2097152", + "green_min_threshold": "1048576", + "yellow_max_threshold": "2097152", + "yellow_min_threshold": "1048576", + "red_max_threshold": "2097152", + "red_min_threshold": "1048576", + "green_drop_probability": "5", + "yellow_drop_probability": "5", + "red_drop_probability": "5" + } + } +} + +CONFIG_DB_AS_JSON = { + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "wred_red_enable": "true", + "ecn": "ecn_all", + "green_max_threshold": "2097152", + "green_min_threshold": "1048576", + "yellow_max_threshold": "2097152", + "yellow_min_threshold": "1048576", + "red_max_threshold": "2097152", + "red_min_threshold": "1048576", + "green_drop_probability": "5", + "yellow_drop_probability": "5", + "red_drop_probability": "5" + } + } +} + +CROPPED_CONFIG_DB_AS_JSON = { + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + } +} + +SONIC_YANG_AS_JSON = { + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + } + ] + }, + "sonic-vlan:VLAN": { + "VLAN_LIST": [ + { + "vlan_name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-acl:ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "sonic-port:PORT": { + "PORT_LIST": [ + { + "port_name": "Ethernet0", + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": 100000 + }, + { + "port_name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + } + ] + } + } +} + +SONIC_YANG_AS_JSON_INVALID = { + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + } + ] + } + } +} + +SINGLE_OPERATION_CONFIG_DB_PATCH = jsonpatch.JsonPatch([ + { + "op": "remove", + "path": "/VLAN_MEMBER/Vlan1000|Ethernet8" + } +]) + +SINGLE_OPERATION_SONIC_YANG_PATCH = jsonpatch.JsonPatch([ + { + "op": "remove", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/2" + } +]) + +MULTI_OPERATION_CONFIG_DB_PATCH = jsonpatch.JsonPatch([ + { + "op": "add", + "path": "/PORT/Ethernet3", + "value": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "10000" + } + }, + { + "op": "add", + "path": "/PORT/Ethernet1", + "value": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "10000" + } + }, + { + "op": "add", + "path": "/PORT/Ethernet2", + "value": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "10000" + } + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/lanes", + "value": "65" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/alias", + "value": "Eth1/1" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/description", + "value": "" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/speed", + "value": "10000" + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet2", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet3", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet1", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/1", + "value": "Ethernet1" + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/2", + "value": "Ethernet2" + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/3", + "value": "Ethernet3" + } +]) + +MULTI_OPERATION_SONIC_YANG_PATCH = jsonpatch.JsonPatch([ + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/3", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet2", + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/4", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet3", + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/5", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet1", + "tagging_mode": "untagged" + } + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/lanes", + "value": "65" + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/alias", + "value": "Eth1/1" + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/speed", + "value": 10000 + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/description", + "value": "" + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/3", + "value": { + "port_name": "Ethernet3", + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/4", + "value": { + "port_name": "Ethernet1", + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/5", + "value": { + "port_name": "Ethernet2", + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/1", + "value": "Ethernet1" + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/2", + "value": "Ethernet2" + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/3", + "value": "Ethernet3" + } +]) + +SONIC_YANG_AFTER_MULTI_PATCH = { + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet2", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet3", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet1", + "tagging_mode": "untagged" + } + ] + }, + "sonic-vlan:VLAN": { + "VLAN_LIST": [ + { + "vlan_name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-acl:ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0", + "Ethernet1", + "Ethernet2", + "Ethernet3" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "sonic-port:PORT": { + "PORT_LIST": [ + { + "port_name": "Ethernet0", + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet3", + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet1", + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet2", + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": 10000 + } + ] + } + } +} From f2510067a233b79405103317a3a36caba144ac37 Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 6 Apr 2021 01:18:33 -0700 Subject: [PATCH 04/32] restructure files --- generic_config_updater/__init__.py | 0 .../generic_updater.py | 183 +- tests/generic_config_updater/__init__.py | 0 .../files/config_db_as_dict.py-dict | 121 ++ .../files/config_db_as_json.json | 103 + .../files/cropped_config_db_as_json.json | 86 + ...multi_operation_config_db_patch.json-patch | 88 + ...ulti_operation_sonic_yang_patch.json-patch | 97 + ...ingle_operation_config_db_patch.json-patch | 6 + ...ngle_operation_sonic_yang_patch.json-patch | 6 + .../files/sonic_yang_after_multi_patch.json | 153 ++ .../files/sonic_yang_as_json.json | 114 ++ .../files/sonic_yang_as_json_invalid.json | 13 + .../generic_updater_test.py | 991 ++++++++++ tests/generic_update_test.py | 1744 ----------------- 15 files changed, 1870 insertions(+), 1835 deletions(-) create mode 100644 generic_config_updater/__init__.py rename config/generic_update.py => generic_config_updater/generic_updater.py (77%) create mode 100644 tests/generic_config_updater/__init__.py create mode 100644 tests/generic_config_updater/files/config_db_as_dict.py-dict create mode 100644 tests/generic_config_updater/files/config_db_as_json.json create mode 100644 tests/generic_config_updater/files/cropped_config_db_as_json.json create mode 100644 tests/generic_config_updater/files/multi_operation_config_db_patch.json-patch create mode 100644 tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch create mode 100644 tests/generic_config_updater/files/single_operation_config_db_patch.json-patch create mode 100644 tests/generic_config_updater/files/single_operation_sonic_yang_patch.json-patch create mode 100644 tests/generic_config_updater/files/sonic_yang_after_multi_patch.json create mode 100644 tests/generic_config_updater/files/sonic_yang_as_json.json create mode 100644 tests/generic_config_updater/files/sonic_yang_as_json_invalid.json create mode 100644 tests/generic_config_updater/generic_updater_test.py delete mode 100644 tests/generic_update_test.py diff --git a/generic_config_updater/__init__.py b/generic_config_updater/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/generic_update.py b/generic_config_updater/generic_updater.py similarity index 77% rename from config/generic_update.py rename to generic_config_updater/generic_updater.py index cb24bf41fd..ab641de6b8 100644 --- a/config/generic_update.py +++ b/generic_config_updater/generic_updater.py @@ -39,9 +39,9 @@ def __init__(self, default_config_db_connector = None, yang_dir = YANG_DIR): self.yang_dir = YANG_DIR def get_config_db_as_json(self): - configdb = self.__create_and_connect_config_db() + config_db = self.__create_and_connect_config_db() data = dict() - deep_update(data, FormatConverter.db_to_output(configdb.get_config())) + deep_update(data, FormatConverter.db_to_output(config_db.get_config())) return FormatConverter.to_serialized(data) def get_sonic_yang_as_json(self): @@ -52,6 +52,7 @@ def convert_config_db_to_sonic_yang(self, config_db_as_json): sy = sonic_yang.SonicYang(self.yang_dir) sy.loadYangModel() + # Crop config_db tables that do not have sonic yang models cropped_config_db_as_json = self.crop_tables_without_yang(config_db_as_json) sonic_yang_as_json = dict() @@ -111,9 +112,9 @@ def __create_and_connect_config_db(self): db_kwargs = dict() data = dict() - configdb = ConfigDBConnector(**db_kwargs) - configdb.connect() - return configdb + config_db = ConfigDBConnector(**db_kwargs) + config_db.connect() + return config_db class DryRunConfigWrapper(ConfigWrapper): # TODO: implement DryRunConfigWrapper @@ -121,8 +122,8 @@ class DryRunConfigWrapper(ConfigWrapper): pass class PatchWrapper(): - def __init__(self, configwrapper = ConfigWrapper()): - self.configwrapper = configwrapper + def __init__(self, config_wrapper = ConfigWrapper()): + self.config_wrapper = config_wrapper def validate_config_db_patch(self, patch): config_db = {} @@ -136,7 +137,7 @@ def validate_config_db_patch(self, patch): else: config_db[tokens[0]] = {} - cropped_config_db = self.configwrapper.crop_tables_without_yang(config_db) + cropped_config_db = self.config_wrapper.crop_tables_without_yang(config_db) # valid if no tables dropped during cropping return len(cropped_config_db.keys()) == len(config_db.keys()) @@ -155,11 +156,11 @@ def convert_config_db_patch_to_sonic_yang_patch(self, patch): if not(self.validate_config_db_patch(patch)): raise Exception(f"Given patch is not valid") - current_config_db = self.configwrapper.get_config_db_as_json() + current_config_db = self.config_wrapper.get_config_db_as_json() target_config_db = self.simulate_patch(patch, current_config_db) - current_yang = self.configwrapper.convert_config_db_to_sonic_yang(current_config_db) - target_yang = self.configwrapper.convert_config_db_to_sonic_yang(target_config_db) + current_yang = self.config_wrapper.convert_config_db_to_sonic_yang(current_config_db) + target_yang = self.config_wrapper.convert_config_db_to_sonic_yang(target_config_db) return self.generate_patch(current_yang, target_yang) @@ -172,22 +173,22 @@ def __init__( \ self, \ patchorderer = PatchOrderer(), \ changeapplier = ChangeApplier(), \ - configwrapper = ConfigWrapper(), \ - patchwrapper = PatchWrapper()): + config_wrapper = ConfigWrapper(), \ + patch_wrapper = PatchWrapper()): self.patchorderer = patchorderer self.changeapplier = changeapplier - self.configwrapper = configwrapper - self.patchwrapper = patchwrapper + self.config_wrapper = config_wrapper + self.patch_wrapper = patch_wrapper def apply(self, patch): # 1. Get old config as SONiC Yang - old_config = self.configwrapper.get_sonic_yang_as_json() + old_config = self.config_wrapper.get_sonic_yang_as_json() # 2. Generate target config - target_config = self.patchwrapper.simulate_patch(patch, old_config) + target_config = self.patch_wrapper.simulate_patch(patch, old_config) # 3. Validate target config - if not(self.configwrapper.validate_sonic_yang_config(target_config)): + if not(self.config_wrapper.validate_sonic_yang_config(target_config)): raise Exception(f"The given patch is not valid") # 4. Generate list of changes to apply @@ -198,27 +199,27 @@ def apply(self, patch): self.changeapplier.apply(change) # 6. Validate config updated successfully - new_config = self.configwrapper.get_sonic_yang_as_json() - if not(self.patchwrapper.verify_same_json(target_config, new_config)): + new_config = self.config_wrapper.get_sonic_yang_as_json() + if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise Exception(f"After applying patch to config, there are still some parts not updated") class ConfigReplacer: - def __init__(self, patchapplier = PatchApplier(), configwrapper = ConfigWrapper(), patchwrapper = PatchWrapper()): - self.patchapplier = patchapplier - self.configwrapper = configwrapper - self.patchwrapper = patchwrapper + def __init__(self, patch_applier = PatchApplier(), config_wrapper = ConfigWrapper(), patch_wrapper = PatchWrapper()): + self.patch_applier = patch_applier + self.config_wrapper = config_wrapper + self.patch_wrapper = patch_wrapper def replace(self, target_config): - if not(self.configwrapper.validate_sonic_yang_config(target_config)): + if not(self.config_wrapper.validate_sonic_yang_config(target_config)): raise Exception(f"The given target config is not valid") - old_config = self.configwrapper.get_sonic_yang_as_json() - patch = self.patchwrapper.generate_patch(old_config, target_config) + old_config = self.config_wrapper.get_sonic_yang_as_json() + patch = self.patch_wrapper.generate_patch(old_config, target_config) - self.patchapplier.apply(patch) + self.patch_applier.apply(patch) - new_config = self.configwrapper.get_sonic_yang_as_json() - if not(self.patchwrapper.verify_same_json(target_config, new_config)): + new_config = self.config_wrapper.get_sonic_yang_as_json() + if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise Exception(f"After applying patch to config, there is still some parts not updated") CHECKPOINTS_DIR = "/etc/sonic/checkpoints" @@ -228,10 +229,10 @@ def __init__( \ self, \ checkpoints_dir = CHECKPOINTS_DIR, \ config_replacer = ConfigReplacer(), \ - configwrapper = ConfigWrapper()): + config_wrapper = ConfigWrapper()): self.checkpoints_dir = checkpoints_dir self.config_replacer = config_replacer - self.configwrapper = configwrapper + self.config_wrapper = config_wrapper def rollback(self, checkpoint_name): if not self.__check_checkpoint_exists(checkpoint_name): @@ -242,7 +243,7 @@ def rollback(self, checkpoint_name): self.config_replacer.replace(target_config) def checkpoint(self, checkpoint_name): - json_content = self.configwrapper.get_sonic_yang_as_json() + json_content = self.config_wrapper.get_sonic_yang_as_json() path = self.__get_checkpoint_full_path(checkpoint_name) @@ -292,20 +293,64 @@ def __delete_checkpoint(self, name): path = self.__get_checkpoint_full_path(name) return os.remove(path) +class ConfigDbDecorator(PatchApplier, ConfigReplacer): + def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None): + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.patch_wrapper = patch_wrapper + self.config_wrapper = config_wrapper + + def apply(self, patch): + yang_patch = self.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + self.decorated_patch_applier.apply(yang_patch) + + def replace(self, target_config): + yang_target_config = self.config_wrapper.convert_config_db_to_sonic_yang(target_config) + self.decorated_config_replacer.replace(yang_target_config) + +class ConfigLockDecorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): + def __init__( \ + self, \ + decorated_patch_applier=None, \ + decorated_config_replacer=None, \ + decorated_config_rollbacker=None, \ + config_lock = ConfigLock()): + self.config_lock = config_lock + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.decorated_config_rollbacker = decorated_config_rollbacker + + def apply(self, patch): + self.execute_write_action(self.decorated_patch_applier.apply, patch) + + def replace(self, target_config): + self.execute_write_action(self.decorated_config_replacer.replace, target_config) + + def rollback(self, checkpoint_name): + self.execute_write_action(self.decorated_config_rollbacker.rollback, checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.execute_write_action(self.decorated_config_rollbacker.checkpoint, checkpoint_name) + + def execute_write_action(self, action, *args): + self.config_lock.acquire_lock() + action(*args) + self.config_lock.release_lock() + class GenericUpdateFactory: def create_patch_applier(self, config_format, verbose, dry_run): self.init_verbose_logging(verbose) - configwrapper = self.get_config_wrapper(dry_run) + config_wrapper = self.get_config_wrapper(dry_run) - patch_applier = PatchApplier(configwrapper=configwrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper) - patchwrapper = PatchWrapper(configwrapper) + patch_wrapper = PatchWrapper(config_wrapper) if config_format == ConfigFormat.CONFIGDB: patch_applier = \ ConfigDbDecorator( \ - decorated_patch_applier = patch_applier, patchwrapper=patchwrapper, configwrapper=configwrapper) + decorated_patch_applier = patch_applier, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) elif config_format == ConfigFormat.SONICYANG: pass else: @@ -319,17 +364,17 @@ def create_patch_applier(self, config_format, verbose, dry_run): def create_config_replacer(self, config_format, verbose, dry_run): self.init_verbose_logging(verbose) - configwrapper = self.get_config_wrapper(dry_run) + config_wrapper = self.get_config_wrapper(dry_run) - patchapplier = PatchApplier(configwrapper=configwrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper) - patchwrapper = PatchWrapper(configwrapper) + patch_wrapper = PatchWrapper(config_wrapper) - config_replacer = ConfigReplacer(patchapplier=patchapplier, configwrapper=configwrapper) + config_replacer = ConfigReplacer(patch_applier=patch_applier, config_wrapper=config_wrapper) if config_format == ConfigFormat.CONFIGDB: config_replacer = \ ConfigDbDecorator( \ - decorated_config_replacer = config_replacer, patchwrapper=patchwrapper, configwrapper=configwrapper) + decorated_config_replacer = config_replacer, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) elif config_format == ConfigFormat.SONICYANG: pass else: @@ -343,11 +388,11 @@ def create_config_replacer(self, config_format, verbose, dry_run): def create_config_rollbacker(self, verbose, dry_run): self.init_verbose_logging(verbose) - configwrapper = self.get_config_wrapper(dry_run) + config_wrapper = self.get_config_wrapper(dry_run) - patchapplier = PatchApplier(configwrapper=configwrapper) - config_replacer = ConfigReplacer(configwrapper=configwrapper, patchapplier=patchapplier) - config_rollbacker = FileSystemConfigRollbacker(configwrapper = configwrapper, config_replacer = config_replacer) + patch_applier = PatchApplier(config_wrapper=config_wrapper) + config_replacer = ConfigReplacer(config_wrapper=config_wrapper, patch_applier=patch_applier) + config_rollbacker = FileSystemConfigRollbacker(config_wrapper = config_wrapper, config_replacer = config_replacer) if not dry_run: config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker = config_rollbacker) @@ -393,48 +438,4 @@ def delete_checkpoint(self, checkpoint_name, verbose, dry_run): def list_checkpoints(self, verbose, dry_run): config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) - return config_rollbacker.list_checkpoints() - -class ConfigDbDecorator(PatchApplier, ConfigReplacer): - def __init__(self, patchwrapper, configwrapper, decorated_patch_applier=None, decorated_config_replacer=None): - self.decorated_patch_applier = decorated_patch_applier - self.decorated_config_replacer = decorated_config_replacer - self.patchwrapper = patchwrapper - self.configwrapper = configwrapper - - def apply(self, patch): - yang_patch = self.patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - self.decorated_patch_applier.apply(yang_patch) - - def replace(self, target_config): - yang_target_config = self.configwrapper.convert_config_db_to_sonic_yang(target_config) - self.decorated_config_replacer.replace(yang_target_config) - -class ConfigLockDecorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): - def __init__( \ - self, \ - decorated_patch_applier=None, \ - decorated_config_replacer=None, \ - decorated_config_rollbacker=None, \ - configlock = ConfigLock()): - self.configlock = configlock - self.decorated_patch_applier = decorated_patch_applier - self.decorated_config_replacer = decorated_config_replacer - self.decorated_config_rollbacker = decorated_config_rollbacker - - def apply(self, patch): - self.execute_write_action(self.decorated_patch_applier.apply, patch) - - def replace(self, target_config): - self.execute_write_action(self.decorated_config_replacer.replace, target_config) - - def rollback(self, checkpoint_name): - self.execute_write_action(self.decorated_config_rollbacker.rollback, checkpoint_name) - - def checkpoint(self, checkpoint_name): - self.execute_write_action(self.decorated_config_rollbacker.checkpoint, checkpoint_name) - - def execute_write_action(self, action, *args): - self.configlock.acquire_lock() - action(*args) - self.configlock.release_lock() + return config_rollbacker.list_checkpoints() \ No newline at end of file diff --git a/tests/generic_config_updater/__init__.py b/tests/generic_config_updater/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/generic_config_updater/files/config_db_as_dict.py-dict b/tests/generic_config_updater/files/config_db_as_dict.py-dict new file mode 100644 index 0000000000..9d76c2fc12 --- /dev/null +++ b/tests/generic_config_updater/files/config_db_as_dict.py-dict @@ -0,0 +1,121 @@ +{ + "VLAN_MEMBER": { + ("Vlan1000", "Ethernet0"): { + "tagging_mode": "untagged" + }, + ("Vlan1000", "Ethernet4"): { + "tagging_mode": "untagged" + }, + ("Vlan1000", "Ethernet8"): { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + }, + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "wred_red_enable": "true", + "ecn": "ecn_all", + "green_max_threshold": "2097152", + "green_min_threshold": "1048576", + "yellow_max_threshold": "2097152", + "yellow_min_threshold": "1048576", + "red_max_threshold": "2097152", + "red_min_threshold": "1048576", + "green_drop_probability": "5", + "yellow_drop_probability": "5", + "red_drop_probability": "5" + } + } +} \ No newline at end of file diff --git a/tests/generic_config_updater/files/config_db_as_json.json b/tests/generic_config_updater/files/config_db_as_json.json new file mode 100644 index 0000000000..a6d8b3eb03 --- /dev/null +++ b/tests/generic_config_updater/files/config_db_as_json.json @@ -0,0 +1,103 @@ +{ + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "wred_red_enable": "true", + "ecn": "ecn_all", + "green_max_threshold": "2097152", + "green_min_threshold": "1048576", + "yellow_max_threshold": "2097152", + "yellow_min_threshold": "1048576", + "red_max_threshold": "2097152", + "red_min_threshold": "1048576", + "green_drop_probability": "5", + "yellow_drop_probability": "5", + "red_drop_probability": "5" + } + } +} diff --git a/tests/generic_config_updater/files/cropped_config_db_as_json.json b/tests/generic_config_updater/files/cropped_config_db_as_json.json new file mode 100644 index 0000000000..261e912c71 --- /dev/null +++ b/tests/generic_config_updater/files/cropped_config_db_as_json.json @@ -0,0 +1,86 @@ +{ + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + } + } +} diff --git a/tests/generic_config_updater/files/multi_operation_config_db_patch.json-patch b/tests/generic_config_updater/files/multi_operation_config_db_patch.json-patch new file mode 100644 index 0000000000..8eddd7a19d --- /dev/null +++ b/tests/generic_config_updater/files/multi_operation_config_db_patch.json-patch @@ -0,0 +1,88 @@ +[ + { + "op": "add", + "path": "/PORT/Ethernet3", + "value": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "10000" + } + }, + { + "op": "add", + "path": "/PORT/Ethernet1", + "value": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "10000" + } + }, + { + "op": "add", + "path": "/PORT/Ethernet2", + "value": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "10000" + } + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/lanes", + "value": "65" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/alias", + "value": "Eth1/1" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/description", + "value": "" + }, + { + "op": "replace", + "path": "/PORT/Ethernet0/speed", + "value": "10000" + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet2", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet3", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/VLAN_MEMBER/Vlan100|Ethernet1", + "value": { + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/1", + "value": "Ethernet1" + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/2", + "value": "Ethernet2" + }, + { + "op": "add", + "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/3", + "value": "Ethernet3" + } +] diff --git a/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch b/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch new file mode 100644 index 0000000000..8c33c554ae --- /dev/null +++ b/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch @@ -0,0 +1,97 @@ +[ + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/3", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet2", + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/4", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet3", + "tagging_mode": "untagged" + } + }, + { + "op": "add", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/5", + "value": { + "vlan_name": "Vlan100", + "port": "Ethernet1", + "tagging_mode": "untagged" + } + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/lanes", + "value": "65" + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/alias", + "value": "Eth1/1" + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/speed", + "value": 10000 + }, + { + "op": "replace", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/description", + "value": "" + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/3", + "value": { + "port_name": "Ethernet3", + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/4", + "value": { + "port_name": "Ethernet1", + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/5", + "value": { + "port_name": "Ethernet2", + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": 10000 + } + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/1", + "value": "Ethernet1" + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/2", + "value": "Ethernet2" + }, + { + "op": "add", + "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/3", + "value": "Ethernet3" + } +] diff --git a/tests/generic_config_updater/files/single_operation_config_db_patch.json-patch b/tests/generic_config_updater/files/single_operation_config_db_patch.json-patch new file mode 100644 index 0000000000..7cc0967bf0 --- /dev/null +++ b/tests/generic_config_updater/files/single_operation_config_db_patch.json-patch @@ -0,0 +1,6 @@ +[ + { + "op": "remove", + "path": "/VLAN_MEMBER/Vlan1000|Ethernet8" + } +] diff --git a/tests/generic_config_updater/files/single_operation_sonic_yang_patch.json-patch b/tests/generic_config_updater/files/single_operation_sonic_yang_patch.json-patch new file mode 100644 index 0000000000..5a46560496 --- /dev/null +++ b/tests/generic_config_updater/files/single_operation_sonic_yang_patch.json-patch @@ -0,0 +1,6 @@ +[ + { + "op": "remove", + "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/2" + } +] diff --git a/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json b/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json new file mode 100644 index 0000000000..05e0379dcc --- /dev/null +++ b/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json @@ -0,0 +1,153 @@ +{ + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet2", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet3", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan100", + "port": "Ethernet1", + "tagging_mode": "untagged" + } + ] + }, + "sonic-vlan:VLAN": { + "VLAN_LIST": [ + { + "vlan_name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-acl:ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0", + "Ethernet1", + "Ethernet2", + "Ethernet3" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "sonic-port:PORT": { + "PORT_LIST": [ + { + "port_name": "Ethernet0", + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet3", + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet1", + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": 10000 + }, + { + "port_name": "Ethernet2", + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": 10000 + } + ] + } + } +} diff --git a/tests/generic_config_updater/files/sonic_yang_as_json.json b/tests/generic_config_updater/files/sonic_yang_as_json.json new file mode 100644 index 0000000000..c921fa5384 --- /dev/null +++ b/tests/generic_config_updater/files/sonic_yang_as_json.json @@ -0,0 +1,114 @@ +{ + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "vlan_name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + } + ] + }, + "sonic-vlan:VLAN": { + "VLAN_LIST": [ + { + "vlan_name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-acl:ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "sonic-port:PORT": { + "PORT_LIST": [ + { + "port_name": "Ethernet0", + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": 100000 + }, + { + "port_name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "port_name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + } + ] + } + } +} diff --git a/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json b/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json new file mode 100644 index 0000000000..7a8c37f7a2 --- /dev/null +++ b/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json @@ -0,0 +1,13 @@ +{ + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "vlan_name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + } + ] + } + } +} diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py new file mode 100644 index 0000000000..63e606947d --- /dev/null +++ b/tests/generic_config_updater/generic_updater_test.py @@ -0,0 +1,991 @@ +import json +import jsonpatch +import os +import shutil +import unittest +from imp import load_source +from unittest.mock import Mock, call +load_source('gu', \ + os.path.join(os.path.dirname(__file__), '../..', 'generic_config_updater', 'generic_updater.py')) +import gu + +class MockSideEffectDict: + def __init__(self, map): + self.map = map + + def side_effect_func(self, *args): + l = [str(arg) for arg in args] + key = tuple(l) + value = self.map.get(key) + if value == None: + raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") + + return value + +def create_side_effect_dict(map): + return MockSideEffectDict(map).side_effect_func + +class FilesLoader: + def __getattr__(self, attr): + return self.__load(attr) + + def __load(self, file_name): + normalized_file_name = file_name.lower() + + # Try load dict file + json_file_path = os.path.join("files", f"{normalized_file_name}.py-dict") + if os.path.isfile(json_file_path): + with open(json_file_path) as fh: + text = fh.read() + return eval(text) + + # Try load json file + json_file_path = os.path.join("files", f"{normalized_file_name}.json") + if os.path.isfile(json_file_path): + with open(json_file_path) as fh: + text = fh.read() + return json.loads(text) + + # Try load json-patch file + jsonpatch_file_path = os.path.join("files", f"{normalized_file_name}.json-patch") + if os.path.isfile(jsonpatch_file_path): + with open(jsonpatch_file_path) as fh: + text = fh.read() + return jsonpatch.JsonPatch(json.loads(text)) + + raise ValueError(f"There is no file called '{file_name}' in 'files/' directory") + +# Files.File_Name will look for a file called "file_name" in the "files/" directory +Files = FilesLoader() + +class TestConfigWrapper(unittest.TestCase): + def test_ctor__default_values_set(self): + config_wrapper = gu.ConfigWrapper() + + self.assertEqual(None, config_wrapper.default_config_db_connector) + self.assertEqual("/usr/local/yang-models", gu.YANG_DIR) + + def test_get_config_db_as_json__returns_config_db_as_json(self): + # Arrange + config_db_connector_mock = self.__get_config_db_connector_mock(Files.CONFIG_DB_AS_DICT) + config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + expected = Files.CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.get_config_db_as_json() + + # Assert + self.assertDictEqual(expected, actual) + + def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): + # Arrange + config_db_connector_mock = self.__get_config_db_connector_mock(Files.CONFIG_DB_AS_DICT) + config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + expected = Files.SONIC_YANG_AS_JSON + + # Act + actual = config_wrapper.get_sonic_yang_as_json() + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__empty_config_db__returns_empty_sonic_yang(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = {} + + # Act + actual = config_wrapper.convert_config_db_to_sonic_yang({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__non_empty_config_db__returns_sonic_yang_as_json(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = Files.SONIC_YANG_AS_JSON + + # Act + actual = config_wrapper.convert_config_db_to_sonic_yang(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__empty_sonic_yang__returns_empty_config_db(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = {} + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_validate_sonic_yang_config__valid_config__returns_true(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = True + + # Act + actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_sonic_yang_config__invvalid_config__returns_false(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = False + + # Act + actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON_INVALID) + + # Assert + self.assertEqual(expected, actual) + + def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.crop_tables_without_yang(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def __get_config_db_connector_mock(self, config_db_as_dict): + mock_connector = Mock() + mock_connector.get_config.return_value = config_db_as_dict + return mock_connector + +class TestPatchWrapper(unittest.TestCase): + def test_validate_config_db_patch__table_without_yang_model__returns_false(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + expected = False + + # Act + actual = patch_wrapper.validate_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_config_db_patch__table_with_yang_model__returns_true(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] + expected = True + + # Act + actual = patch_wrapper.validate_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__failure(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + + # Act and Assert + self.assertRaises(Exception, patch_wrapper.convert_config_db_patch_to_sonic_yang_patch, patch) + + def test_same_patch__no_diff__returns_true(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + + # Act and Assert + self.assertTrue(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON)) + + def test_same_patch__diff__returns_false(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + + # Act and Assert + self.assertFalse(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CROPPED_CONFIG_DB_AS_JSON)) + + def test_generate_patch__no_diff__empty_patch(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + + # Act + patch = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertFalse(patch) + + def test_simulate_patch__empty_patch__no_changes(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + patch = jsonpatch.JsonPatch([]) + expected = Files.CONFIG_DB_AS_JSON + + # Act + actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_simulate_patch__non_empty_patch__changes_applied(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) + + # Act + actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_generate_patch__diff__non_empty_patch(self): + # Arrange + patch_wrapper = gu.PatchWrapper() + after_update_json = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + + # Act + actual = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, after_update_json) + + # Assert + self.assertTrue(actual) + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch = jsonpatch.JsonPatch([]) + expected = jsonpatch.JsonPatch([]) + + # Act + actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + expected = Files.SINGLE_OPERATION_SONIC_YANG_PATCH + + # Act + actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + config_db_patch = Files.MULTI_OPERATION_CONFIG_DB_PATCH + + # Act + sonic_yang_patch = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(config_db_patch) + + # Assert + self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) + + def __assert_same_patch(self, config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper): + sonic_yang = config_wrapper.get_sonic_yang_as_json() + config_db = config_wrapper.get_config_db_as_json() + + after_update_sonic_yang = patch_wrapper.simulate_patch(sonic_yang_patch, sonic_yang) + after_update_config_db = patch_wrapper.simulate_patch(config_db_patch, config_db) + + after_update_config_db_as_sonic_yang = \ + config_wrapper.convert_config_db_to_sonic_yang(after_update_config_db) + + self.assertTrue(patch_wrapper.verify_same_json(after_update_sonic_yang, after_update_config_db_as_sonic_yang)) + + def __get_config_wrapper_mock(self, config_db_as_dict): + config_db_connector_mock = self.__get_config_db_connector_mock(config_db_as_dict) + config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + return config_wrapper + + def __get_config_db_connector_mock(self, config_db_as_dict): + mock_connector = Mock() + mock_connector.get_config.return_value = config_db_as_dict + return mock_connector + +class TestPatchApplier(unittest.TestCase): + def test_apply__invalid_sonic_yang__failure(self): + # Arrange + patch_applier = self.__create_patch_applier(valid_sonic_yang=False) + + # Act and assert + self.assertRaises(Exception, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + + def test_apply__json_not_fully_updated__failure(self): + # Arrange + patch_applier = self.__create_patch_applier(verified_same_config=False) + + # Act and assert + self.assertRaises(Exception, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + + def test_apply__no_errors__update_successful(self): + # Arrange + changes = [Mock(), Mock()] + patch_applier = self.__create_patch_applier(changes) + + # Act + patch_applier.apply(Files.MULTI_OPERATION_SONIC_YANG_PATCH) + + # Assert + patch_applier.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + patch_applier.patch_wrapper.simulate_patch.assert_has_calls( \ + [call(Files.MULTI_OPERATION_SONIC_YANG_PATCH, Files.SONIC_YANG_AS_JSON)]) + patch_applier.config_wrapper.validate_sonic_yang_config.assert_has_calls( \ + [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + patch_applier.patchorderer.order.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) + patch_applier.changeapplier.apply.assert_has_calls([call(changes[0]), call(changes[1])]) + patch_applier.patch_wrapper.verify_same_json.assert_has_calls( \ + [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + + def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + config_wrapper = Mock() + config_wrapper.get_sonic_yang_as_json.side_effect = \ + [Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH] + config_wrapper.validate_sonic_yang_config.side_effect = \ + create_side_effect_dict({(str(Files.SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) + + patch_wrapper = Mock() + patch_wrapper.simulate_patch.side_effect = \ + create_side_effect_dict( \ + {(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH), str(Files.SONIC_YANG_AS_JSON)): \ + Files.SONIC_YANG_AFTER_MULTI_PATCH}) + patch_wrapper.verify_same_json.side_effect = \ + create_side_effect_dict( \ + {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + verified_same_config}) + + changes = [Mock(), Mock()] if not changes else changes + patchorderer = Mock() + patchorderer.order.side_effect = \ + create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) + + changeapplier = Mock() + changeapplier.apply.side_effect = create_side_effect_dict({(str(changes[0]),): 0, (str(changes[1]),): 0}) + + return gu.PatchApplier(patchorderer, changeapplier, config_wrapper, patch_wrapper) + +class TestConfigReplacer(unittest.TestCase): + def test_replace__invalid_sonic_yang__failure(self): + # Arrange + config_replacer = self.__create_config_replacer(valid_sonic_yang=False) + + # Act and assert + self.assertRaises(Exception, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + + def test_replace__json_not_fully_updated__failure(self): + # Arrange + config_replacer = self.__create_config_replacer(verified_same_config=False) + + # Act and assert + self.assertRaises(Exception, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + + def test_replace__no_errors__update_successful(self): + # Arrange + config_replacer = self.__create_config_replacer() + + # Act + config_replacer.replace(Files.SONIC_YANG_AFTER_MULTI_PATCH) + + # Assert + config_replacer.config_wrapper.validate_sonic_yang_config.assert_has_calls( \ + [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + config_replacer.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + config_replacer.patch_wrapper.generate_patch.assert_has_calls( \ + [call(Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + config_replacer.patch_applier.apply.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) + config_replacer.patch_wrapper.verify_same_json.assert_has_calls( \ + [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + + def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + config_wrapper = Mock() + config_wrapper.validate_sonic_yang_config.side_effect = \ + create_side_effect_dict({(str(Files.SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) + config_wrapper.get_sonic_yang_as_json.side_effect = \ + [Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH] + + patch_wrapper = Mock() + patch_wrapper.generate_patch.side_effect = \ + create_side_effect_dict( \ + {(str(Files.SONIC_YANG_AS_JSON), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + Files.MULTI_OPERATION_SONIC_YANG_PATCH}) + patch_wrapper.verify_same_json.side_effect = \ + create_side_effect_dict( \ + {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + verified_same_config}) + + changes = [Mock(), Mock()] if not changes else changes + patchorderer = Mock() + patchorderer.order.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): \ + changes}) + + patch_applier = Mock() + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): 0}) + + return gu.ConfigReplacer(patch_applier, config_wrapper, patch_wrapper) + +class TestFileSystemConfigRollbacker(unittest.TestCase): + def setUp(self): + self.checkpoints_dir = os.path.join(os.getcwd(),"checkpoints") + self.checkpoint_ext = ".cp.json" + self.any_checkpoint_name = "anycheckpoint" + self.any_other_checkpoint_name = "anyothercheckpoint" + self.any_config = {} + self.clean_up() + + def tearDown(self): + self.clean_up() + + def test_rollback__checkpoint_does_not_exist__failure(self): + # Arrange + rollbacker = self.create_rollbacker() + + # Act and assert + self.assertRaises(Exception, rollbacker.rollback, "NonExistingCheckpoint") + + def test_rollback__no_errors__success(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + + # Act + rollbacker.rollback(self.any_checkpoint_name) + + # Assert + rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) + + def test_checkpoint__checkpoints_dir_does_not_exist__checkpoint_created(self): + # Arrange + rollbacker = self.create_rollbacker() + self.assertFalse(os.path.isdir(self.checkpoints_dir)) + + # Act + rollbacker.checkpoint(self.any_checkpoint_name) + + # Assert + self.assertTrue(os.path.isdir(self.checkpoints_dir)) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + def test_checkpoint__checkpoints_dir_exists__checkpoint_created(self): + # Arrange + self.create_checkpoints_dir() + rollbacker = self.create_rollbacker() + + # Act + rollbacker.checkpoint(self.any_checkpoint_name) + + # Assert + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + def test_list_checkpoints__checkpoints_dir_does_not_exist__empty_list(self): + # Arrange + rollbacker = self.create_rollbacker() + self.assertFalse(os.path.isdir(self.checkpoints_dir)) + expected = [] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + + def test_list_checkpoints__checkpoints_dir_exist_but_no_files__empty_list(self): + # Arrange + self.create_checkpoints_dir() + rollbacker = self.create_rollbacker() + expected = [] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_list_checkpoints__checkpoints_dir_has_multiple_files__multiple_files(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + self.add_checkpoint(self.any_other_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + expected = [self.any_checkpoint_name, self.any_other_checkpoint_name] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_list_checkpoints__checkpoints_names_have_special_characters__multiple_files(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint("check.point1", self.any_config) + self.add_checkpoint(".checkpoint2", self.any_config) + self.add_checkpoint("checkpoint3.", self.any_config) + rollbacker = self.create_rollbacker() + expected = ["check.point1", ".checkpoint2", "checkpoint3."] + + # Act + actual = rollbacker.list_checkpoints() + + # Assert + self.assertListEqual(expected, actual) + + def test_delete_checkpoint__checkpoint_does_not_exist__failure(self): + # Arrange + rollbacker = self.create_rollbacker() + + # Act and assert + self.assertRaises(Exception, rollbacker.delete_checkpoint, self.any_checkpoint_name) + + def test_delete_checkpoint__checkpoint_exist__success(self): + # Arrange + self.create_checkpoints_dir() + self.add_checkpoint(self.any_checkpoint_name, self.any_config) + rollbacker = self.create_rollbacker() + + # Act + rollbacker.delete_checkpoint(self.any_checkpoint_name) + + # Assert + self.assertFalse(self.check_checkpoint_exists(self.any_checkpoint_name)) + + def test_multiple_operations(self): + rollbacker = self.create_rollbacker() + + self.assertListEqual([], rollbacker.list_checkpoints()) + + rollbacker.checkpoint(self.any_checkpoint_name) + self.assertListEqual([self.any_checkpoint_name], rollbacker.list_checkpoints()) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + + rollbacker.rollback(self.any_checkpoint_name) + rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) + + rollbacker.checkpoint(self.any_other_checkpoint_name) + self.assertListEqual([self.any_checkpoint_name, self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + self.assertEqual(self.any_config, self.get_checkpoint(self.any_other_checkpoint_name)) + + rollbacker.delete_checkpoint(self.any_checkpoint_name) + self.assertListEqual([self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + + rollbacker.delete_checkpoint(self.any_other_checkpoint_name) + self.assertListEqual([], rollbacker.list_checkpoints()) + + def clean_up(self): + if os.path.isdir(self.checkpoints_dir): + shutil.rmtree(self.checkpoints_dir) + + def create_checkpoints_dir(self): + os.makedirs(self.checkpoints_dir) + + def add_checkpoint(self, name, json_content): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + with open(path, "w") as fh: + fh.write(json.dumps(json_content)) + + def get_checkpoint(self, name): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + with open(path) as fh: + text = fh.read() + return json.loads(text) + + def check_checkpoint_exists(self, name): + path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") + return os.path.isfile(path) + + def create_rollbacker(self): + replacer = Mock() + replacer.replace.side_effect = create_side_effect_dict({(str(self.any_config),): 0}) + + config_wrapper = Mock() + config_wrapper.get_sonic_yang_as_json.return_value = self.any_config + + return gu.FileSystemConfigRollbacker( \ + checkpoints_dir=self.checkpoints_dir, \ + config_replacer=replacer, \ + config_wrapper=config_wrapper) + +class TestGenericUpdateFactory(unittest.TestCase): + def setUp(self): + self.any_verbose=True + self.any_dry_run=True + + def test_create_patch_applier__invalid_config_format__failure(self): + # Arrange + factory = gu.GenericUpdateFactory() + + # Act and assert + self.assertRaises( \ + ValueError, factory.create_patch_applier, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) + + def test_create_patch_applier__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: gu.ConfigLockDecorator}}, + { + "config_format": { + gu.ConfigFormat.SONICYANG: None, + gu.ConfigFormat.CONFIGDB: gu.ConfigDbDecorator + } + }, + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_patch_applier) + + def test_create_config_replacer__invalid_config_format__failure(self): + # Arrange + factory = gu.GenericUpdateFactory() + + # Act and assert + self.assertRaises( \ + ValueError, factory.create_config_replacer, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) + + def test_create_config_replacer__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: gu.ConfigLockDecorator}}, + { + "config_format": { + gu.ConfigFormat.SONICYANG: None, + gu.ConfigFormat.CONFIGDB: gu.ConfigDbDecorator + } + }, + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_replacer) + + def test_create_config_rollbacker__different_options(self): + # Arrange + options = [ + {"verbose": {True: None, False: None}}, + {"dry_run": {True: None, False: gu.ConfigLockDecorator}} + ] + + # Act and assert + self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_rollbacker) + + def recursively_test_create_func(self, options, cur_option, params, expected_decorators, create_func): + if cur_option == len(options): + create_func(params, expected_decorators) + return + + param = list(options[cur_option].keys())[0] + for key in options[cur_option][param]: + params[param] = key + decorator = options[cur_option][param][key] + if decorator != None: + expected_decorators.append(decorator) + self.recursively_test_create_func(options, cur_option+1, params, expected_decorators, create_func) + if decorator != None: + expected_decorators.pop() + + def validate_create_patch_applier(self, params, expected_decorators): + factory = gu.GenericUpdateFactory() + patch_applier = factory.create_patch_applier(params["config_format"], params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(patch_applier, decorator_type) + + patch_applier = patch_applier.decorated_patch_applier + + self.assertIsInstance(patch_applier, gu.PatchApplier) + if params["dry_run"]: + self.assertIsInstance(patch_applier.config_wrapper, gu.DryRunConfigWrapper) + else: + self.assertIsInstance(patch_applier.config_wrapper, gu.ConfigWrapper) + + def validate_create_config_replacer(self, params, expected_decorators): + factory = gu.GenericUpdateFactory() + config_replacer = factory.create_config_replacer(params["config_format"], params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(config_replacer, decorator_type) + + config_replacer = config_replacer.decorated_config_replacer + + self.assertIsInstance(config_replacer, gu.ConfigReplacer) + if params["dry_run"]: + self.assertIsInstance(config_replacer.config_wrapper, gu.DryRunConfigWrapper) + self.assertIsInstance(config_replacer.patch_applier.config_wrapper, gu.DryRunConfigWrapper) + else: + self.assertIsInstance(config_replacer.config_wrapper, gu.ConfigWrapper) + self.assertIsInstance(config_replacer.patch_applier.config_wrapper, gu.ConfigWrapper) + + def validate_create_config_rollbacker(self, params, expected_decorators): + factory = gu.GenericUpdateFactory() + config_rollbacker = factory.create_config_rollbacker(params["verbose"], params["dry_run"]) + for decorator_type in expected_decorators: + self.assertIsInstance(config_rollbacker, decorator_type) + + config_rollbacker = config_rollbacker.decorated_config_rollbacker + + self.assertIsInstance(config_rollbacker, gu.FileSystemConfigRollbacker) + if params["dry_run"]: + self.assertIsInstance(config_rollbacker.config_wrapper, gu.DryRunConfigWrapper) + self.assertIsInstance(config_rollbacker.config_replacer.config_wrapper, gu.DryRunConfigWrapper) + self.assertIsInstance( \ + config_rollbacker.config_replacer.patch_applier.config_wrapper, gu.DryRunConfigWrapper) + else: + self.assertIsInstance(config_rollbacker.config_wrapper, gu.ConfigWrapper) + self.assertIsInstance(config_rollbacker.config_replacer.config_wrapper, gu.ConfigWrapper) + self.assertIsInstance( \ + config_rollbacker.config_replacer.patch_applier.config_wrapper, gu.ConfigWrapper) + +class TestGenericUpdater(unittest.TestCase): + def setUp(self): + self.any_checkpoint_name = "anycheckpoint" + self.any_other_checkpoint_name = "anyothercheckpoint" + self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] + self.any_config_format = gu.ConfigFormat.SONICYANG + self.any_verbose = True + self.any_dry_run = True + + def test_apply_patch__creates_applier_and_apply(self): + # Arrange + patch_applier = Mock() + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) + + factory = Mock() + factory.create_patch_applier.side_effect = \ + create_side_effect_dict( \ + {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): patch_applier}) + + generic_updater = gu.GenericUpdater(factory) + + # Act + generic_updater.apply_patch( \ + Files.SINGLE_OPERATION_SONIC_YANG_PATCH, self.any_config_format, self.any_verbose, self.any_dry_run) + + # Assert + patch_applier.apply.assert_has_calls([call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) + + def test_replace__creates_replacer_and_replace(self): + # Arrange + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + + factory = Mock() + factory.create_config_replacer.side_effect = \ + create_side_effect_dict( \ + {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): config_replacer}) + + generic_updater = gu.GenericUpdater(factory) + + # Act + generic_updater.replace(Files.SONIC_YANG_AS_JSON, self.any_config_format, self.any_verbose, self.any_dry_run) + + # Assert + config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + + def test_rollback__creates_rollbacker_and_rollback(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = gu.GenericUpdater(factory) + + # Act + generic_updater.rollback(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_checkpoint__creates_rollbacker_and_checkpoint(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = gu.GenericUpdater(factory) + + # Act + generic_updater.checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_delete_checkpoint__creates_rollbacker_and_deletes_checkpoint(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.delete_checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = gu.GenericUpdater(factory) + + # Act + generic_updater.delete_checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + + # Assert + config_rollbacker.delete_checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): + # Arrange + config_rollbacker = Mock() + config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list + + factory = Mock() + factory.create_config_rollbacker.side_effect = \ + create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + + generic_updater = gu.GenericUpdater(factory) + + expected = self.any_checkpoints_list + + # Act + actual = generic_updater.list_checkpoints(self.any_verbose, self.any_dry_run) + + # Assert + self.assertListEqual(expected, actual) + +class TestConfigDbDecorator(unittest.TestCase): + def test_apply__converts_to_yang_and_calls_decorated_class(self): + # Arrange + config_db_decorator = self.__create_config_db_decorator() + + # Act + config_db_decorator.apply(Files.CONFIG_DB_AS_JSON) + + # Assert + config_db_decorator.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( \ + [call(Files.CONFIG_DB_AS_JSON)]) + config_db_decorator.decorated_patch_applier.apply.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + + def test_replace__converts_to_yang_and_calls_decorated_class(self): + # Arrange + config_db_decorator = self.__create_config_db_decorator() + + # Act + config_db_decorator.replace(Files.CONFIG_DB_AS_JSON) + + # Assert + config_db_decorator.config_wrapper.convert_config_db_to_sonic_yang.assert_has_calls( \ + [call(Files.CONFIG_DB_AS_JSON)]) + config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + + def __create_config_db_decorator(self): + patch_applier = Mock() + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + + patch_wrapper = Mock() + patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.side_effect = \ + create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): Files.SONIC_YANG_AS_JSON}) + + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + + config_wrapper = Mock() + config_wrapper.convert_config_db_to_sonic_yang.side_effect = \ + create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): Files.SONIC_YANG_AS_JSON}) + + return gu.ConfigDbDecorator( \ + decorated_patch_applier=patch_applier, \ + decorated_config_replacer=config_replacer, \ + patch_wrapper=patch_wrapper, \ + config_wrapper=config_wrapper) + +class TestConfigLockDecorator(unittest.TestCase): + def setUp(self): + self.any_checkpoint_name = "anycheckpoint" + + def test_apply__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.apply(Files.SINGLE_OPERATION_SONIC_YANG_PATCH) + + # Assert + config_lock_decorator.config_lock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_patch_applier.apply.assert_has_calls( \ + [call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) + config_lock_decorator.config_lock.release_lock.assert_called_once() + + def test_replace__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.replace(Files.SONIC_YANG_AS_JSON) + + # Assert + config_lock_decorator.config_lock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + config_lock_decorator.config_lock.release_lock.assert_called_once() + + def test_rollback__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.rollback(self.any_checkpoint_name) + + # Assert + config_lock_decorator.config_lock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) + config_lock_decorator.config_lock.release_lock.assert_called_once() + + def test_checkpoint__lock_config(self): + # Arrange + config_lock_decorator = self.__create_config_lock_decorator() + + # Act + config_lock_decorator.checkpoint(self.any_checkpoint_name) + + # Assert + config_lock_decorator.config_lock.acquire_lock.assert_called_once() + config_lock_decorator.decorated_config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + config_lock_decorator.config_lock.release_lock.assert_called_once() + + def __create_config_lock_decorator(self): + config_lock = Mock() + + patch_applier = Mock() + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) + + config_replacer = Mock() + config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + + config_rollbacker = Mock() + config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + + return gu.ConfigLockDecorator( \ + config_lock=config_lock, \ + decorated_patch_applier=patch_applier, \ + decorated_config_replacer=config_replacer, \ + decorated_config_rollbacker=config_rollbacker) diff --git a/tests/generic_update_test.py b/tests/generic_update_test.py deleted file mode 100644 index bfe6de48e0..0000000000 --- a/tests/generic_update_test.py +++ /dev/null @@ -1,1744 +0,0 @@ -import json -import jsonpatch -import os -import shutil -import unittest -from imp import load_source -from unittest.mock import Mock, call -load_source('generic_update', \ - os.path.join(os.path.dirname(__file__), '..', 'config', 'generic_update.py')) -import generic_update - -class MockSideEffectDict: - def __init__(self, map): - self.map = map - - def side_effect_func(self, *args): - l = [str(arg) for arg in args] - key = tuple(l) - value = self.map.get(key) - if value == None: - raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") - - return value - -def create_side_effect_dict(map): - return MockSideEffectDict(map).side_effect_func - -class TestConfigWrapper(unittest.TestCase): - def test_ctor__default_values_set(self): - configwrapper = generic_update.ConfigWrapper() - - self.assertEqual(None, configwrapper.default_config_db_connector) - self.assertEqual("/usr/local/yang-models", generic_update.YANG_DIR) - - def test_get_config_db_as_json__returns_config_db_as_json(self): - # Arrange - config_db_connector_mock = self.__get_config_db_connector_mock(CONFIG_DB_AS_DICT) - configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) - expected = CONFIG_DB_AS_JSON - - # Act - actual = configwrapper.get_config_db_as_json() - - # Assert - self.assertDictEqual(expected, actual) - - def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): - # Arrange - config_db_connector_mock = self.__get_config_db_connector_mock(CONFIG_DB_AS_DICT) - configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) - expected = SONIC_YANG_AS_JSON - - # Act - actual = configwrapper.get_sonic_yang_as_json() - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_config_db_to_sonic_yang__empty_config_db__returns_empty_sonic_yang(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = {} - - # Act - actual = configwrapper.convert_config_db_to_sonic_yang({}) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_config_db_to_sonic_yang__non_empty_config_db__returns_sonic_yang_as_json(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = SONIC_YANG_AS_JSON - - # Act - actual = configwrapper.convert_config_db_to_sonic_yang(CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__empty_sonic_yang__returns_empty_config_db(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = {} - - # Act - actual = configwrapper.convert_sonic_yang_to_config_db({}) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_db_as_json(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = CROPPED_CONFIG_DB_AS_JSON - - # Act - actual = configwrapper.convert_sonic_yang_to_config_db(SONIC_YANG_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_validate_sonic_yang_config__valid_config__returns_true(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = True - - # Act - actual = configwrapper.validate_sonic_yang_config(SONIC_YANG_AS_JSON) - - # Assert - self.assertEqual(expected, actual) - - def test_validate_sonic_yang_config__invvalid_config__returns_false(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = False - - # Act - actual = configwrapper.validate_sonic_yang_config(SONIC_YANG_AS_JSON_INVALID) - - # Assert - self.assertEqual(expected, actual) - - def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): - # Arrange - configwrapper = generic_update.ConfigWrapper() - expected = CROPPED_CONFIG_DB_AS_JSON - - # Act - actual = configwrapper.crop_tables_without_yang(CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def __get_config_db_connector_mock(self, config_db_as_dict): - mock_connector = Mock() - mock_connector.get_config.return_value = config_db_as_dict - return mock_connector - -class TestPatchWrapper(unittest.TestCase): - def test_validate_config_db_patch__table_without_yang_model__returns_false(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] - expected = False - - # Act - actual = patchwrapper.validate_config_db_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_validate_config_db_patch__table_with_yang_model__returns_true(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] - expected = True - - # Act - actual = patchwrapper.validate_config_db_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__failure(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] - - # Act and Assert - self.assertRaises(Exception, patchwrapper.convert_config_db_patch_to_sonic_yang_patch, patch) - - def test_same_patch__no_diff__returns_true(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - - # Act and Assert - self.assertTrue(patchwrapper.verify_same_json(CONFIG_DB_AS_JSON, CONFIG_DB_AS_JSON)) - - def test_same_patch__diff__returns_false(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - - # Act and Assert - self.assertFalse(patchwrapper.verify_same_json(CONFIG_DB_AS_JSON, CROPPED_CONFIG_DB_AS_JSON)) - - def test_generate_patch__no_diff__empty_patch(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - - # Act - patch = patchwrapper.generate_patch(CONFIG_DB_AS_JSON, CONFIG_DB_AS_JSON) - - # Assert - self.assertFalse(patch) - - def test_simulate_patch__empty_patch__no_changes(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - patch = jsonpatch.JsonPatch([]) - expected = CONFIG_DB_AS_JSON - - # Act - actual = patchwrapper.simulate_patch(patch, CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_simulate_patch__non_empty_patch__changes_applied(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - patch = SINGLE_OPERATION_CONFIG_DB_PATCH - expected = SINGLE_OPERATION_CONFIG_DB_PATCH.apply(CONFIG_DB_AS_JSON) - - # Act - actual = patchwrapper.simulate_patch(patch, CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_generate_patch__diff__non_empty_patch(self): - # Arrange - patchwrapper = generic_update.PatchWrapper() - after_update_json = SINGLE_OPERATION_CONFIG_DB_PATCH.apply(CONFIG_DB_AS_JSON) - expected = SINGLE_OPERATION_CONFIG_DB_PATCH - - # Act - actual = patchwrapper.generate_patch(CONFIG_DB_AS_JSON, after_update_json) - - # Assert - self.assertTrue(actual) - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): - # Arrange - configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) - patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) - patch = jsonpatch.JsonPatch([]) - expected = jsonpatch.JsonPatch([]) - - # Act - actual = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): - # Arrange - configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) - patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) - patch = SINGLE_OPERATION_CONFIG_DB_PATCH - expected = SINGLE_OPERATION_SONIC_YANG_PATCH - - # Act - actual = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): - # Arrange - configwrapper = self.__get_configwrapper_mock(CONFIG_DB_AS_DICT) - patchwrapper = generic_update.PatchWrapper(configwrapper = configwrapper) - config_db_patch = MULTI_OPERATION_CONFIG_DB_PATCH - - # Act - sonic_yang_patch = patchwrapper.convert_config_db_patch_to_sonic_yang_patch(config_db_patch) - - # Assert - self.__assert_same_patch(config_db_patch, sonic_yang_patch, configwrapper, patchwrapper) - - def __assert_same_patch(self, config_db_patch, sonic_yang_patch, configwrapper, patchwrapper): - sonic_yang = configwrapper.get_sonic_yang_as_json() - config_db = configwrapper.get_config_db_as_json() - - after_update_sonic_yang = patchwrapper.simulate_patch(sonic_yang_patch, sonic_yang) - after_update_config_db = patchwrapper.simulate_patch(config_db_patch, config_db) - - after_update_config_db_as_sonic_yang = \ - configwrapper.convert_config_db_to_sonic_yang(after_update_config_db) - - self.assertTrue(patchwrapper.verify_same_json(after_update_sonic_yang, after_update_config_db_as_sonic_yang)) - - def __get_configwrapper_mock(self, config_db_as_dict): - config_db_connector_mock = self.__get_config_db_connector_mock(config_db_as_dict) - configwrapper = generic_update.ConfigWrapper(default_config_db_connector = config_db_connector_mock) - return configwrapper - - def __get_config_db_connector_mock(self, config_db_as_dict): - mock_connector = Mock() - mock_connector.get_config.return_value = config_db_as_dict - return mock_connector - -class TestPatchApplier(unittest.TestCase): - def test_apply__invalid_sonic_yang__failure(self): - # Arrange - patchapplier = self.__create_patch_applier(valid_sonic_yang=False) - - # Act and assert - self.assertRaises(Exception, patchapplier.apply, MULTI_OPERATION_SONIC_YANG_PATCH) - - def test_apply__json_not_fully_updated__failure(self): - # Arrange - patchapplier = self.__create_patch_applier(verified_same_config=False) - - # Act and assert - self.assertRaises(Exception, patchapplier.apply, MULTI_OPERATION_SONIC_YANG_PATCH) - - def test_apply__no_errors__update_successful(self): - # Arrange - changes = [Mock(), Mock()] - patchapplier = self.__create_patch_applier(changes) - - # Act - patchapplier.apply(MULTI_OPERATION_SONIC_YANG_PATCH) - - # Assert - patchapplier.configwrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) - patchapplier.patchwrapper.simulate_patch.assert_has_calls( \ - [call(MULTI_OPERATION_SONIC_YANG_PATCH, SONIC_YANG_AS_JSON)]) - patchapplier.configwrapper.validate_sonic_yang_config.assert_has_calls([call(SONIC_YANG_AFTER_MULTI_PATCH)]) - patchapplier.patchorderer.order.assert_has_calls([call(MULTI_OPERATION_SONIC_YANG_PATCH)]) - patchapplier.changeapplier.apply.assert_has_calls([call(changes[0]), call(changes[1])]) - patchapplier.patchwrapper.verify_same_json.assert_has_calls( \ - [call(SONIC_YANG_AFTER_MULTI_PATCH, SONIC_YANG_AFTER_MULTI_PATCH)]) - - def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_same_config=True): - configwrapper = Mock() - configwrapper.get_sonic_yang_as_json.side_effect = [SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH] - configwrapper.validate_sonic_yang_config.side_effect = \ - create_side_effect_dict({(str(SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) - - patchwrapper = Mock() - patchwrapper.simulate_patch.side_effect = \ - create_side_effect_dict( \ - {(str(MULTI_OPERATION_SONIC_YANG_PATCH), str(SONIC_YANG_AS_JSON)): SONIC_YANG_AFTER_MULTI_PATCH}) - patchwrapper.verify_same_json.side_effect = \ - create_side_effect_dict( \ - {(str(SONIC_YANG_AFTER_MULTI_PATCH), str(SONIC_YANG_AFTER_MULTI_PATCH)): verified_same_config}) - - changes = [Mock(), Mock()] if not changes else changes - patchorderer = Mock() - patchorderer.order.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) - - changeapplier = Mock() - changeapplier.apply.side_effect = create_side_effect_dict({(str(changes[0]),): 0, (str(changes[1]),): 0}) - - return generic_update.PatchApplier(patchorderer, changeapplier, configwrapper, patchwrapper) - -class TestConfigReplacer(unittest.TestCase): - def test_replace__invalid_sonic_yang__failure(self): - # Arrange - configreplacer = self.__create_config_replacer(valid_sonic_yang=False) - - # Act and assert - self.assertRaises(Exception, configreplacer.replace, SONIC_YANG_AFTER_MULTI_PATCH) - - def test_replace__json_not_fully_updated__failure(self): - # Arrange - configreplacer = self.__create_config_replacer(verified_same_config=False) - - # Act and assert - self.assertRaises(Exception, configreplacer.replace, SONIC_YANG_AFTER_MULTI_PATCH) - - def test_replace__no_errors__update_successful(self): - # Arrange - configreplacer = self.__create_config_replacer() - - # Act - configreplacer.replace(SONIC_YANG_AFTER_MULTI_PATCH) - - # Assert - configreplacer.configwrapper.validate_sonic_yang_config.assert_has_calls([call(SONIC_YANG_AFTER_MULTI_PATCH)]) - configreplacer.configwrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) - configreplacer.patchwrapper.generate_patch.assert_has_calls( \ - [call(SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH)]) - configreplacer.patchapplier.apply.assert_has_calls([call(MULTI_OPERATION_SONIC_YANG_PATCH)]) - configreplacer.patchwrapper.verify_same_json.assert_has_calls( \ - [call(SONIC_YANG_AFTER_MULTI_PATCH, SONIC_YANG_AFTER_MULTI_PATCH)]) - - def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified_same_config=True): - configwrapper = Mock() - configwrapper.validate_sonic_yang_config.side_effect = \ - create_side_effect_dict({(str(SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) - configwrapper.get_sonic_yang_as_json.side_effect = [SONIC_YANG_AS_JSON, SONIC_YANG_AFTER_MULTI_PATCH] - - patchwrapper = Mock() - patchwrapper.generate_patch.side_effect = \ - create_side_effect_dict( \ - {(str(SONIC_YANG_AS_JSON), str(SONIC_YANG_AFTER_MULTI_PATCH)): MULTI_OPERATION_SONIC_YANG_PATCH}) - patchwrapper.verify_same_json.side_effect = \ - create_side_effect_dict( \ - {(str(SONIC_YANG_AFTER_MULTI_PATCH), str(SONIC_YANG_AFTER_MULTI_PATCH)): verified_same_config}) - - changes = [Mock(), Mock()] if not changes else changes - patchorderer = Mock() - patchorderer.order.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) - - patchapplier = Mock() - patchapplier.apply.side_effect = create_side_effect_dict({(str(MULTI_OPERATION_SONIC_YANG_PATCH),): 0}) - - return generic_update.ConfigReplacer(patchapplier, configwrapper, patchwrapper) - -class TestFileSystemConfigRollbacker(unittest.TestCase): - def setUp(self): - self.checkpoints_dir = os.path.join(os.getcwd(),"checkpoints") - self.checkpoint_ext = ".cp.json" - self.any_checkpoint_name = "anycheckpoint" - self.any_other_checkpoint_name = "anyothercheckpoint" - self.any_config = {} - self.clean_up() - - def tearDown(self): - self.clean_up() - - def test_rollback__checkpoint_does_not_exist__failure(self): - # Arrange - rollbacker = self.create_rollbacker() - - # Act and assert - self.assertRaises(Exception, rollbacker.rollback, "NonExistingCheckpoint") - - def test_rollback__no_errors__success(self): - # Arrange - self.create_checkpoints_dir() - self.add_checkpoint(self.any_checkpoint_name, self.any_config) - rollbacker = self.create_rollbacker() - - # Act - rollbacker.rollback(self.any_checkpoint_name) - - # Assert - rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) - - def test_checkpoint__checkpoints_dir_does_not_exist__checkpoint_created(self): - # Arrange - rollbacker = self.create_rollbacker() - self.assertFalse(os.path.isdir(self.checkpoints_dir)) - - # Act - rollbacker.checkpoint(self.any_checkpoint_name) - - # Assert - self.assertTrue(os.path.isdir(self.checkpoints_dir)) - self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) - - def test_checkpoint__checkpoints_dir_exists__checkpoint_created(self): - # Arrange - self.create_checkpoints_dir() - rollbacker = self.create_rollbacker() - - # Act - rollbacker.checkpoint(self.any_checkpoint_name) - - # Assert - self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) - - def test_list_checkpoints__checkpoints_dir_does_not_exist__empty_list(self): - # Arrange - rollbacker = self.create_rollbacker() - self.assertFalse(os.path.isdir(self.checkpoints_dir)) - expected = [] - - # Act - actual = rollbacker.list_checkpoints() - - # Assert - self.assertListEqual(expected, actual) - - - def test_list_checkpoints__checkpoints_dir_exist_but_no_files__empty_list(self): - # Arrange - self.create_checkpoints_dir() - rollbacker = self.create_rollbacker() - expected = [] - - # Act - actual = rollbacker.list_checkpoints() - - # Assert - self.assertListEqual(expected, actual) - - def test_list_checkpoints__checkpoints_dir_has_multiple_files__multiple_files(self): - # Arrange - self.create_checkpoints_dir() - self.add_checkpoint(self.any_checkpoint_name, self.any_config) - self.add_checkpoint(self.any_other_checkpoint_name, self.any_config) - rollbacker = self.create_rollbacker() - expected = [self.any_checkpoint_name, self.any_other_checkpoint_name] - - # Act - actual = rollbacker.list_checkpoints() - - # Assert - self.assertListEqual(expected, actual) - - def test_list_checkpoints__checkpoints_names_have_special_characters__multiple_files(self): - # Arrange - self.create_checkpoints_dir() - self.add_checkpoint("check.point1", self.any_config) - self.add_checkpoint(".checkpoint2", self.any_config) - self.add_checkpoint("checkpoint3.", self.any_config) - rollbacker = self.create_rollbacker() - expected = ["check.point1", ".checkpoint2", "checkpoint3."] - - # Act - actual = rollbacker.list_checkpoints() - - # Assert - self.assertListEqual(expected, actual) - - def test_delete_checkpoint__checkpoint_does_not_exist__failure(self): - # Arrange - rollbacker = self.create_rollbacker() - - # Act and assert - self.assertRaises(Exception, rollbacker.delete_checkpoint, self.any_checkpoint_name) - - def test_delete_checkpoint__checkpoint_exist__success(self): - # Arrange - self.create_checkpoints_dir() - self.add_checkpoint(self.any_checkpoint_name, self.any_config) - rollbacker = self.create_rollbacker() - - # Act - rollbacker.delete_checkpoint(self.any_checkpoint_name) - - # Assert - self.assertFalse(self.check_checkpoint_exists(self.any_checkpoint_name)) - - def test_multiple_operations(self): - rollbacker = self.create_rollbacker() - - self.assertListEqual([], rollbacker.list_checkpoints()) - - rollbacker.checkpoint(self.any_checkpoint_name) - self.assertListEqual([self.any_checkpoint_name], rollbacker.list_checkpoints()) - self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) - - rollbacker.rollback(self.any_checkpoint_name) - rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) - - rollbacker.checkpoint(self.any_other_checkpoint_name) - self.assertListEqual([self.any_checkpoint_name, self.any_other_checkpoint_name], rollbacker.list_checkpoints()) - self.assertEqual(self.any_config, self.get_checkpoint(self.any_other_checkpoint_name)) - - rollbacker.delete_checkpoint(self.any_checkpoint_name) - self.assertListEqual([self.any_other_checkpoint_name], rollbacker.list_checkpoints()) - - rollbacker.delete_checkpoint(self.any_other_checkpoint_name) - self.assertListEqual([], rollbacker.list_checkpoints()) - - def clean_up(self): - if os.path.isdir(self.checkpoints_dir): - shutil.rmtree(self.checkpoints_dir) - - def create_checkpoints_dir(self): - os.makedirs(self.checkpoints_dir) - - def add_checkpoint(self, name, json_content): - path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") - with open(path, "w") as fh: - fh.write(json.dumps(json_content)) - - def get_checkpoint(self, name): - path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") - with open(path) as fh: - text = fh.read() - return json.loads(text) - - def check_checkpoint_exists(self, name): - path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") - return os.path.isfile(path) - - def create_rollbacker(self): - replacer = Mock() - replacer.replace.side_effect = create_side_effect_dict({(str(self.any_config),): 0}) - - configwrapper = Mock() - configwrapper.get_sonic_yang_as_json.return_value = self.any_config - - return generic_update.FileSystemConfigRollbacker( \ - checkpoints_dir=self.checkpoints_dir, \ - config_replacer=replacer, \ - configwrapper=configwrapper) - -class TestGenericUpdateFactory(unittest.TestCase): - def setUp(self): - self.any_verbose=True - self.any_dry_run=True - - def test_create_patch_applier__invalid_config_format__failure(self): - # Arrange - factory = generic_update.GenericUpdateFactory() - - # Act and assert - self.assertRaises( \ - ValueError, factory.create_patch_applier, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) - - def test_create_patch_applier__different_options(self): - # Arrange - options = [ - {"verbose": {True: None, False: None}}, - {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}}, - { - "config_format": { - generic_update.ConfigFormat.SONICYANG: None, - generic_update.ConfigFormat.CONFIGDB: generic_update.ConfigDbDecorator - } - }, - ] - - # Act and assert - self.recursively_test_create_func(options, 0, {}, [], self.validate_create_patch_applier) - - def test_create_config_replacer__invalid_config_format__failure(self): - # Arrange - factory = generic_update.GenericUpdateFactory() - - # Act and assert - self.assertRaises( \ - ValueError, factory.create_config_replacer, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) - - def test_create_config_replacer__different_options(self): - # Arrange - options = [ - {"verbose": {True: None, False: None}}, - {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}}, - { - "config_format": { - generic_update.ConfigFormat.SONICYANG: None, - generic_update.ConfigFormat.CONFIGDB: generic_update.ConfigDbDecorator - } - }, - ] - - # Act and assert - self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_replacer) - - def test_create_config_rollbacker__different_options(self): - # Arrange - options = [ - {"verbose": {True: None, False: None}}, - {"dry_run": {True: None, False: generic_update.ConfigLockDecorator}} - ] - - # Act and assert - self.recursively_test_create_func(options, 0, {}, [], self.validate_create_config_rollbacker) - - def recursively_test_create_func(self, options, cur_option, params, expected_decorators, create_func): - if cur_option == len(options): - create_func(params, expected_decorators) - return - - param = list(options[cur_option].keys())[0] - for key in options[cur_option][param]: - params[param] = key - decorator = options[cur_option][param][key] - if decorator != None: - expected_decorators.append(decorator) - self.recursively_test_create_func(options, cur_option+1, params, expected_decorators, create_func) - if decorator != None: - expected_decorators.pop() - - def validate_create_patch_applier(self, params, expected_decorators): - factory = generic_update.GenericUpdateFactory() - patch_applier = factory.create_patch_applier(params["config_format"], params["verbose"], params["dry_run"]) - for decorator_type in expected_decorators: - self.assertIsInstance(patch_applier, decorator_type) - - patch_applier = patch_applier.decorated_patch_applier - - self.assertIsInstance(patch_applier, generic_update.PatchApplier) - if params["dry_run"]: - self.assertIsInstance(patch_applier.configwrapper, generic_update.DryRunConfigWrapper) - else: - self.assertIsInstance(patch_applier.configwrapper, generic_update.ConfigWrapper) - - def validate_create_config_replacer(self, params, expected_decorators): - factory = generic_update.GenericUpdateFactory() - config_replacer = factory.create_config_replacer(params["config_format"], params["verbose"], params["dry_run"]) - for decorator_type in expected_decorators: - self.assertIsInstance(config_replacer, decorator_type) - - config_replacer = config_replacer.decorated_config_replacer - - self.assertIsInstance(config_replacer, generic_update.ConfigReplacer) - if params["dry_run"]: - self.assertIsInstance(config_replacer.configwrapper, generic_update.DryRunConfigWrapper) - self.assertIsInstance(config_replacer.patchapplier.configwrapper, generic_update.DryRunConfigWrapper) - else: - self.assertIsInstance(config_replacer.configwrapper, generic_update.ConfigWrapper) - self.assertIsInstance(config_replacer.patchapplier.configwrapper, generic_update.ConfigWrapper) - - def validate_create_config_rollbacker(self, params, expected_decorators): - factory = generic_update.GenericUpdateFactory() - config_rollbacker = factory.create_config_rollbacker(params["verbose"], params["dry_run"]) - for decorator_type in expected_decorators: - self.assertIsInstance(config_rollbacker, decorator_type) - - config_rollbacker = config_rollbacker.decorated_config_rollbacker - - self.assertIsInstance(config_rollbacker, generic_update.FileSystemConfigRollbacker) - if params["dry_run"]: - self.assertIsInstance(config_rollbacker.configwrapper, generic_update.DryRunConfigWrapper) - self.assertIsInstance(config_rollbacker.config_replacer.configwrapper, generic_update.DryRunConfigWrapper) - self.assertIsInstance( \ - config_rollbacker.config_replacer.patchapplier.configwrapper, generic_update.DryRunConfigWrapper) - else: - self.assertIsInstance(config_rollbacker.configwrapper, generic_update.ConfigWrapper) - self.assertIsInstance(config_rollbacker.config_replacer.configwrapper, generic_update.ConfigWrapper) - self.assertIsInstance( \ - config_rollbacker.config_replacer.patchapplier.configwrapper, generic_update.ConfigWrapper) - -class TestGenericUpdater(unittest.TestCase): - def setUp(self): - self.any_checkpoint_name = "anycheckpoint" - self.any_other_checkpoint_name = "anyothercheckpoint" - self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] - self.any_config_format = generic_update.ConfigFormat.SONICYANG - self.any_verbose = True - self.any_dry_run = True - - def test_apply_patch__creates_applier_and_apply(self): - # Arrange - patch_applier = Mock() - patch_applier.apply.side_effect = create_side_effect_dict({(str(SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) - - factory = Mock() - factory.create_patch_applier.side_effect = \ - create_side_effect_dict( \ - {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): patch_applier}) - - generic_updater = generic_update.GenericUpdater(factory) - - # Act - generic_updater.apply_patch( \ - SINGLE_OPERATION_SONIC_YANG_PATCH, self.any_config_format, self.any_verbose, self.any_dry_run) - - # Assert - patch_applier.apply.assert_has_calls([call(SINGLE_OPERATION_SONIC_YANG_PATCH)]) - - def test_replace__creates_replacer_and_replace(self): - # Arrange - config_replacer = Mock() - config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) - - factory = Mock() - factory.create_config_replacer.side_effect = \ - create_side_effect_dict( \ - {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): config_replacer}) - - generic_updater = generic_update.GenericUpdater(factory) - - # Act - generic_updater.replace(SONIC_YANG_AS_JSON, self.any_config_format, self.any_verbose, self.any_dry_run) - - # Assert - config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) - - def test_rollback__creates_rollbacker_and_rollback(self): - # Arrange - config_rollbacker = Mock() - config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - - factory = Mock() - factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) - - generic_updater = generic_update.GenericUpdater(factory) - - # Act - generic_updater.rollback(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) - - # Assert - config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) - - def test_checkpoint__creates_rollbacker_and_checkpoint(self): - # Arrange - config_rollbacker = Mock() - config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - - factory = Mock() - factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) - - generic_updater = generic_update.GenericUpdater(factory) - - # Act - generic_updater.checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) - - # Assert - config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) - - def test_delete_checkpoint__creates_rollbacker_and_deletes_checkpoint(self): - # Arrange - config_rollbacker = Mock() - config_rollbacker.delete_checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - - factory = Mock() - factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) - - generic_updater = generic_update.GenericUpdater(factory) - - # Act - generic_updater.delete_checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) - - # Assert - config_rollbacker.delete_checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) - - def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): - # Arrange - config_rollbacker = Mock() - config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list - - factory = Mock() - factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) - - generic_updater = generic_update.GenericUpdater(factory) - - expected = self.any_checkpoints_list - - # Act - actual = generic_updater.list_checkpoints(self.any_verbose, self.any_dry_run) - - # Assert - self.assertListEqual(expected, actual) - -class TestConfigDbDecorator(unittest.TestCase): - def test_apply__converts_to_yang_and_calls_decorated_class(self): - # Arrange - config_db_decorator = self.__create_config_db_decorator() - - # Act - config_db_decorator.apply(CONFIG_DB_AS_JSON) - - # Assert - config_db_decorator.patchwrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( \ - [call(CONFIG_DB_AS_JSON)]) - config_db_decorator.decorated_patch_applier.apply.assert_has_calls([call(SONIC_YANG_AS_JSON)]) - - def test_replace__converts_to_yang_and_calls_decorated_class(self): - # Arrange - config_db_decorator = self.__create_config_db_decorator() - - # Act - config_db_decorator.replace(CONFIG_DB_AS_JSON) - - # Assert - config_db_decorator.configwrapper.convert_config_db_to_sonic_yang.assert_has_calls([call(CONFIG_DB_AS_JSON)]) - config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) - - def __create_config_db_decorator(self): - patchapplier = Mock() - patchapplier.apply.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) - - patchwrapper = Mock() - patchwrapper.convert_config_db_patch_to_sonic_yang_patch.side_effect = \ - create_side_effect_dict({(str(CONFIG_DB_AS_JSON),): SONIC_YANG_AS_JSON}) - - config_replacer = Mock() - config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) - - configwrapper = Mock() - configwrapper.convert_config_db_to_sonic_yang.side_effect = \ - create_side_effect_dict({(str(CONFIG_DB_AS_JSON),): SONIC_YANG_AS_JSON}) - - return generic_update.ConfigDbDecorator( \ - decorated_patch_applier=patchapplier, \ - decorated_config_replacer=config_replacer, \ - patchwrapper=patchwrapper, \ - configwrapper=configwrapper) - -class TestConfigLockDecorator(unittest.TestCase): - def setUp(self): - self.any_checkpoint_name = "anycheckpoint" - - def test_apply__lock_config(self): - # Arrange - config_lock_decorator = self.__create_config_lock_decorator() - - # Act - config_lock_decorator.apply(SINGLE_OPERATION_SONIC_YANG_PATCH) - - # Assert - config_lock_decorator.configlock.acquire_lock.assert_called_once() - config_lock_decorator.decorated_patch_applier.apply.assert_has_calls([call(SINGLE_OPERATION_SONIC_YANG_PATCH)]) - config_lock_decorator.configlock.release_lock.assert_called_once() - - def test_replace__lock_config(self): - # Arrange - config_lock_decorator = self.__create_config_lock_decorator() - - # Act - config_lock_decorator.replace(SONIC_YANG_AS_JSON) - - # Assert - config_lock_decorator.configlock.acquire_lock.assert_called_once() - config_lock_decorator.decorated_config_replacer.replace.assert_has_calls([call(SONIC_YANG_AS_JSON)]) - config_lock_decorator.configlock.release_lock.assert_called_once() - - def test_rollback__lock_config(self): - # Arrange - config_lock_decorator = self.__create_config_lock_decorator() - - # Act - config_lock_decorator.rollback(self.any_checkpoint_name) - - # Assert - config_lock_decorator.configlock.acquire_lock.assert_called_once() - config_lock_decorator.decorated_config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) - config_lock_decorator.configlock.release_lock.assert_called_once() - - def test_checkpoint__lock_config(self): - # Arrange - config_lock_decorator = self.__create_config_lock_decorator() - - # Act - config_lock_decorator.checkpoint(self.any_checkpoint_name) - - # Assert - config_lock_decorator.configlock.acquire_lock.assert_called_once() - config_lock_decorator.decorated_config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) - config_lock_decorator.configlock.release_lock.assert_called_once() - - def __create_config_lock_decorator(self): - configlock = Mock() - - patchapplier = Mock() - patchapplier.apply.side_effect = create_side_effect_dict({(str(SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) - - config_replacer = Mock() - config_replacer.replace.side_effect = create_side_effect_dict({(str(SONIC_YANG_AS_JSON),): 0}) - - config_rollbacker = Mock() - config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - - return generic_update.ConfigLockDecorator( \ - configlock=configlock, \ - decorated_patch_applier=patchapplier, \ - decorated_config_replacer=config_replacer, \ - decorated_config_rollbacker=config_rollbacker) - -#### resources #### -CONFIG_DB_AS_DICT = { - "VLAN_MEMBER": { - ("Vlan1000", "Ethernet0"): { - "tagging_mode": "untagged" - }, - ("Vlan1000", "Ethernet4"): { - "tagging_mode": "untagged" - }, - ("Vlan1000", "Ethernet8"): { - "tagging_mode": "untagged" - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "ACL_TABLE": { - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - }, - "DATAACL": { - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - "EVERFLOW": { - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - "EVERFLOWV6": { - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - }, - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "PORT": { - "Ethernet0": { - "alias": "Eth1", - "lanes": "65, 66, 67, 68", - "description": "Ethernet0 100G link", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": "1", - "lanes": "29,30,31,32", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": "2", - "lanes": "33,34,35,36", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - } - }, - "WRED_PROFILE": { - "AZURE_LOSSLESS": { - "wred_green_enable": "true", - "wred_yellow_enable": "true", - "wred_red_enable": "true", - "ecn": "ecn_all", - "green_max_threshold": "2097152", - "green_min_threshold": "1048576", - "yellow_max_threshold": "2097152", - "yellow_min_threshold": "1048576", - "red_max_threshold": "2097152", - "red_min_threshold": "1048576", - "green_drop_probability": "5", - "yellow_drop_probability": "5", - "red_drop_probability": "5" - } - } -} - -CONFIG_DB_AS_JSON = { - "VLAN_MEMBER": { - "Vlan1000|Ethernet0": { - "tagging_mode": "untagged" - }, - "Vlan1000|Ethernet4": { - "tagging_mode": "untagged" - }, - "Vlan1000|Ethernet8": { - "tagging_mode": "untagged" - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "ACL_TABLE": { - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - }, - "DATAACL": { - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - "EVERFLOW": { - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - "EVERFLOWV6": { - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - } - }, - "PORT": { - "Ethernet0": { - "alias": "Eth1", - "lanes": "65, 66, 67, 68", - "description": "Ethernet0 100G link", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": "1", - "lanes": "29,30,31,32", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": "2", - "lanes": "33,34,35,36", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - } - }, - "WRED_PROFILE": { - "AZURE_LOSSLESS": { - "wred_green_enable": "true", - "wred_yellow_enable": "true", - "wred_red_enable": "true", - "ecn": "ecn_all", - "green_max_threshold": "2097152", - "green_min_threshold": "1048576", - "yellow_max_threshold": "2097152", - "yellow_min_threshold": "1048576", - "red_max_threshold": "2097152", - "red_min_threshold": "1048576", - "green_drop_probability": "5", - "yellow_drop_probability": "5", - "red_drop_probability": "5" - } - } -} - -CROPPED_CONFIG_DB_AS_JSON = { - "VLAN_MEMBER": { - "Vlan1000|Ethernet0": { - "tagging_mode": "untagged" - }, - "Vlan1000|Ethernet4": { - "tagging_mode": "untagged" - }, - "Vlan1000|Ethernet8": { - "tagging_mode": "untagged" - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "ACL_TABLE": { - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - }, - "DATAACL": { - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - "EVERFLOW": { - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - "EVERFLOWV6": { - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - } - }, - "PORT": { - "Ethernet0": { - "alias": "Eth1", - "lanes": "65, 66, 67, 68", - "description": "Ethernet0 100G link", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": "1", - "lanes": "29,30,31,32", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": "2", - "lanes": "33,34,35,36", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - } - } -} - -SONIC_YANG_AS_JSON = { - "sonic-vlan:sonic-vlan": { - "sonic-vlan:VLAN_MEMBER": { - "VLAN_MEMBER_LIST": [ - { - "vlan_name": "Vlan1000", - "port": "Ethernet0", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan1000", - "port": "Ethernet4", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan1000", - "port": "Ethernet8", - "tagging_mode": "untagged" - } - ] - }, - "sonic-vlan:VLAN": { - "VLAN_LIST": [ - { - "vlan_name": "Vlan1000", - "vlanid": 1000, - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - ] - } - }, - "sonic-acl:sonic-acl": { - "sonic-acl:ACL_TABLE": { - "ACL_TABLE_LIST": [ - { - "ACL_TABLE_NAME": "NO-NSW-PACL-V4", - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - }, - { - "ACL_TABLE_NAME": "DATAACL", - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - { - "ACL_TABLE_NAME": "EVERFLOW", - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - { - "ACL_TABLE_NAME": "EVERFLOWV6", - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - } - ] - } - }, - "sonic-port:sonic-port": { - "sonic-port:PORT": { - "PORT_LIST": [ - { - "port_name": "Ethernet0", - "alias": "Eth1", - "lanes": "65, 66, 67, 68", - "description": "Ethernet0 100G link", - "speed": 100000 - }, - { - "port_name": "Ethernet4", - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": 1, - "lanes": "29,30,31,32", - "mtu": 9100, - "pfc_asym": "off", - "speed": 40000 - }, - { - "port_name": "Ethernet8", - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": 2, - "lanes": "33,34,35,36", - "mtu": 9100, - "pfc_asym": "off", - "speed": 40000 - } - ] - } - } -} - -SONIC_YANG_AS_JSON_INVALID = { - "sonic-vlan:sonic-vlan": { - "sonic-vlan:VLAN_MEMBER": { - "VLAN_MEMBER_LIST": [ - { - "vlan_name": "Vlan1000", - "port": "Ethernet4", - "tagging_mode": "untagged" - } - ] - } - } -} - -SINGLE_OPERATION_CONFIG_DB_PATCH = jsonpatch.JsonPatch([ - { - "op": "remove", - "path": "/VLAN_MEMBER/Vlan1000|Ethernet8" - } -]) - -SINGLE_OPERATION_SONIC_YANG_PATCH = jsonpatch.JsonPatch([ - { - "op": "remove", - "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/2" - } -]) - -MULTI_OPERATION_CONFIG_DB_PATCH = jsonpatch.JsonPatch([ - { - "op": "add", - "path": "/PORT/Ethernet3", - "value": { - "alias": "Eth1/4", - "lanes": "68", - "description": "", - "speed": "10000" - } - }, - { - "op": "add", - "path": "/PORT/Ethernet1", - "value": { - "alias": "Eth1/2", - "lanes": "66", - "description": "", - "speed": "10000" - } - }, - { - "op": "add", - "path": "/PORT/Ethernet2", - "value": { - "alias": "Eth1/3", - "lanes": "67", - "description": "", - "speed": "10000" - } - }, - { - "op": "replace", - "path": "/PORT/Ethernet0/lanes", - "value": "65" - }, - { - "op": "replace", - "path": "/PORT/Ethernet0/alias", - "value": "Eth1/1" - }, - { - "op": "replace", - "path": "/PORT/Ethernet0/description", - "value": "" - }, - { - "op": "replace", - "path": "/PORT/Ethernet0/speed", - "value": "10000" - }, - { - "op": "add", - "path": "/VLAN_MEMBER/Vlan100|Ethernet2", - "value": { - "tagging_mode": "untagged" - } - }, - { - "op": "add", - "path": "/VLAN_MEMBER/Vlan100|Ethernet3", - "value": { - "tagging_mode": "untagged" - } - }, - { - "op": "add", - "path": "/VLAN_MEMBER/Vlan100|Ethernet1", - "value": { - "tagging_mode": "untagged" - } - }, - { - "op": "add", - "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/1", - "value": "Ethernet1" - }, - { - "op": "add", - "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/2", - "value": "Ethernet2" - }, - { - "op": "add", - "path": "/ACL_TABLE/NO-NSW-PACL-V4/ports/3", - "value": "Ethernet3" - } -]) - -MULTI_OPERATION_SONIC_YANG_PATCH = jsonpatch.JsonPatch([ - { - "op": "add", - "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/3", - "value": { - "vlan_name": "Vlan100", - "port": "Ethernet2", - "tagging_mode": "untagged" - } - }, - { - "op": "add", - "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/4", - "value": { - "vlan_name": "Vlan100", - "port": "Ethernet3", - "tagging_mode": "untagged" - } - }, - { - "op": "add", - "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/5", - "value": { - "vlan_name": "Vlan100", - "port": "Ethernet1", - "tagging_mode": "untagged" - } - }, - { - "op": "replace", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/lanes", - "value": "65" - }, - { - "op": "replace", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/alias", - "value": "Eth1/1" - }, - { - "op": "replace", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/speed", - "value": 10000 - }, - { - "op": "replace", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/0/description", - "value": "" - }, - { - "op": "add", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/3", - "value": { - "port_name": "Ethernet3", - "alias": "Eth1/4", - "lanes": "68", - "description": "", - "speed": 10000 - } - }, - { - "op": "add", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/4", - "value": { - "port_name": "Ethernet1", - "alias": "Eth1/2", - "lanes": "66", - "description": "", - "speed": 10000 - } - }, - { - "op": "add", - "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/5", - "value": { - "port_name": "Ethernet2", - "alias": "Eth1/3", - "lanes": "67", - "description": "", - "speed": 10000 - } - }, - { - "op": "add", - "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/1", - "value": "Ethernet1" - }, - { - "op": "add", - "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/2", - "value": "Ethernet2" - }, - { - "op": "add", - "path": "/sonic-acl:sonic-acl/sonic-acl:ACL_TABLE/ACL_TABLE_LIST/0/ports/3", - "value": "Ethernet3" - } -]) - -SONIC_YANG_AFTER_MULTI_PATCH = { - "sonic-vlan:sonic-vlan": { - "sonic-vlan:VLAN_MEMBER": { - "VLAN_MEMBER_LIST": [ - { - "vlan_name": "Vlan1000", - "port": "Ethernet0", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan1000", - "port": "Ethernet4", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan1000", - "port": "Ethernet8", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan100", - "port": "Ethernet2", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan100", - "port": "Ethernet3", - "tagging_mode": "untagged" - }, - { - "vlan_name": "Vlan100", - "port": "Ethernet1", - "tagging_mode": "untagged" - } - ] - }, - "sonic-vlan:VLAN": { - "VLAN_LIST": [ - { - "vlan_name": "Vlan1000", - "vlanid": 1000, - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - ] - } - }, - "sonic-acl:sonic-acl": { - "sonic-acl:ACL_TABLE": { - "ACL_TABLE_LIST": [ - { - "ACL_TABLE_NAME": "NO-NSW-PACL-V4", - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0", - "Ethernet1", - "Ethernet2", - "Ethernet3" - ] - }, - { - "ACL_TABLE_NAME": "DATAACL", - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - { - "ACL_TABLE_NAME": "EVERFLOW", - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - { - "ACL_TABLE_NAME": "EVERFLOWV6", - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - } - ] - } - }, - "sonic-port:sonic-port": { - "sonic-port:PORT": { - "PORT_LIST": [ - { - "port_name": "Ethernet0", - "alias": "Eth1/1", - "lanes": "65", - "description": "", - "speed": 10000 - }, - { - "port_name": "Ethernet4", - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": 1, - "lanes": "29,30,31,32", - "mtu": 9100, - "pfc_asym": "off", - "speed": 40000 - }, - { - "port_name": "Ethernet8", - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": 2, - "lanes": "33,34,35,36", - "mtu": 9100, - "pfc_asym": "off", - "speed": 40000 - }, - { - "port_name": "Ethernet3", - "alias": "Eth1/4", - "lanes": "68", - "description": "", - "speed": 10000 - }, - { - "port_name": "Ethernet1", - "alias": "Eth1/2", - "lanes": "66", - "description": "", - "speed": 10000 - }, - { - "port_name": "Ethernet2", - "alias": "Eth1/3", - "lanes": "67", - "description": "", - "speed": 10000 - } - ] - } - } -} From 93e0347f685ad22feac9e135ee7e2908ae2dc67f Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 6 Apr 2021 09:14:38 -0700 Subject: [PATCH 05/32] fixing compilation errors/warnings --- generic_config_updater/generic_updater.py | 58 +++++++++---- .../files/config_db_as_dict.py-dict | 2 +- .../generic_updater_test.py | 85 +++++++++++++++++-- 3 files changed, 120 insertions(+), 25 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index ab641de6b8..a3b62a17e1 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -13,7 +13,7 @@ class JsonChange: # TODO: Implement JsonChange pass -class ConfigLock(): +class ConfigLock: def acquire_lock(self): # TODO: Implement ConfigLock pass @@ -121,7 +121,7 @@ class DryRunConfigWrapper(ConfigWrapper): # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. pass -class PatchWrapper(): +class PatchWrapper: def __init__(self, config_wrapper = ConfigWrapper()): self.config_wrapper = config_wrapper @@ -293,44 +293,72 @@ def __delete_checkpoint(self, name): path = self.__get_checkpoint_full_path(name) return os.remove(path) -class ConfigDbDecorator(PatchApplier, ConfigReplacer): - def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None): +class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): + def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None): + # initing base classes to make LGTM happy + PatchApplier.__init__(self) + ConfigReplacer.__init__(self) + FileSystemConfigRollbacker.__init__(self) + self.decorated_patch_applier = decorated_patch_applier self.decorated_config_replacer = decorated_config_replacer + self.decorated_config_rollbacker = decorated_config_rollbacker + + def apply(self, patch): + self.decorated_patch_applier.apply(patch) + + def replace(self, target_config): + self.decorated_config_replacer.replace(target_config) + + def rollback(self, checkpoint_name): + self.decorated_config_rollbacker.rollback(checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.checkpoint(checkpoint_name) + + def list_checkpoints(self): + return self.decorated_config_rollbacker.list_checkpoints() + + def delete_checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) + +class ConfigDbDecorator(Decorator): + def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer) + self.patch_wrapper = patch_wrapper self.config_wrapper = config_wrapper def apply(self, patch): yang_patch = self.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - self.decorated_patch_applier.apply(yang_patch) + Decorator.apply(self, yang_patch) def replace(self, target_config): yang_target_config = self.config_wrapper.convert_config_db_to_sonic_yang(target_config) - self.decorated_config_replacer.replace(yang_target_config) + Decorator.replace(self, yang_target_config) -class ConfigLockDecorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): +class ConfigLockDecorator(Decorator): def __init__( \ self, \ decorated_patch_applier=None, \ decorated_config_replacer=None, \ decorated_config_rollbacker=None, \ config_lock = ConfigLock()): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker) + self.config_lock = config_lock - self.decorated_patch_applier = decorated_patch_applier - self.decorated_config_replacer = decorated_config_replacer - self.decorated_config_rollbacker = decorated_config_rollbacker def apply(self, patch): - self.execute_write_action(self.decorated_patch_applier.apply, patch) - + self.execute_write_action(Decorator.apply, self, patch) + def replace(self, target_config): - self.execute_write_action(self.decorated_config_replacer.replace, target_config) + self.execute_write_action(Decorator.replace, self, target_config) def rollback(self, checkpoint_name): - self.execute_write_action(self.decorated_config_rollbacker.rollback, checkpoint_name) + self.execute_write_action(Decorator.rollback, self, checkpoint_name) def checkpoint(self, checkpoint_name): - self.execute_write_action(self.decorated_config_rollbacker.checkpoint, checkpoint_name) + self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) def execute_write_action(self, action, *args): self.config_lock.acquire_lock() diff --git a/tests/generic_config_updater/files/config_db_as_dict.py-dict b/tests/generic_config_updater/files/config_db_as_dict.py-dict index 9d76c2fc12..f83ee09f4f 100644 --- a/tests/generic_config_updater/files/config_db_as_dict.py-dict +++ b/tests/generic_config_updater/files/config_db_as_dict.py-dict @@ -118,4 +118,4 @@ "red_drop_probability": "5" } } -} \ No newline at end of file +} diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 63e606947d..cb53573ac9 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -26,6 +26,9 @@ def create_side_effect_dict(map): return MockSideEffectDict(map).side_effect_func class FilesLoader: + def __init__(self): + self.test_path = os.path.dirname(os.path.abspath(__file__)) + def __getattr__(self, attr): return self.__load(attr) @@ -33,21 +36,21 @@ def __load(self, file_name): normalized_file_name = file_name.lower() # Try load dict file - json_file_path = os.path.join("files", f"{normalized_file_name}.py-dict") + json_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.py-dict") if os.path.isfile(json_file_path): with open(json_file_path) as fh: text = fh.read() return eval(text) # Try load json file - json_file_path = os.path.join("files", f"{normalized_file_name}.json") + json_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.json") if os.path.isfile(json_file_path): with open(json_file_path) as fh: text = fh.read() return json.loads(text) # Try load json-patch file - jsonpatch_file_path = os.path.join("files", f"{normalized_file_name}.json-patch") + jsonpatch_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.json-patch") if os.path.isfile(jsonpatch_file_path): with open(jsonpatch_file_path) as fh: text = fh.read() @@ -872,18 +875,79 @@ def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): # Assert self.assertListEqual(expected, actual) +class TestDecorator(unittest.TestCase): + def setUp(self): + self.decorated_patch_applier = Mock() + self.decorated_config_replacer = Mock() + self.decorated_config_rollbacker = Mock() + + self.any_checkpoint_name = "anycheckpoint" + self.any_other_checkpoint_name = "anyothercheckpoint" + self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] + self.decorated_config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list + + self.decorator = gu.Decorator( \ + self.decorated_patch_applier, self.decorated_config_replacer, self.decorated_config_rollbacker) + + def test_apply__calls_decorated_applier(self): + # Act + self.decorator.apply(Files.SINGLE_OPERATION_SONIC_YANG_PATCH) + + # Assert + self.decorated_patch_applier.apply.assert_has_calls([call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) + + def test_replace__calls_decorated_replacer(self): + # Act + self.decorator.replace(Files.SONIC_YANG_AS_JSON) + + # Assert + self.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + + def test_rollback__calls_decorated_rollbacker(self): + # Act + self.decorator.rollback(self.any_checkpoint_name) + + # Assert + self.decorated_config_rollbacker.rollback.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_checkpoint__calls_decorated_rollbacker(self): + # Act + self.decorator.checkpoint(self.any_checkpoint_name) + + # Assert + self.decorated_config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_delete_checkpoint__calls_decorated_rollbacker(self): + # Act + self.decorator.delete_checkpoint(self.any_checkpoint_name) + + # Assert + self.decorated_config_rollbacker.delete_checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) + + def test_list_checkpoints__calls_decorated_rollbacker(self): + # Arrange + expected = self.any_checkpoints_list + + # Act + actual = self.decorator.list_checkpoints() + + # Assert + self.decorated_config_rollbacker.list_checkpoints.assert_called_once() + self.assertListEqual(expected, actual) + class TestConfigDbDecorator(unittest.TestCase): def test_apply__converts_to_yang_and_calls_decorated_class(self): # Arrange config_db_decorator = self.__create_config_db_decorator() # Act - config_db_decorator.apply(Files.CONFIG_DB_AS_JSON) + config_db_decorator.apply(Files.SINGLE_OPERATION_CONFIG_DB_PATCH) # Assert config_db_decorator.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( \ - [call(Files.CONFIG_DB_AS_JSON)]) - config_db_decorator.decorated_patch_applier.apply.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + [call(Files.SINGLE_OPERATION_CONFIG_DB_PATCH)]) + config_db_decorator.decorated_patch_applier.apply.assert_has_calls( \ + [call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) def test_replace__converts_to_yang_and_calls_decorated_class(self): # Arrange @@ -896,14 +960,15 @@ def test_replace__converts_to_yang_and_calls_decorated_class(self): config_db_decorator.config_wrapper.convert_config_db_to_sonic_yang.assert_has_calls( \ [call(Files.CONFIG_DB_AS_JSON)]) config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) - + def __create_config_db_decorator(self): patch_applier = Mock() - patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) patch_wrapper = Mock() patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.side_effect = \ - create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): Files.SONIC_YANG_AS_JSON}) + create_side_effect_dict({(str(Files.SINGLE_OPERATION_CONFIG_DB_PATCH),): \ + Files.SINGLE_OPERATION_SONIC_YANG_PATCH}) config_replacer = Mock() config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) @@ -984,6 +1049,8 @@ def __create_config_lock_decorator(self): config_rollbacker.rollback.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) config_rollbacker.checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + config_rollbacker.delete_checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) + return gu.ConfigLockDecorator( \ config_lock=config_lock, \ decorated_patch_applier=patch_applier, \ From 2852f9b5ee747b6fee4a47b897407c63b1af2428 Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 6 Apr 2021 09:28:09 -0700 Subject: [PATCH 06/32] fixing lgtm warnings --- generic_config_updater/generic_updater.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index a3b62a17e1..a5416388da 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -23,12 +23,12 @@ def release_lock(self): pass class PatchOrderer: - def order(patch): + def order(self, patch): # TODO: Implement patch orderer pass class ChangeApplier: - def apply(change): + def apply(self, change): # TODO: Implement change applier pass @@ -111,7 +111,6 @@ def __create_and_connect_config_db(self): return self.default_config_db_connector db_kwargs = dict() - data = dict() config_db = ConfigDBConnector(**db_kwargs) config_db.connect() return config_db From 4399fde93031f1c4b4ca434e08ac8cecbb221dff Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 6 Apr 2021 12:43:16 -0700 Subject: [PATCH 07/32] fixing compliation errors according to recent yang models --- generic_config_updater/generic_updater.py | 7 ++--- ...ulti_operation_sonic_yang_patch.json-patch | 12 ++++----- .../files/sonic_yang_after_multi_patch.json | 26 +++++++++---------- .../files/sonic_yang_as_json.json | 14 +++++----- .../files/sonic_yang_as_json_invalid.json | 2 +- .../generic_updater_test.py | 2 +- 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index a5416388da..b10e640559 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -9,6 +9,10 @@ load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter +YANG_DIR = "/usr/local/yang-models" +CHECKPOINTS_DIR = "/etc/sonic/checkpoints" +CHECKPOINT_EXT = ".cp.json" + class JsonChange: # TODO: Implement JsonChange pass @@ -32,7 +36,6 @@ def apply(self, change): # TODO: Implement change applier pass -YANG_DIR = "/usr/local/yang-models" class ConfigWrapper: def __init__(self, default_config_db_connector = None, yang_dir = YANG_DIR): self.default_config_db_connector = default_config_db_connector @@ -221,8 +224,6 @@ def replace(self, target_config): if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise Exception(f"After applying patch to config, there is still some parts not updated") -CHECKPOINTS_DIR = "/etc/sonic/checkpoints" -CHECKPOINT_EXT = ".cp.json" class FileSystemConfigRollbacker: def __init__( \ self, \ diff --git a/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch b/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch index 8c33c554ae..f7005bb4a0 100644 --- a/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch +++ b/tests/generic_config_updater/files/multi_operation_sonic_yang_patch.json-patch @@ -3,7 +3,7 @@ "op": "add", "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/3", "value": { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet2", "tagging_mode": "untagged" } @@ -12,7 +12,7 @@ "op": "add", "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/4", "value": { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet3", "tagging_mode": "untagged" } @@ -21,7 +21,7 @@ "op": "add", "path": "/sonic-vlan:sonic-vlan/sonic-vlan:VLAN_MEMBER/VLAN_MEMBER_LIST/5", "value": { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet1", "tagging_mode": "untagged" } @@ -50,7 +50,7 @@ "op": "add", "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/3", "value": { - "port_name": "Ethernet3", + "name": "Ethernet3", "alias": "Eth1/4", "lanes": "68", "description": "", @@ -61,7 +61,7 @@ "op": "add", "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/4", "value": { - "port_name": "Ethernet1", + "name": "Ethernet1", "alias": "Eth1/2", "lanes": "66", "description": "", @@ -72,7 +72,7 @@ "op": "add", "path": "/sonic-port:sonic-port/sonic-port:PORT/PORT_LIST/5", "value": { - "port_name": "Ethernet2", + "name": "Ethernet2", "alias": "Eth1/3", "lanes": "67", "description": "", diff --git a/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json b/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json index 05e0379dcc..0c9ddd4546 100644 --- a/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json +++ b/tests/generic_config_updater/files/sonic_yang_after_multi_patch.json @@ -3,32 +3,32 @@ "sonic-vlan:VLAN_MEMBER": { "VLAN_MEMBER_LIST": [ { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet0", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet4", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet8", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet2", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet3", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan100", + "name": "Vlan100", "port": "Ethernet1", "tagging_mode": "untagged" } @@ -37,7 +37,7 @@ "sonic-vlan:VLAN": { "VLAN_LIST": [ { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "vlanid": 1000, "dhcp_servers": [ "192.0.0.1", @@ -98,14 +98,14 @@ "sonic-port:PORT": { "PORT_LIST": [ { - "port_name": "Ethernet0", + "name": "Ethernet0", "alias": "Eth1/1", "lanes": "65", "description": "", "speed": 10000 }, { - "port_name": "Ethernet4", + "name": "Ethernet4", "admin_status": "up", "alias": "fortyGigE0/4", "description": "Servers0:eth0", @@ -116,7 +116,7 @@ "speed": 40000 }, { - "port_name": "Ethernet8", + "name": "Ethernet8", "admin_status": "up", "alias": "fortyGigE0/8", "description": "Servers1:eth0", @@ -127,21 +127,21 @@ "speed": 40000 }, { - "port_name": "Ethernet3", + "name": "Ethernet3", "alias": "Eth1/4", "lanes": "68", "description": "", "speed": 10000 }, { - "port_name": "Ethernet1", + "name": "Ethernet1", "alias": "Eth1/2", "lanes": "66", "description": "", "speed": 10000 }, { - "port_name": "Ethernet2", + "name": "Ethernet2", "alias": "Eth1/3", "lanes": "67", "description": "", diff --git a/tests/generic_config_updater/files/sonic_yang_as_json.json b/tests/generic_config_updater/files/sonic_yang_as_json.json index c921fa5384..37f0fe6ba7 100644 --- a/tests/generic_config_updater/files/sonic_yang_as_json.json +++ b/tests/generic_config_updater/files/sonic_yang_as_json.json @@ -3,17 +3,17 @@ "sonic-vlan:VLAN_MEMBER": { "VLAN_MEMBER_LIST": [ { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet0", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet4", "tagging_mode": "untagged" }, { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet8", "tagging_mode": "untagged" } @@ -22,7 +22,7 @@ "sonic-vlan:VLAN": { "VLAN_LIST": [ { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "vlanid": 1000, "dhcp_servers": [ "192.0.0.1", @@ -80,14 +80,14 @@ "sonic-port:PORT": { "PORT_LIST": [ { - "port_name": "Ethernet0", + "name": "Ethernet0", "alias": "Eth1", "lanes": "65, 66, 67, 68", "description": "Ethernet0 100G link", "speed": 100000 }, { - "port_name": "Ethernet4", + "name": "Ethernet4", "admin_status": "up", "alias": "fortyGigE0/4", "description": "Servers0:eth0", @@ -98,7 +98,7 @@ "speed": 40000 }, { - "port_name": "Ethernet8", + "name": "Ethernet8", "admin_status": "up", "alias": "fortyGigE0/8", "description": "Servers1:eth0", diff --git a/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json b/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json index 7a8c37f7a2..4f67d7e6a6 100644 --- a/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json +++ b/tests/generic_config_updater/files/sonic_yang_as_json_invalid.json @@ -3,7 +3,7 @@ "sonic-vlan:VLAN_MEMBER": { "VLAN_MEMBER_LIST": [ { - "vlan_name": "Vlan1000", + "name": "Vlan1000", "port": "Ethernet4", "tagging_mode": "untagged" } diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index cb53573ac9..8448869a2b 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -17,7 +17,7 @@ def side_effect_func(self, *args): l = [str(arg) for arg in args] key = tuple(l) value = self.map.get(key) - if value == None: + if value is None: raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") return value From b9c81b8ed0d5e6dc743873726687ddaea8729f44 Mon Sep 17 00:00:00 2001 From: ghooo Date: Tue, 6 Apr 2021 14:54:11 -0700 Subject: [PATCH 08/32] using assertCountEqual instead of assertListEqual --- tests/generic_config_updater/generic_updater_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 8448869a2b..ee1f3df30f 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -554,7 +554,8 @@ def test_list_checkpoints__checkpoints_names_have_special_characters__multiple_f actual = rollbacker.list_checkpoints() # Assert - self.assertListEqual(expected, actual) + # 'assertCountEqual' does check same count, same elements ignoring order + self.assertCountEqual(expected, actual) def test_delete_checkpoint__checkpoint_does_not_exist__failure(self): # Arrange From e72407b62d3f1c4bdfaf34925f25a546340abc0c Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 14 Apr 2021 07:07:35 -0700 Subject: [PATCH 09/32] catching/raising non-general exceptions --- generic_config_updater/generic_updater.py | 169 +++++++++--------- ...c_yang_as_json_with_unexpected_colons.json | 114 ++++++++++++ .../sonic_yang_as_json_without_colons.json | 114 ++++++++++++ .../generic_updater_test.py | 128 +++++++------ 4 files changed, 390 insertions(+), 135 deletions(-) create mode 100644 tests/generic_config_updater/files/sonic_yang_as_json_with_unexpected_colons.json create mode 100644 tests/generic_config_updater/files/sonic_yang_as_json_without_colons.json diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index b10e640559..c75021f84c 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -13,6 +13,9 @@ CHECKPOINTS_DIR = "/etc/sonic/checkpoints" CHECKPOINT_EXT = ".cp.json" +class ConfigNotCompletelyUpdatedError(Exception): + pass + class JsonChange: # TODO: Implement JsonChange pass @@ -26,9 +29,9 @@ def release_lock(self): # TODO: Implement ConfigLock pass -class PatchOrderer: +class PatchSorter: def order(self, patch): - # TODO: Implement patch orderer + # TODO: Implement patch sorter pass class ChangeApplier: @@ -42,7 +45,7 @@ def __init__(self, default_config_db_connector = None, yang_dir = YANG_DIR): self.yang_dir = YANG_DIR def get_config_db_as_json(self): - config_db = self.__create_and_connect_config_db() + config_db = self._create_and_connect_config_db() data = dict() deep_update(data, FormatConverter.db_to_output(config_db.get_config())) return FormatConverter.to_serialized(data) @@ -70,10 +73,13 @@ def convert_sonic_yang_to_config_db(self, sonic_yang_as_json): # replace container of the format 'module:table' with just 'table' new_sonic_yang_json = {} - for module_top in sonic_yang_as_json.keys(): + for module_top in sonic_yang_as_json: new_sonic_yang_json[module_top] = {} - for container in sonic_yang_as_json[module_top].keys(): - table = container.split(':')[1] + for container in sonic_yang_as_json[module_top]: + tokens = container.split(':') + if len(tokens) > 2: + raise ValueError(f"Expecting ':' or '
', found {container}") + table = container if len(tokens) == 1 else tokens[1] new_sonic_yang_json[module_top][table] = sonic_yang_as_json[module_top][container] config_db_as_json = dict() @@ -94,7 +100,7 @@ def validate_sonic_yang_config(self, sonic_yang_as_json): sy.validate_data_tree() return True - except Exception as ex: + except sonic_yang.SonicYangException as ex: return False def crop_tables_without_yang(self, config_db_as_json): @@ -109,12 +115,12 @@ def crop_tables_without_yang(self, config_db_as_json): return sy.jIn - def __create_and_connect_config_db(self): + def _create_and_connect_config_db(self): if self.default_config_db_connector != None: return self.default_config_db_connector db_kwargs = dict() - config_db = ConfigDBConnector(**db_kwargs) + config_db = ConfigDBConnector() config_db.connect() return config_db @@ -124,8 +130,8 @@ class DryRunConfigWrapper(ConfigWrapper): pass class PatchWrapper: - def __init__(self, config_wrapper = ConfigWrapper()): - self.config_wrapper = config_wrapper + def __init__(self, config_wrapper=None): + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() def validate_config_db_patch(self, patch): config_db = {} @@ -135,7 +141,7 @@ def validate_config_db_patch(self, patch): tables_dict = {table_name: {} for table_name in operation['value']} config_db.update(tables_dict) elif not tokens[0]: # Not empty - raise Exception("Table name in patch cannot be empty") + raise ValueError("Table name in patch cannot be empty") else: config_db[tokens[0]] = {} @@ -156,7 +162,7 @@ def simulate_patch(self, patch, jsonconfig): def convert_config_db_patch_to_sonic_yang_patch(self, patch): if not(self.validate_config_db_patch(patch)): - raise Exception(f"Given patch is not valid") + raise ValueError(f"Given patch is not valid") current_config_db = self.config_wrapper.get_config_db_as_json() target_config_db = self.simulate_patch(patch, current_config_db) @@ -171,49 +177,48 @@ class ConfigFormat(Enum): CONFIGDB = 2 class PatchApplier: - def __init__( \ - self, \ - patchorderer = PatchOrderer(), \ - changeapplier = ChangeApplier(), \ - config_wrapper = ConfigWrapper(), \ - patch_wrapper = PatchWrapper()): - self.patchorderer = patchorderer - self.changeapplier = changeapplier - self.config_wrapper = config_wrapper - self.patch_wrapper = patch_wrapper + def __init__(self, + patchsorter=None, + changeapplier=None, + config_wrapper=None, + patch_wrapper=None): + self.patchsorter = patchsorter if patchsorter is not None else PatchSorter() + self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier() + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def apply(self, patch): - # 1. Get old config as SONiC Yang + # Get old config as SONiC Yang old_config = self.config_wrapper.get_sonic_yang_as_json() - # 2. Generate target config + # Generate target config target_config = self.patch_wrapper.simulate_patch(patch, old_config) - # 3. Validate target config + # Validate target config if not(self.config_wrapper.validate_sonic_yang_config(target_config)): - raise Exception(f"The given patch is not valid") + raise ValueError(f"The given patch is not valid") - # 4. Generate list of changes to apply - changes = self.patchorderer.order(patch) + # Generate list of changes to apply + changes = self.patchsorter.order(patch) - # 5. Apply changes in order + # Apply changes in order for change in changes: self.changeapplier.apply(change) - # 6. Validate config updated successfully + # Validate config updated successfully new_config = self.config_wrapper.get_sonic_yang_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise Exception(f"After applying patch to config, there are still some parts not updated") + raise ConfigNotCompletelyUpdatedError(f"After applying patch to config, there are still some parts not updated") class ConfigReplacer: - def __init__(self, patch_applier = PatchApplier(), config_wrapper = ConfigWrapper(), patch_wrapper = PatchWrapper()): - self.patch_applier = patch_applier - self.config_wrapper = config_wrapper - self.patch_wrapper = patch_wrapper + def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None): + self.patch_applier = patch_applier if patch_applier is not None else PatchApplier() + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def replace(self, target_config): if not(self.config_wrapper.validate_sonic_yang_config(target_config)): - raise Exception(f"The given target config is not valid") + raise ValueError(f"The given target config is not valid") old_config = self.config_wrapper.get_sonic_yang_as_json() patch = self.patch_wrapper.generate_patch(old_config, target_config) @@ -222,75 +227,81 @@ def replace(self, target_config): new_config = self.config_wrapper.get_sonic_yang_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise Exception(f"After applying patch to config, there is still some parts not updated") + raise ConfigNotCompletelyUpdatedError(f"After replacing config, there is still some parts not updated") class FileSystemConfigRollbacker: - def __init__( \ - self, \ - checkpoints_dir = CHECKPOINTS_DIR, \ - config_replacer = ConfigReplacer(), \ - config_wrapper = ConfigWrapper()): + def __init__(self, + checkpoints_dir=CHECKPOINTS_DIR, + config_replacer=None, + config_wrapper=None): self.checkpoints_dir = checkpoints_dir - self.config_replacer = config_replacer - self.config_wrapper = config_wrapper + self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer() + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() def rollback(self, checkpoint_name): - if not self.__check_checkpoint_exists(checkpoint_name): - raise Exception(f"Checkpoint '{checkpoint_name}' does not exist") + if not self._check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - target_config = self.__get_checkpoint_content(checkpoint_name) + target_config = self._get_checkpoint_content(checkpoint_name) self.config_replacer.replace(target_config) def checkpoint(self, checkpoint_name): json_content = self.config_wrapper.get_sonic_yang_as_json() - path = self.__get_checkpoint_full_path(checkpoint_name) + path = self._get_checkpoint_full_path(checkpoint_name) - self.__ensure_checkpoints_dir_exists() + self._ensure_checkpoints_dir_exists() - self.__save_json_file(path, json_content) + self._save_json_file(path, json_content) def list_checkpoints(self): - if not self.__checkpoints_dir_exist(): + if not self._checkpoints_dir_exist(): return [] - return self.__get_checkpoint_names() + return self._get_checkpoint_names() def delete_checkpoint(self, checkpoint_name): - if not self.__check_checkpoint_exists(checkpoint_name): - raise Exception("Checkpoint does not exist") + if not self._check_checkpoint_exists(checkpoint_name): + raise ValueError("Checkpoint does not exist") - self.__delete_checkpoint(checkpoint_name) + self._delete_checkpoint(checkpoint_name) - def __ensure_checkpoints_dir_exists(self): + def _ensure_checkpoints_dir_exists(self): os.makedirs(self.checkpoints_dir, exist_ok=True) - def __save_json_file(self, path, json_content): + def _save_json_file(self, path, json_content): with open(path, "w") as fh: fh.write(json.dumps(json_content)) - def __get_checkpoint_content(self, checkpoint_name): - path = self.__get_checkpoint_full_path(checkpoint_name) + def _get_checkpoint_content(self, checkpoint_name): + path = self._get_checkpoint_full_path(checkpoint_name) with open(path) as fh: text = fh.read() return json.loads(text) - def __get_checkpoint_full_path(self, name): + def _get_checkpoint_full_path(self, name): return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") - def __get_checkpoint_names(self): - return [f[:-len(CHECKPOINT_EXT)] for f in os.listdir(self.checkpoints_dir) if f.endswith(CHECKPOINT_EXT)] + def _get_checkpoint_names(self): + file_names = [] + for file_name in os.listdir(self.checkpoints_dir): + if file_name.endswith(CHECKPOINT_EXT): + # Remove extension from file name. + # Example assuming ext is '.cp.json', then 'checkpoint1.cp.json' becomes 'checkpoint1' + file_names.append(file_name[:-len(CHECKPOINT_EXT)]) + + return file_names - def __checkpoints_dir_exist(self): + def _checkpoints_dir_exist(self): return os.path.isdir(self.checkpoints_dir) - def __check_checkpoint_exists(self, name): - path = self.__get_checkpoint_full_path(name) + def _check_checkpoint_exists(self, name): + path = self._get_checkpoint_full_path(name) return os.path.isfile(path) - def __delete_checkpoint(self, name): - path = self.__get_checkpoint_full_path(name) + def _delete_checkpoint(self, name): + path = self._get_checkpoint_full_path(name) return os.remove(path) class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): @@ -338,12 +349,11 @@ def replace(self, target_config): Decorator.replace(self, yang_target_config) class ConfigLockDecorator(Decorator): - def __init__( \ - self, \ - decorated_patch_applier=None, \ - decorated_config_replacer=None, \ - decorated_config_rollbacker=None, \ - config_lock = ConfigLock()): + def __init__(self, + decorated_patch_applier=None, + decorated_config_replacer=None, + decorated_config_rollbacker=None, + config_lock = ConfigLock()): Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker) self.config_lock = config_lock @@ -376,8 +386,7 @@ def create_patch_applier(self, config_format, verbose, dry_run): patch_wrapper = PatchWrapper(config_wrapper) if config_format == ConfigFormat.CONFIGDB: - patch_applier = \ - ConfigDbDecorator( \ + patch_applier = ConfigDbDecorator( decorated_patch_applier = patch_applier, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) elif config_format == ConfigFormat.SONICYANG: pass @@ -400,8 +409,7 @@ def create_config_replacer(self, config_format, verbose, dry_run): config_replacer = ConfigReplacer(patch_applier=patch_applier, config_wrapper=config_wrapper) if config_format == ConfigFormat.CONFIGDB: - config_replacer = \ - ConfigDbDecorator( \ + config_replacer = ConfigDbDecorator( decorated_config_replacer = config_replacer, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) elif config_format == ConfigFormat.SONICYANG: pass @@ -441,8 +449,9 @@ def get_config_wrapper(self, dry_run): return ConfigWrapper() class GenericUpdater: - def __init__(self, generic_update_factory = GenericUpdateFactory()): - self.generic_update_factory = generic_update_factory + def __init__(self, generic_update_factory=None): + self.generic_update_factory = \ + generic_update_factory if generic_update_factory is not None else GenericUpdateFactory() def apply_patch(self, patch, config_format, verbose, dry_run): patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run) diff --git a/tests/generic_config_updater/files/sonic_yang_as_json_with_unexpected_colons.json b/tests/generic_config_updater/files/sonic_yang_as_json_with_unexpected_colons.json new file mode 100644 index 0000000000..aac97da42b --- /dev/null +++ b/tests/generic_config_updater/files/sonic_yang_as_json_with_unexpected_colons.json @@ -0,0 +1,114 @@ +{ + "sonic-vlan:sonic-vlan": { + "sonic-vlan::VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + } + ] + }, + "sonic-vlan::VLAN": { + "VLAN_LIST": [ + { + "name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-vlan::ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "sonic-vlan::PORT": { + "PORT_LIST": [ + { + "name": "Ethernet0", + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": 100000 + }, + { + "name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + } + ] + } + } +} diff --git a/tests/generic_config_updater/files/sonic_yang_as_json_without_colons.json b/tests/generic_config_updater/files/sonic_yang_as_json_without_colons.json new file mode 100644 index 0000000000..ad4ab15f4a --- /dev/null +++ b/tests/generic_config_updater/files/sonic_yang_as_json_without_colons.json @@ -0,0 +1,114 @@ +{ + "sonic-vlan:sonic-vlan": { + "VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "name": "Vlan1000", + "port": "Ethernet0", + "tagging_mode": "untagged" + }, + { + "name": "Vlan1000", + "port": "Ethernet4", + "tagging_mode": "untagged" + }, + { + "name": "Vlan1000", + "port": "Ethernet8", + "tagging_mode": "untagged" + } + ] + }, + "VLAN": { + "VLAN_LIST": [ + { + "name": "Vlan1000", + "vlanid": 1000, + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + ] + } + }, + "sonic-acl:sonic-acl": { + "ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "NO-NSW-PACL-V4", + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0" + ] + }, + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + ] + } + }, + "sonic-port:sonic-port": { + "PORT": { + "PORT_LIST": [ + { + "name": "Ethernet0", + "alias": "Eth1", + "lanes": "65, 66, 67, 68", + "description": "Ethernet0 100G link", + "speed": 100000 + }, + { + "name": "Ethernet4", + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": 1, + "lanes": "29,30,31,32", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + }, + { + "name": "Ethernet8", + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": 2, + "lanes": "33,34,35,36", + "mtu": 9100, + "pfc_asym": "off", + "speed": 40000 + } + ] + } + } +} diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index ee1f3df30f..1a7d8d409a 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -2,12 +2,12 @@ import jsonpatch import os import shutil +import sys import unittest from imp import load_source from unittest.mock import Mock, call -load_source('gu', \ - os.path.join(os.path.dirname(__file__), '../..', 'generic_config_updater', 'generic_updater.py')) -import gu +sys.path.insert(1, '../../generic_config_updater') +import generic_updater as gu class MockSideEffectDict: def __init__(self, map): @@ -136,6 +136,27 @@ def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_d # Assert self.assertDictEqual(expected, actual) + def test_convert_sonic_yang_to_config_db__table_name_without_colons__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON_WITHOUT_COLONS) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__table_name_with_unexpected_colons__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act and assert + self.assertRaises(ValueError, + config_wrapper.convert_sonic_yang_to_config_db, + Files.SONIC_YANG_AS_JSON_WITH_UNEXPECTED_COLONS) + def test_validate_sonic_yang_config__valid_config__returns_true(self): # Arrange config_wrapper = gu.ConfigWrapper() @@ -205,7 +226,7 @@ def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__f patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] # Act and Assert - self.assertRaises(Exception, patch_wrapper.convert_config_db_patch_to_sonic_yang_patch, patch) + self.assertRaises(ValueError, patch_wrapper.convert_config_db_patch_to_sonic_yang_patch, patch) def test_same_patch__no_diff__returns_true(self): # Arrange @@ -334,14 +355,14 @@ def test_apply__invalid_sonic_yang__failure(self): patch_applier = self.__create_patch_applier(valid_sonic_yang=False) # Act and assert - self.assertRaises(Exception, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + self.assertRaises(ValueError, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) def test_apply__json_not_fully_updated__failure(self): # Arrange patch_applier = self.__create_patch_applier(verified_same_config=False) # Act and assert - self.assertRaises(Exception, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + self.assertRaises(gu.ConfigNotCompletelyUpdatedError, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) def test_apply__no_errors__update_successful(self): # Arrange @@ -353,13 +374,13 @@ def test_apply__no_errors__update_successful(self): # Assert patch_applier.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) - patch_applier.patch_wrapper.simulate_patch.assert_has_calls( \ + patch_applier.patch_wrapper.simulate_patch.assert_has_calls( [call(Files.MULTI_OPERATION_SONIC_YANG_PATCH, Files.SONIC_YANG_AS_JSON)]) - patch_applier.config_wrapper.validate_sonic_yang_config.assert_has_calls( \ + patch_applier.config_wrapper.validate_sonic_yang_config.assert_has_calls( [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) - patch_applier.patchorderer.order.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) + patch_applier.patchsorter.order.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) patch_applier.changeapplier.apply.assert_has_calls([call(changes[0]), call(changes[1])]) - patch_applier.patch_wrapper.verify_same_json.assert_has_calls( \ + patch_applier.patch_wrapper.verify_same_json.assert_has_calls( [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_same_config=True): @@ -371,23 +392,23 @@ def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_s patch_wrapper = Mock() patch_wrapper.simulate_patch.side_effect = \ - create_side_effect_dict( \ - {(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH), str(Files.SONIC_YANG_AS_JSON)): \ + create_side_effect_dict( + {(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH), str(Files.SONIC_YANG_AS_JSON)): Files.SONIC_YANG_AFTER_MULTI_PATCH}) patch_wrapper.verify_same_json.side_effect = \ - create_side_effect_dict( \ - {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + create_side_effect_dict( + {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): verified_same_config}) changes = [Mock(), Mock()] if not changes else changes - patchorderer = Mock() - patchorderer.order.side_effect = \ + patchsorter = Mock() + patchsorter.order.side_effect = \ create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) changeapplier = Mock() changeapplier.apply.side_effect = create_side_effect_dict({(str(changes[0]),): 0, (str(changes[1]),): 0}) - return gu.PatchApplier(patchorderer, changeapplier, config_wrapper, patch_wrapper) + return gu.PatchApplier(patchsorter, changeapplier, config_wrapper, patch_wrapper) class TestConfigReplacer(unittest.TestCase): def test_replace__invalid_sonic_yang__failure(self): @@ -395,14 +416,14 @@ def test_replace__invalid_sonic_yang__failure(self): config_replacer = self.__create_config_replacer(valid_sonic_yang=False) # Act and assert - self.assertRaises(Exception, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + self.assertRaises(ValueError, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) def test_replace__json_not_fully_updated__failure(self): # Arrange config_replacer = self.__create_config_replacer(verified_same_config=False) # Act and assert - self.assertRaises(Exception, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + self.assertRaises(gu.ConfigNotCompletelyUpdatedError, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) def test_replace__no_errors__update_successful(self): # Arrange @@ -412,13 +433,13 @@ def test_replace__no_errors__update_successful(self): config_replacer.replace(Files.SONIC_YANG_AFTER_MULTI_PATCH) # Assert - config_replacer.config_wrapper.validate_sonic_yang_config.assert_has_calls( \ + config_replacer.config_wrapper.validate_sonic_yang_config.assert_has_calls( [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) config_replacer.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) - config_replacer.patch_wrapper.generate_patch.assert_has_calls( \ + config_replacer.patch_wrapper.generate_patch.assert_has_calls( [call(Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) config_replacer.patch_applier.apply.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) - config_replacer.patch_wrapper.verify_same_json.assert_has_calls( \ + config_replacer.patch_wrapper.verify_same_json.assert_has_calls( [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified_same_config=True): @@ -430,17 +451,17 @@ def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified patch_wrapper = Mock() patch_wrapper.generate_patch.side_effect = \ - create_side_effect_dict( \ - {(str(Files.SONIC_YANG_AS_JSON), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + create_side_effect_dict( + {(str(Files.SONIC_YANG_AS_JSON), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): Files.MULTI_OPERATION_SONIC_YANG_PATCH}) patch_wrapper.verify_same_json.side_effect = \ - create_side_effect_dict( \ + create_side_effect_dict( {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ verified_same_config}) changes = [Mock(), Mock()] if not changes else changes - patchorderer = Mock() - patchorderer.order.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): \ + patchsorter = Mock() + patchsorter.order.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): \ changes}) patch_applier = Mock() @@ -465,7 +486,7 @@ def test_rollback__checkpoint_does_not_exist__failure(self): rollbacker = self.create_rollbacker() # Act and assert - self.assertRaises(Exception, rollbacker.rollback, "NonExistingCheckpoint") + self.assertRaises(ValueError, rollbacker.rollback, "NonExistingCheckpoint") def test_rollback__no_errors__success(self): # Arrange @@ -562,7 +583,7 @@ def test_delete_checkpoint__checkpoint_does_not_exist__failure(self): rollbacker = self.create_rollbacker() # Act and assert - self.assertRaises(Exception, rollbacker.delete_checkpoint, self.any_checkpoint_name) + self.assertRaises(ValueError, rollbacker.delete_checkpoint, self.any_checkpoint_name) def test_delete_checkpoint__checkpoint_exist__success(self): # Arrange @@ -627,10 +648,9 @@ def create_rollbacker(self): config_wrapper = Mock() config_wrapper.get_sonic_yang_as_json.return_value = self.any_config - return gu.FileSystemConfigRollbacker( \ - checkpoints_dir=self.checkpoints_dir, \ - config_replacer=replacer, \ - config_wrapper=config_wrapper) + return gu.FileSystemConfigRollbacker(checkpoints_dir=self.checkpoints_dir, + config_replacer=replacer, + config_wrapper=config_wrapper) class TestGenericUpdateFactory(unittest.TestCase): def setUp(self): @@ -642,7 +662,7 @@ def test_create_patch_applier__invalid_config_format__failure(self): factory = gu.GenericUpdateFactory() # Act and assert - self.assertRaises( \ + self.assertRaises( ValueError, factory.create_patch_applier, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) def test_create_patch_applier__different_options(self): @@ -666,7 +686,7 @@ def test_create_config_replacer__invalid_config_format__failure(self): factory = gu.GenericUpdateFactory() # Act and assert - self.assertRaises( \ + self.assertRaises( ValueError, factory.create_config_replacer, "INVALID_FORMAT", self.any_verbose, self.any_dry_run) def test_create_config_replacer__different_options(self): @@ -752,12 +772,12 @@ def validate_create_config_rollbacker(self, params, expected_decorators): if params["dry_run"]: self.assertIsInstance(config_rollbacker.config_wrapper, gu.DryRunConfigWrapper) self.assertIsInstance(config_rollbacker.config_replacer.config_wrapper, gu.DryRunConfigWrapper) - self.assertIsInstance( \ + self.assertIsInstance( config_rollbacker.config_replacer.patch_applier.config_wrapper, gu.DryRunConfigWrapper) else: self.assertIsInstance(config_rollbacker.config_wrapper, gu.ConfigWrapper) self.assertIsInstance(config_rollbacker.config_replacer.config_wrapper, gu.ConfigWrapper) - self.assertIsInstance( \ + self.assertIsInstance( config_rollbacker.config_replacer.patch_applier.config_wrapper, gu.ConfigWrapper) class TestGenericUpdater(unittest.TestCase): @@ -776,13 +796,13 @@ def test_apply_patch__creates_applier_and_apply(self): factory = Mock() factory.create_patch_applier.side_effect = \ - create_side_effect_dict( \ + create_side_effect_dict( {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): patch_applier}) generic_updater = gu.GenericUpdater(factory) # Act - generic_updater.apply_patch( \ + generic_updater.apply_patch( Files.SINGLE_OPERATION_SONIC_YANG_PATCH, self.any_config_format, self.any_verbose, self.any_dry_run) # Assert @@ -795,7 +815,7 @@ def test_replace__creates_replacer_and_replace(self): factory = Mock() factory.create_config_replacer.side_effect = \ - create_side_effect_dict( \ + create_side_effect_dict( {(str(self.any_config_format), str(self.any_verbose), str(self.any_dry_run),): config_replacer}) generic_updater = gu.GenericUpdater(factory) @@ -887,7 +907,7 @@ def setUp(self): self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] self.decorated_config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list - self.decorator = gu.Decorator( \ + self.decorator = gu.Decorator( self.decorated_patch_applier, self.decorated_config_replacer, self.decorated_config_rollbacker) def test_apply__calls_decorated_applier(self): @@ -945,9 +965,9 @@ def test_apply__converts_to_yang_and_calls_decorated_class(self): config_db_decorator.apply(Files.SINGLE_OPERATION_CONFIG_DB_PATCH) # Assert - config_db_decorator.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( \ + config_db_decorator.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( [call(Files.SINGLE_OPERATION_CONFIG_DB_PATCH)]) - config_db_decorator.decorated_patch_applier.apply.assert_has_calls( \ + config_db_decorator.decorated_patch_applier.apply.assert_has_calls( [call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) def test_replace__converts_to_yang_and_calls_decorated_class(self): @@ -958,7 +978,7 @@ def test_replace__converts_to_yang_and_calls_decorated_class(self): config_db_decorator.replace(Files.CONFIG_DB_AS_JSON) # Assert - config_db_decorator.config_wrapper.convert_config_db_to_sonic_yang.assert_has_calls( \ + config_db_decorator.config_wrapper.convert_config_db_to_sonic_yang.assert_has_calls( [call(Files.CONFIG_DB_AS_JSON)]) config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) @@ -978,11 +998,10 @@ def __create_config_db_decorator(self): config_wrapper.convert_config_db_to_sonic_yang.side_effect = \ create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): Files.SONIC_YANG_AS_JSON}) - return gu.ConfigDbDecorator( \ - decorated_patch_applier=patch_applier, \ - decorated_config_replacer=config_replacer, \ - patch_wrapper=patch_wrapper, \ - config_wrapper=config_wrapper) + return gu.ConfigDbDecorator(decorated_patch_applier=patch_applier, + decorated_config_replacer=config_replacer, + patch_wrapper=patch_wrapper, + config_wrapper=config_wrapper) class TestConfigLockDecorator(unittest.TestCase): def setUp(self): @@ -997,7 +1016,7 @@ def test_apply__lock_config(self): # Assert config_lock_decorator.config_lock.acquire_lock.assert_called_once() - config_lock_decorator.decorated_patch_applier.apply.assert_has_calls( \ + config_lock_decorator.decorated_patch_applier.apply.assert_has_calls( [call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) config_lock_decorator.config_lock.release_lock.assert_called_once() @@ -1052,8 +1071,7 @@ def __create_config_lock_decorator(self): config_rollbacker.delete_checkpoint.side_effect = create_side_effect_dict({(self.any_checkpoint_name,): 0}) - return gu.ConfigLockDecorator( \ - config_lock=config_lock, \ - decorated_patch_applier=patch_applier, \ - decorated_config_replacer=config_replacer, \ - decorated_config_rollbacker=config_rollbacker) + return gu.ConfigLockDecorator(config_lock=config_lock, + decorated_patch_applier=patch_applier, + decorated_config_replacer=config_replacer, + decorated_config_rollbacker=config_rollbacker) From 30c12659101d4871972b349e06d5fc41132560d0 Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 14 Apr 2021 08:45:30 -0700 Subject: [PATCH 10/32] patch-applier to expect input of ConfigDb patch instead SonicYang patch --- generic_config_updater/generic_updater.py | 74 ++++-- .../files/config_db_after_multi_patch.json | 133 ++++++++++ .../files/config_db_as_json_invalid.json | 7 + .../generic_updater_test.py | 227 ++++++++++++------ 4 files changed, 341 insertions(+), 100 deletions(-) create mode 100644 tests/generic_config_updater/files/config_db_after_multi_patch.json create mode 100644 tests/generic_config_updater/files/config_db_as_json_invalid.json diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index c75021f84c..fdd0e9efd4 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -30,7 +30,7 @@ def release_lock(self): pass class PatchSorter: - def order(self, patch): + def sort(self, patch): # TODO: Implement patch sorter pass @@ -103,6 +103,18 @@ def validate_sonic_yang_config(self, sonic_yang_as_json): except sonic_yang.SonicYangException as ex: return False + def validate_config_db_config(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + try: + sy.loadData(config_db_as_json) + + sy.validate_data_tree() + return True + except sonic_yang.SonicYangException as ex: + return False + def crop_tables_without_yang(self, config_db_as_json): sy = sonic_yang.SonicYang(self.yang_dir) sy.loadYangModel() @@ -119,7 +131,6 @@ def _create_and_connect_config_db(self): if self.default_config_db_connector != None: return self.default_config_db_connector - db_kwargs = dict() config_db = ConfigDBConnector() config_db.connect() return config_db @@ -133,7 +144,7 @@ class PatchWrapper: def __init__(self, config_wrapper=None): self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() - def validate_config_db_patch(self, patch): + def validate_config_db_patch_has_yang_models(self, patch): config_db = {} for operation in patch: tokens = operation['path'].split('/')[1:] @@ -161,7 +172,7 @@ def simulate_patch(self, patch, jsonconfig): return patch.apply(jsonconfig) def convert_config_db_patch_to_sonic_yang_patch(self, patch): - if not(self.validate_config_db_patch(patch)): + if not(self.validate_config_db_patch_has_yang_models(patch)): raise ValueError(f"Given patch is not valid") current_config_db = self.config_wrapper.get_config_db_as_json() @@ -172,6 +183,15 @@ def convert_config_db_patch_to_sonic_yang_patch(self, patch): return self.generate_patch(current_yang, target_yang) + def convert_sonic_yang_patch_to_config_db_patch(self, patch): + current_yang = self.config_wrapper.get_sonic_yang_as_json() + target_yang = self.simulate_patch(patch, current_yang) + + current_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(current_yang) + target_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(target_yang) + + return self.generate_patch(current_config_db, target_config_db) + class ConfigFormat(Enum): SONICYANG = 1 CONFIGDB = 2 @@ -188,25 +208,29 @@ def __init__(self, self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def apply(self, patch): - # Get old config as SONiC Yang - old_config = self.config_wrapper.get_sonic_yang_as_json() + # validate patch is only updating tables with yang models + if not(self.patch_wrapper.validate_config_db_patch_has_yang_models(patch)): + raise ValueError(f"Given patch is not valid because it has changes to tables without YANG models") + + # Get old config + old_config = self.config_wrapper.get_config_db_as_json() # Generate target config target_config = self.patch_wrapper.simulate_patch(patch, old_config) # Validate target config - if not(self.config_wrapper.validate_sonic_yang_config(target_config)): - raise ValueError(f"The given patch is not valid") + if not(self.config_wrapper.validate_config_db_config(target_config)): + raise ValueError(f"Given patch is not valid because it will result in an invalid config") # Generate list of changes to apply - changes = self.patchsorter.order(patch) + changes = self.patchsorter.sort(patch) # Apply changes in order for change in changes: self.changeapplier.apply(change) # Validate config updated successfully - new_config = self.config_wrapper.get_sonic_yang_as_json() + new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise ConfigNotCompletelyUpdatedError(f"After applying patch to config, there are still some parts not updated") @@ -217,15 +241,15 @@ def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None): self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def replace(self, target_config): - if not(self.config_wrapper.validate_sonic_yang_config(target_config)): + if not(self.config_wrapper.validate_config_db_config(target_config)): raise ValueError(f"The given target config is not valid") - old_config = self.config_wrapper.get_sonic_yang_as_json() + old_config = self.config_wrapper.get_config_db_as_json() patch = self.patch_wrapper.generate_patch(old_config, target_config) self.patch_applier.apply(patch) - new_config = self.config_wrapper.get_sonic_yang_as_json() + new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise ConfigNotCompletelyUpdatedError(f"After replacing config, there is still some parts not updated") @@ -333,7 +357,7 @@ def list_checkpoints(self): def delete_checkpoint(self, checkpoint_name): self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) -class ConfigDbDecorator(Decorator): +class SonicYangDecorator(Decorator): def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None): Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer) @@ -341,12 +365,12 @@ def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, self.config_wrapper = config_wrapper def apply(self, patch): - yang_patch = self.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - Decorator.apply(self, yang_patch) + config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + Decorator.apply(self, config_db_patch) def replace(self, target_config): - yang_target_config = self.config_wrapper.convert_config_db_to_sonic_yang(target_config) - Decorator.replace(self, yang_target_config) + config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config) + Decorator.replace(self, config_db_target_config) class ConfigLockDecorator(Decorator): def __init__(self, @@ -386,10 +410,10 @@ def create_patch_applier(self, config_format, verbose, dry_run): patch_wrapper = PatchWrapper(config_wrapper) if config_format == ConfigFormat.CONFIGDB: - patch_applier = ConfigDbDecorator( - decorated_patch_applier = patch_applier, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) - elif config_format == ConfigFormat.SONICYANG: pass + elif config_format == ConfigFormat.SONICYANG: + patch_applier = SonicYangDecorator( + decorated_patch_applier = patch_applier, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) else: raise ValueError(f"config-format '{config_format}' is not supported") @@ -409,10 +433,10 @@ def create_config_replacer(self, config_format, verbose, dry_run): config_replacer = ConfigReplacer(patch_applier=patch_applier, config_wrapper=config_wrapper) if config_format == ConfigFormat.CONFIGDB: - config_replacer = ConfigDbDecorator( - decorated_config_replacer = config_replacer, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) - elif config_format == ConfigFormat.SONICYANG: pass + elif config_format == ConfigFormat.SONICYANG: + config_replacer = SonicYangDecorator( + decorated_config_replacer = config_replacer, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) else: raise ValueError(f"config-format '{config_format}' is not supported") @@ -475,4 +499,4 @@ def delete_checkpoint(self, checkpoint_name, verbose, dry_run): def list_checkpoints(self, verbose, dry_run): config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) - return config_rollbacker.list_checkpoints() \ No newline at end of file + return config_rollbacker.list_checkpoints() diff --git a/tests/generic_config_updater/files/config_db_after_multi_patch.json b/tests/generic_config_updater/files/config_db_after_multi_patch.json new file mode 100644 index 0000000000..71313bd4e0 --- /dev/null +++ b/tests/generic_config_updater/files/config_db_after_multi_patch.json @@ -0,0 +1,133 @@ +{ + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet2": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet3": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet1": { + "tagging_mode": "untagged" + } + }, + "VLAN": { + "Vlan1000": { + "vlanid": "1000", + "dhcp_servers": [ + "192.0.0.1", + "192.0.0.2", + "192.0.0.3", + "192.0.0.4" + ] + } + }, + "ACL_TABLE": { + "NO-NSW-PACL-V4": { + "type": "L3", + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0", + "Ethernet1", + "Ethernet2", + "Ethernet3" + ] + }, + "DATAACL": { + "policy_desc": "DATAACL", + "ports": [ + "Ethernet4" + ], + "stage": "ingress", + "type": "L3" + }, + "EVERFLOW": { + "policy_desc": "EVERFLOW", + "ports": [ + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRROR" + }, + "EVERFLOWV6": { + "policy_desc": "EVERFLOWV6", + "ports": [ + "Ethernet4", + "Ethernet8" + ], + "stage": "ingress", + "type": "MIRRORV6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": "10000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "fortyGigE0/4", + "description": "Servers0:eth0", + "index": "1", + "lanes": "29,30,31,32", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "fortyGigE0/8", + "description": "Servers1:eth0", + "index": "2", + "lanes": "33,34,35,36", + "mtu": "9100", + "pfc_asym": "off", + "speed": "40000" + }, + "Ethernet3": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "10000" + }, + "Ethernet1": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "10000" + }, + "Ethernet2": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "10000" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "wred_red_enable": "true", + "ecn": "ecn_all", + "green_max_threshold": "2097152", + "green_min_threshold": "1048576", + "yellow_max_threshold": "2097152", + "yellow_min_threshold": "1048576", + "red_max_threshold": "2097152", + "red_min_threshold": "1048576", + "green_drop_probability": "5", + "yellow_drop_probability": "5", + "red_drop_probability": "5" + } + } +} \ No newline at end of file diff --git a/tests/generic_config_updater/files/config_db_as_json_invalid.json b/tests/generic_config_updater/files/config_db_as_json_invalid.json new file mode 100644 index 0000000000..a2cfdc91df --- /dev/null +++ b/tests/generic_config_updater/files/config_db_as_json_invalid.json @@ -0,0 +1,7 @@ +{ + "VLAN_MEMBER": { + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + } + } +} diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 1a7d8d409a..430197499f 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -179,6 +179,28 @@ def test_validate_sonic_yang_config__invvalid_config__returns_false(self): # Assert self.assertEqual(expected, actual) + def test_validate_config_db_config__valid_config__returns_true(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = True + + # Act + actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertEqual(expected, actual) + + def test_config_db_config__invvalid_config__returns_false(self): + # Arrange + config_wrapper = gu.ConfigWrapper() + expected = False + + # Act + actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON_INVALID) + + # Assert + self.assertEqual(expected, actual) + def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): # Arrange config_wrapper = gu.ConfigWrapper() @@ -196,26 +218,26 @@ def __get_config_db_connector_mock(self, config_db_as_dict): return mock_connector class TestPatchWrapper(unittest.TestCase): - def test_validate_config_db_patch__table_without_yang_model__returns_false(self): + def test_validate_config_db_patch_has_yang_models__table_without_yang_model__returns_false(self): # Arrange patch_wrapper = gu.PatchWrapper() patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] expected = False # Act - actual = patch_wrapper.validate_config_db_patch(patch) + actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) # Assert self.assertEqual(expected, actual) - def test_validate_config_db_patch__table_with_yang_model__returns_true(self): + def test_validate_config_db_patch_has_yang_models__table_with_yang_model__returns_true(self): # Arrange patch_wrapper = gu.PatchWrapper() patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] expected = True # Act - actual = patch_wrapper.validate_config_db_patch(patch) + actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) # Assert self.assertEqual(expected, actual) @@ -327,17 +349,56 @@ def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch_ # Assert self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) + def test_convert_sonic_yang_patch_to_config_db_patch__empty_patch__returns_empty_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch = jsonpatch.JsonPatch([]) + expected = jsonpatch.JsonPatch([]) + + # Act + actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_sonic_yang_patch_to_config_db_patch__single_operation_patch__returns_config_db_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch = Files.SINGLE_OPERATION_SONIC_YANG_PATCH + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + + # Act + actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_sonic_yang_patch_to_config_db_patch__multiple_operations_patch__returns_config_db_patch(self): + # Arrange + config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + sonic_yang_patch = Files.MULTI_OPERATION_SONIC_YANG_PATCH + + # Act + config_db_patch = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(sonic_yang_patch) + + # Assert + self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) + def __assert_same_patch(self, config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper): sonic_yang = config_wrapper.get_sonic_yang_as_json() config_db = config_wrapper.get_config_db_as_json() after_update_sonic_yang = patch_wrapper.simulate_patch(sonic_yang_patch, sonic_yang) after_update_config_db = patch_wrapper.simulate_patch(config_db_patch, config_db) + after_update_config_db_cropped = config_wrapper.crop_tables_without_yang(after_update_config_db) - after_update_config_db_as_sonic_yang = \ - config_wrapper.convert_config_db_to_sonic_yang(after_update_config_db) + after_update_sonic_yang_as_config_db = \ + config_wrapper.convert_sonic_yang_to_config_db(after_update_sonic_yang) - self.assertTrue(patch_wrapper.verify_same_json(after_update_sonic_yang, after_update_config_db_as_sonic_yang)) + self.assertTrue(patch_wrapper.verify_same_json(after_update_config_db_cropped, after_update_sonic_yang_as_config_db)) def __get_config_wrapper_mock(self, config_db_as_dict): config_db_connector_mock = self.__get_config_db_connector_mock(config_db_as_dict) @@ -350,19 +411,26 @@ def __get_config_db_connector_mock(self, config_db_as_dict): return mock_connector class TestPatchApplier(unittest.TestCase): - def test_apply__invalid_sonic_yang__failure(self): + def test_apply__invalid_patch_updating_tables_without_yang_models__failure(self): + # Arrange + patch_applier = self.__create_patch_applier(valid_patch_only_tables_with_yang_models=False) + + # Act and assert + self.assertRaises(ValueError, patch_applier.apply, Files.MULTI_OPERATION_CONFIG_DB_PATCH) + + def test_apply__invalid_config_db__failure(self): # Arrange - patch_applier = self.__create_patch_applier(valid_sonic_yang=False) + patch_applier = self.__create_patch_applier(valid_config_db=False) # Act and assert - self.assertRaises(ValueError, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + self.assertRaises(ValueError, patch_applier.apply, Files.MULTI_OPERATION_CONFIG_DB_PATCH) def test_apply__json_not_fully_updated__failure(self): # Arrange patch_applier = self.__create_patch_applier(verified_same_config=False) # Act and assert - self.assertRaises(gu.ConfigNotCompletelyUpdatedError, patch_applier.apply, Files.MULTI_OPERATION_SONIC_YANG_PATCH) + self.assertRaises(gu.ConfigNotCompletelyUpdatedError, patch_applier.apply, Files.MULTI_OPERATION_CONFIG_DB_PATCH) def test_apply__no_errors__update_successful(self): # Arrange @@ -370,40 +438,49 @@ def test_apply__no_errors__update_successful(self): patch_applier = self.__create_patch_applier(changes) # Act - patch_applier.apply(Files.MULTI_OPERATION_SONIC_YANG_PATCH) + patch_applier.apply(Files.MULTI_OPERATION_CONFIG_DB_PATCH) # Assert - patch_applier.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + patch_applier.patch_wrapper.validate_config_db_patch_has_yang_models.assert_has_calls( + [call(Files.MULTI_OPERATION_CONFIG_DB_PATCH)]) + patch_applier.config_wrapper.get_config_db_as_json.assert_has_calls([call(), call()]) patch_applier.patch_wrapper.simulate_patch.assert_has_calls( - [call(Files.MULTI_OPERATION_SONIC_YANG_PATCH, Files.SONIC_YANG_AS_JSON)]) - patch_applier.config_wrapper.validate_sonic_yang_config.assert_has_calls( - [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) - patch_applier.patchsorter.order.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) + [call(Files.MULTI_OPERATION_CONFIG_DB_PATCH, Files.CONFIG_DB_AS_JSON)]) + patch_applier.config_wrapper.validate_config_db_config.assert_has_calls( + [call(Files.CONFIG_DB_AFTER_MULTI_PATCH)]) + patch_applier.patchsorter.sort.assert_has_calls([call(Files.MULTI_OPERATION_CONFIG_DB_PATCH)]) patch_applier.changeapplier.apply.assert_has_calls([call(changes[0]), call(changes[1])]) patch_applier.patch_wrapper.verify_same_json.assert_has_calls( - [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + [call(Files.CONFIG_DB_AFTER_MULTI_PATCH, Files.CONFIG_DB_AFTER_MULTI_PATCH)]) - def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + def __create_patch_applier(self, + changes=None, + valid_patch_only_tables_with_yang_models=True, + valid_config_db=True, + verified_same_config=True): config_wrapper = Mock() - config_wrapper.get_sonic_yang_as_json.side_effect = \ - [Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH] - config_wrapper.validate_sonic_yang_config.side_effect = \ - create_side_effect_dict({(str(Files.SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) + config_wrapper.get_config_db_as_json.side_effect = \ + [Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AFTER_MULTI_PATCH] + config_wrapper.validate_config_db_config.side_effect = \ + create_side_effect_dict({(str(Files.CONFIG_DB_AFTER_MULTI_PATCH),): valid_config_db}) patch_wrapper = Mock() + patch_wrapper.validate_config_db_patch_has_yang_models.side_effect = \ + create_side_effect_dict( + {(str(Files.MULTI_OPERATION_CONFIG_DB_PATCH),): valid_patch_only_tables_with_yang_models}) patch_wrapper.simulate_patch.side_effect = \ create_side_effect_dict( - {(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH), str(Files.SONIC_YANG_AS_JSON)): - Files.SONIC_YANG_AFTER_MULTI_PATCH}) + {(str(Files.MULTI_OPERATION_CONFIG_DB_PATCH), str(Files.CONFIG_DB_AS_JSON)): + Files.CONFIG_DB_AFTER_MULTI_PATCH}) patch_wrapper.verify_same_json.side_effect = \ create_side_effect_dict( - {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): + {(str(Files.CONFIG_DB_AFTER_MULTI_PATCH), str(Files.CONFIG_DB_AFTER_MULTI_PATCH)): verified_same_config}) changes = [Mock(), Mock()] if not changes else changes patchsorter = Mock() - patchsorter.order.side_effect = \ - create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): changes}) + patchsorter.sort.side_effect = \ + create_side_effect_dict({(str(Files.MULTI_OPERATION_CONFIG_DB_PATCH),): changes}) changeapplier = Mock() changeapplier.apply.side_effect = create_side_effect_dict({(str(changes[0]),): 0, (str(changes[1]),): 0}) @@ -411,61 +488,61 @@ def __create_patch_applier(self, changes=None, valid_sonic_yang=True, verified_s return gu.PatchApplier(patchsorter, changeapplier, config_wrapper, patch_wrapper) class TestConfigReplacer(unittest.TestCase): - def test_replace__invalid_sonic_yang__failure(self): + def test_replace__invalid_config_db__failure(self): # Arrange - config_replacer = self.__create_config_replacer(valid_sonic_yang=False) + config_replacer = self.__create_config_replacer(valid_config_db=False) # Act and assert - self.assertRaises(ValueError, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + self.assertRaises(ValueError, config_replacer.replace, Files.CONFIG_DB_AFTER_MULTI_PATCH) def test_replace__json_not_fully_updated__failure(self): # Arrange config_replacer = self.__create_config_replacer(verified_same_config=False) # Act and assert - self.assertRaises(gu.ConfigNotCompletelyUpdatedError, config_replacer.replace, Files.SONIC_YANG_AFTER_MULTI_PATCH) + self.assertRaises(gu.ConfigNotCompletelyUpdatedError, config_replacer.replace, Files.CONFIG_DB_AFTER_MULTI_PATCH) def test_replace__no_errors__update_successful(self): # Arrange config_replacer = self.__create_config_replacer() # Act - config_replacer.replace(Files.SONIC_YANG_AFTER_MULTI_PATCH) + config_replacer.replace(Files.CONFIG_DB_AFTER_MULTI_PATCH) # Assert - config_replacer.config_wrapper.validate_sonic_yang_config.assert_has_calls( - [call(Files.SONIC_YANG_AFTER_MULTI_PATCH)]) - config_replacer.config_wrapper.get_sonic_yang_as_json.assert_has_calls([call(), call()]) + config_replacer.config_wrapper.validate_config_db_config.assert_has_calls( + [call(Files.CONFIG_DB_AFTER_MULTI_PATCH)]) + config_replacer.config_wrapper.get_config_db_as_json.assert_has_calls([call(), call()]) config_replacer.patch_wrapper.generate_patch.assert_has_calls( - [call(Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) - config_replacer.patch_applier.apply.assert_has_calls([call(Files.MULTI_OPERATION_SONIC_YANG_PATCH)]) + [call(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AFTER_MULTI_PATCH)]) + config_replacer.patch_applier.apply.assert_has_calls([call(Files.MULTI_OPERATION_CONFIG_DB_PATCH)]) config_replacer.patch_wrapper.verify_same_json.assert_has_calls( - [call(Files.SONIC_YANG_AFTER_MULTI_PATCH, Files.SONIC_YANG_AFTER_MULTI_PATCH)]) + [call(Files.CONFIG_DB_AFTER_MULTI_PATCH, Files.CONFIG_DB_AFTER_MULTI_PATCH)]) - def __create_config_replacer(self, changes=None, valid_sonic_yang=True, verified_same_config=True): + def __create_config_replacer(self, changes=None, valid_config_db=True, verified_same_config=True): config_wrapper = Mock() - config_wrapper.validate_sonic_yang_config.side_effect = \ - create_side_effect_dict({(str(Files.SONIC_YANG_AFTER_MULTI_PATCH),): valid_sonic_yang}) - config_wrapper.get_sonic_yang_as_json.side_effect = \ - [Files.SONIC_YANG_AS_JSON, Files.SONIC_YANG_AFTER_MULTI_PATCH] + config_wrapper.validate_config_db_config.side_effect = \ + create_side_effect_dict({(str(Files.CONFIG_DB_AFTER_MULTI_PATCH),): valid_config_db}) + config_wrapper.get_config_db_as_json.side_effect = \ + [Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AFTER_MULTI_PATCH] patch_wrapper = Mock() patch_wrapper.generate_patch.side_effect = \ create_side_effect_dict( - {(str(Files.SONIC_YANG_AS_JSON), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): - Files.MULTI_OPERATION_SONIC_YANG_PATCH}) + {(str(Files.CONFIG_DB_AS_JSON), str(Files.CONFIG_DB_AFTER_MULTI_PATCH)): + Files.MULTI_OPERATION_CONFIG_DB_PATCH}) patch_wrapper.verify_same_json.side_effect = \ create_side_effect_dict( - {(str(Files.SONIC_YANG_AFTER_MULTI_PATCH), str(Files.SONIC_YANG_AFTER_MULTI_PATCH)): \ + {(str(Files.CONFIG_DB_AFTER_MULTI_PATCH), str(Files.CONFIG_DB_AFTER_MULTI_PATCH)): \ verified_same_config}) changes = [Mock(), Mock()] if not changes else changes patchsorter = Mock() - patchsorter.order.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): \ + patchsorter.sort.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_CONFIG_DB_PATCH),): \ changes}) patch_applier = Mock() - patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_SONIC_YANG_PATCH),): 0}) + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.MULTI_OPERATION_CONFIG_DB_PATCH),): 0}) return gu.ConfigReplacer(patch_applier, config_wrapper, patch_wrapper) @@ -672,8 +749,8 @@ def test_create_patch_applier__different_options(self): {"dry_run": {True: None, False: gu.ConfigLockDecorator}}, { "config_format": { - gu.ConfigFormat.SONICYANG: None, - gu.ConfigFormat.CONFIGDB: gu.ConfigDbDecorator + gu.ConfigFormat.SONICYANG: gu.SonicYangDecorator, + gu.ConfigFormat.CONFIGDB: None, } }, ] @@ -696,8 +773,8 @@ def test_create_config_replacer__different_options(self): {"dry_run": {True: None, False: gu.ConfigLockDecorator}}, { "config_format": { - gu.ConfigFormat.SONICYANG: None, - gu.ConfigFormat.CONFIGDB: gu.ConfigDbDecorator + gu.ConfigFormat.SONICYANG: gu.SonicYangDecorator, + gu.ConfigFormat.CONFIGDB: None, } }, ] @@ -956,49 +1033,49 @@ def test_list_checkpoints__calls_decorated_rollbacker(self): self.decorated_config_rollbacker.list_checkpoints.assert_called_once() self.assertListEqual(expected, actual) -class TestConfigDbDecorator(unittest.TestCase): - def test_apply__converts_to_yang_and_calls_decorated_class(self): +class TestSonicYangDecorator(unittest.TestCase): + def test_apply__converts_to_config_db_and_calls_decorated_class(self): # Arrange - config_db_decorator = self.__create_config_db_decorator() + sonic_yang_decorator = self.__create_sonic_yang_decorator() # Act - config_db_decorator.apply(Files.SINGLE_OPERATION_CONFIG_DB_PATCH) + sonic_yang_decorator.apply(Files.SINGLE_OPERATION_SONIC_YANG_PATCH) # Assert - config_db_decorator.patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.assert_has_calls( - [call(Files.SINGLE_OPERATION_CONFIG_DB_PATCH)]) - config_db_decorator.decorated_patch_applier.apply.assert_has_calls( + sonic_yang_decorator.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch.assert_has_calls( [call(Files.SINGLE_OPERATION_SONIC_YANG_PATCH)]) + sonic_yang_decorator.decorated_patch_applier.apply.assert_has_calls( + [call(Files.SINGLE_OPERATION_CONFIG_DB_PATCH)]) - def test_replace__converts_to_yang_and_calls_decorated_class(self): + def test_replace__converts_to_config_db_and_calls_decorated_class(self): # Arrange - config_db_decorator = self.__create_config_db_decorator() + sonic_yang_decorator = self.__create_sonic_yang_decorator() # Act - config_db_decorator.replace(Files.CONFIG_DB_AS_JSON) + sonic_yang_decorator.replace(Files.SONIC_YANG_AS_JSON) # Assert - config_db_decorator.config_wrapper.convert_config_db_to_sonic_yang.assert_has_calls( - [call(Files.CONFIG_DB_AS_JSON)]) - config_db_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.SONIC_YANG_AS_JSON)]) + sonic_yang_decorator.config_wrapper.convert_sonic_yang_to_config_db.assert_has_calls( + [call(Files.SONIC_YANG_AS_JSON)]) + sonic_yang_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.CONFIG_DB_AS_JSON)]) - def __create_config_db_decorator(self): + def __create_sonic_yang_decorator(self): patch_applier = Mock() - patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_SONIC_YANG_PATCH),): 0}) + patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_CONFIG_DB_PATCH),): 0}) patch_wrapper = Mock() - patch_wrapper.convert_config_db_patch_to_sonic_yang_patch.side_effect = \ - create_side_effect_dict({(str(Files.SINGLE_OPERATION_CONFIG_DB_PATCH),): \ - Files.SINGLE_OPERATION_SONIC_YANG_PATCH}) + patch_wrapper.convert_sonic_yang_patch_to_config_db_patch.side_effect = \ + create_side_effect_dict({(str(Files.SINGLE_OPERATION_SONIC_YANG_PATCH),): \ + Files.SINGLE_OPERATION_CONFIG_DB_PATCH}) config_replacer = Mock() - config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): 0}) + config_replacer.replace.side_effect = create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): 0}) config_wrapper = Mock() - config_wrapper.convert_config_db_to_sonic_yang.side_effect = \ - create_side_effect_dict({(str(Files.CONFIG_DB_AS_JSON),): Files.SONIC_YANG_AS_JSON}) + config_wrapper.convert_sonic_yang_to_config_db.side_effect = \ + create_side_effect_dict({(str(Files.SONIC_YANG_AS_JSON),): Files.CONFIG_DB_AS_JSON}) - return gu.ConfigDbDecorator(decorated_patch_applier=patch_applier, + return gu.SonicYangDecorator(decorated_patch_applier=patch_applier, decorated_config_replacer=config_replacer, patch_wrapper=patch_wrapper, config_wrapper=config_wrapper) From 9aa31dfbdec9010ea75956efce8d358836b946fc Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 14 Apr 2021 09:02:55 -0700 Subject: [PATCH 11/32] explaining why sonic_cfggen is loaded as a source --- generic_config_updater/generic_updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index fdd0e9efd4..d320ba30c8 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -6,6 +6,7 @@ from enum import Enum from swsssdk import ConfigDBConnector from imp import load_source +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter From 0cca5297c8e0c972182a0c4e7a68d5c3e52fb6a7 Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 14 Apr 2021 09:06:06 -0700 Subject: [PATCH 12/32] trying to include sonic_cfggen as a requirement --- generic_config_updater/generic_updater.py | 2 -- setup.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index d320ba30c8..d002076cdb 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -6,8 +6,6 @@ from enum import Enum from swsssdk import ConfigDBConnector from imp import load_source -# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. -load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter YANG_DIR = "/usr/local/yang-models" diff --git a/setup.py b/setup.py index 4b05f6570c..49e40fae33 100644 --- a/setup.py +++ b/setup.py @@ -167,6 +167,7 @@ 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', + 'sonic_cfggen', 'swsssdk>=2.0.1', 'tabulate==0.8.2', 'xmltodict==0.12.0', From b7ed354dc8135f9deb30962be072b207d922bd2f Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 14 Apr 2021 09:28:22 -0700 Subject: [PATCH 13/32] back to source loading sonic-cfggen --- generic_config_updater/generic_updater.py | 2 ++ setup.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index d002076cdb..d320ba30c8 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -6,6 +6,8 @@ from enum import Enum from swsssdk import ConfigDBConnector from imp import load_source +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. +load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter YANG_DIR = "/usr/local/yang-models" diff --git a/setup.py b/setup.py index 49e40fae33..d7676e8d34 100644 --- a/setup.py +++ b/setup.py @@ -157,6 +157,7 @@ 'click==7.0', 'ipaddress==1.0.23', 'jsondiff==1.2.0', + 'jsonpatch==1.32.0', 'm2crypto==0.31.0', 'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2. Can update once we no longer support Python 2 'netaddr==0.8.0', @@ -167,11 +168,9 @@ 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', - 'sonic_cfggen', 'swsssdk>=2.0.1', 'tabulate==0.8.2', 'xmltodict==0.12.0', - 'jsonpatch==1.32.0', ], setup_requires= [ 'pytest-runner', From 48448afa7d186efa19a4918ad8e6dc585c412a58 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:22:49 -0700 Subject: [PATCH 14/32] adding cli commands --- config/main.py | 134 ++++++++- tests/config_test.py | 637 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 768 insertions(+), 3 deletions(-) diff --git a/config/main.py b/config/main.py index e5a3cf6d0f..916dbfe7ae 100644 --- a/config/main.py +++ b/config/main.py @@ -3,6 +3,7 @@ import click import ipaddress import json +import jsonpatch import netaddr import netifaces import os @@ -35,6 +36,7 @@ from . import vxlan from . import plugins from .config_mgmt import ConfigMgmtDPB +from .generic_updater import GenericUpdater, ConfigFormat # mock masic APIs for unit test try: @@ -986,6 +988,132 @@ def load(filename, yes): log.log_info("'load' executing...") clicommon.run_command(command, display_cmd=True) +@config.command('apply-patch') +@click.argument('patch-file-path', type=str, required=True) +@click.option('-f', '--format', type=click.Choice([e.name.lower() for e in ConfigFormat]), + default=ConfigFormat.CONFIGDB.name.lower(), + help='format of config of the patch is either ConfigDb(ABNF) or SonicYang') +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def apply_patch(ctx, patch_file_path, format, dry_run, verbose): + """Apply given patch of updates to Config. A patch is a JsonPatch which follows rfc6902. + This command can be used do partial updates to the config with minimum disruption to running processes. + It allows addition as well as deletion of configs. The patch file represents a diff of ConfigDb(ABNF) + format or SonicYang format. + + : Path to the patch file on the file-system.""" + try: + with open(patch_file_path, 'r') as fh: + text = fh.read() + patch_as_json = json.loads(text) + patch = jsonpatch.JsonPatch(patch_as_json) + + config_format = ConfigFormat[format.upper()] + + GenericUpdater().apply_patch(patch, config_format, verbose, dry_run) + + click.secho("Patch applied successfully.", fg="cyan", underline=True) + except Exception as ex: + click.secho("Failed to apply patch", fg="red", underline=True) + ctx.fail(ex) + +@config.command() +@click.argument('target-file-path', type=str, required=True) +@click.option('-f', '--format', type=click.Choice([e.name.lower() for e in ConfigFormat]), + default=ConfigFormat.CONFIGDB.name.lower(), + help='format of target config is either ConfigDb(ABNF) or SonicYang') +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def replace(ctx, target_file_path, format, dry_run, verbose): + """Replace the whole config with the specified config. The config is replaced with minimum disruption e.g. + if ACL config is different between current and target config only ACL config is updated, and other config/services + such as DHCP will not be affected. + + **WARNING** The target config file should be the whole config, not just the part intended to be updated. + + : Path to the target file on the file-system.""" + try: + with open(target_file_path, 'r') as fh: + target_config_as_text = fh.read() + target_config = json.loads(target_config_as_text) + + config_format = ConfigFormat[format.upper()] + + GenericUpdater().replace(target_config, config_format, verbose, dry_run) + + click.secho("Config replaced successfully.", fg="cyan", underline=True) + except Exception as ex: + click.secho("Failed to replace config", fg="red", underline=True) + ctx.fail(ex) + +@config.command() +@click.argument('checkpoint-name', type=str, required=True) +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def rollback(ctx, checkpoint_name, dry_run, verbose): + """Rollback the whole config to the specified checkpoint. The config is rolled back with minimum disruption e.g. + if ACL config is different between current and checkpoint config only ACL config is updated, and other config/services + such as DHCP will not be affected. + + : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" + try: + GenericUpdater().rollback(checkpoint_name, verbose, dry_run) + + click.secho("Config rolled back successfully.", fg="cyan", underline=True) + except Exception as ex: + click.secho("Failed to rollback config", fg="red", underline=True) + ctx.fail(ex) + +@config.command() +@click.argument('checkpoint-name', type=str, required=True) +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def checkpoint(ctx, checkpoint_name, dry_run, verbose): + """Take a checkpoint of the whole current config with the specified checkpoint name. + + : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" + try: + GenericUpdater().checkpoint(checkpoint_name, verbose, dry_run) + + click.secho("Checkpoint created successfully.", fg="cyan", underline=True) + except Exception as ex: + click.secho("Failed to create a config checkpoint", fg="red", underline=True) + ctx.fail(ex) + +@config.command('delete-checkpoint') +@click.argument('checkpoint-name', type=str, required=True) +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def delete_checkpoint(ctx, checkpoint_name, dry_run, verbose): + """Delete a checkpoint with the specified checkpoint name. + + : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" + try: + GenericUpdater().delete_checkpoint(checkpoint_name, verbose, dry_run) + + click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) + except Exception as ex: + click.secho("Failed to delete config checkpoint", fg="red", underline=True) + ctx.fail(ex) + +@config.command('list-checkpoints') +@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') +@click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') +@click.pass_context +def list_checkpoints(ctx, dry_run, verbose): + """List the config checkpoints available.""" + try: + checkpoints_list = GenericUpdater().list_checkpoints(verbose, dry_run) + formatted_output = json.dumps(checkpoints_list, indent=4) + click.echo(formatted_output) + except Exception as ex: + click.secho("Failed to list config checkpoints", fg="red", underline=True) + ctx.fail(ex) @config.command() @click.option('-y', '--yes', is_flag=True) @@ -4415,9 +4543,9 @@ def delete(ctx): # Load plugins and register them -helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, config) +# helper = util_base.UtilHelper() +# for plugin in helper.load_plugins(plugins): +# helper.register_plugin(plugin, config) if __name__ == '__main__': diff --git a/tests/config_test.py b/tests/config_test.py index 381ca80304..8c5be79e35 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3,6 +3,8 @@ import os import traceback import json +import jsonpatch +import unittest from unittest import mock import click @@ -11,6 +13,17 @@ from sonic_py_common import device_info from utilities_common.db import Db +import sys +test_path = os.path.dirname(os.path.abspath(__file__)) +modules_path = os.path.dirname(test_path) +generic_config_updater_path = os.path.join(modules_path, "generic_config_updater") +sys.path.insert(0, modules_path) +sys.path.insert(0, generic_config_updater_path) + +from generic_updater import ConfigFormat + +import config.main as config + load_minigraph_command_output="""\ Stopping SONiC target ... Running command: /usr/local/bin/sonic-cfggen -H -m --write-to-db @@ -150,3 +163,627 @@ def teardown_class(cls): from .mock_tables import mock_single_asic importlib.reload(mock_single_asic) dbconnector.load_namespace_config() + +class TestGenericUpdateCommands(unittest.TestCase): + def setUp(self): + os.environ['UTILITIES_UNIT_TESTING'] = "1" + self.runner = CliRunner() + self.any_patch_as_json = [{"op":"remove", "path":"/PORT"}] + self.any_patch = jsonpatch.JsonPatch(self.any_patch_as_json) + self.any_patch_as_text = json.dumps(self.any_patch_as_json) + self.any_path = '/usr/admin/patch.json-patch' + self.any_target_config = {"PORT": {}} + self.any_target_config_as_text = json.dumps(self.any_target_config) + self.any_checkpoint_name = "any_checkpoint_name" + self.any_checkpoints_list = ["checkpoint1", "checkpoint2", "checkpoint3"] + self.any_checkpoints_list_as_text = json.dumps(self.any_checkpoints_list, indent=4) + + def test_apply_patch__no_params__get_required_params_error_msg(self): + # Arrange + unexpected_exit_code = 0 + expected_output = "Error: Missing argument \"PATCH_FILE_PATH\"" + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"]) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_apply_patch__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_apply_patch__only_required_params__default_values_used_for_optional_params(self): + # Arrange + expected_exit_code = 0 + expected_output = "Patch applied successfully" + expected_call_with_default_values = mock.call(self.any_patch, ConfigFormat.CONFIGDB, False, False) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_patch_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"], [self.any_path], catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.apply_patch.assert_called_once() + mock_generic_updater.apply_patch.assert_has_calls([expected_call_with_default_values]) + + def test_apply_patch__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = "Patch applied successfully" + expected_call_with_non_default_values = mock.call(self.any_patch, ConfigFormat.SONICYANG, True, True) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_patch_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"], + [self.any_path, + "--format", ConfigFormat.SONICYANG.name.lower(), + "--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.apply_patch.assert_called_once() + mock_generic_updater.apply_patch.assert_has_calls([expected_call_with_non_default_values]) + + def test_apply_patch__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.apply_patch.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_patch_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"], + [self.any_path], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_apply_patch__optional_parameters_passed_correctly(self): + self.validate_apply_patch_optional_parameter( + ["--format", ConfigFormat.SONICYANG.name.lower()], + mock.call(self.any_patch, ConfigFormat.SONICYANG, False, False)) + self.validate_apply_patch_optional_parameter( + ["--verbose"], + mock.call(self.any_patch, ConfigFormat.CONFIGDB, True, False)) + self.validate_apply_patch_optional_parameter( + ["--dry-run"], + mock.call(self.any_patch, ConfigFormat.CONFIGDB, False, True)) + + def validate_apply_patch_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = "Patch applied successfully" + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_patch_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["apply-patch"], + [self.any_path] + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.apply_patch.assert_called_once() + mock_generic_updater.apply_patch.assert_has_calls([expected_call]) + + def test_replace__no_params__get_required_params_error_msg(self): + # Arrange + unexpected_exit_code = 0 + expected_output = "Error: Missing argument \"TARGET_FILE_PATH\"" + + # Act + result = self.runner.invoke(config.config.commands["replace"]) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_replace__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["replace"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_replace__only_required_params__default_values_used_for_optional_params(self): + # Arrange + expected_exit_code = 0 + expected_output = "Config replaced successfully" + expected_call_with_default_values = mock.call(self.any_target_config, ConfigFormat.CONFIGDB, False, False) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_target_config_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["replace"], [self.any_path], catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.replace.assert_called_once() + mock_generic_updater.replace.assert_has_calls([expected_call_with_default_values]) + + def test_replace__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = "Config replaced successfully" + expected_call_with_non_default_values = mock.call(self.any_target_config, ConfigFormat.SONICYANG, True, True) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_target_config_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["replace"], + [self.any_path, + "--format", ConfigFormat.SONICYANG.name.lower(), + "--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.replace.assert_called_once() + mock_generic_updater.replace.assert_has_calls([expected_call_with_non_default_values]) + + def test_replace__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.replace.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_target_config_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["replace"], + [self.any_path], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_replace__optional_parameters_passed_correctly(self): + self.validate_replace_optional_parameter( + ["--format", ConfigFormat.SONICYANG.name.lower()], + mock.call(self.any_target_config, ConfigFormat.SONICYANG, False, False)) + self.validate_replace_optional_parameter( + ["--verbose"], + mock.call(self.any_target_config, ConfigFormat.CONFIGDB, True, False)) + self.validate_replace_optional_parameter( + ["--dry-run"], + mock.call(self.any_target_config, ConfigFormat.CONFIGDB, False, True)) + + def validate_replace_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = "Config replaced successfully" + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + with mock.patch('builtins.open', mock.mock_open(read_data=self.any_target_config_as_text)): + + # Act + result = self.runner.invoke(config.config.commands["replace"], + [self.any_path] + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.replace.assert_called_once() + mock_generic_updater.replace.assert_has_calls([expected_call]) + + def test_rollback__no_params__get_required_params_error_msg(self): + # Arrange + unexpected_exit_code = 0 + expected_output = "Error: Missing argument \"CHECKPOINT_NAME\"" + + # Act + result = self.runner.invoke(config.config.commands["rollback"]) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_rollback__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["rollback"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_rollback__only_required_params__default_values_used_for_optional_params(self): + # Arrange + expected_exit_code = 0 + expected_output = "Config rolled back successfully" + expected_call_with_default_values = mock.call(self.any_checkpoint_name, False, False) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["rollback"], [self.any_checkpoint_name], catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.rollback.assert_called_once() + mock_generic_updater.rollback.assert_has_calls([expected_call_with_default_values]) + + def test_rollback__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = "Config rolled back successfully" + expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True, True) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["rollback"], + [self.any_checkpoint_name, + "--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.rollback.assert_called_once() + mock_generic_updater.rollback.assert_has_calls([expected_call_with_non_default_values]) + + def test_rollback__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.rollback.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["rollback"], + [self.any_checkpoint_name], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_rollback__optional_parameters_passed_correctly(self): + self.validate_rollback_optional_parameter( + ["--verbose"], + mock.call(self.any_checkpoint_name, True, False)) + self.validate_rollback_optional_parameter( + ["--dry-run"], + mock.call(self.any_checkpoint_name, False, True)) + + def validate_rollback_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = "Config rolled back successfully" + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["rollback"], + [self.any_checkpoint_name] + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.rollback.assert_called_once() + mock_generic_updater.rollback.assert_has_calls([expected_call]) + + def test_checkpoint__no_params__get_required_params_error_msg(self): + # Arrange + unexpected_exit_code = 0 + expected_output = "Error: Missing argument \"CHECKPOINT_NAME\"" + + # Act + result = self.runner.invoke(config.config.commands["checkpoint"]) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_checkpoint__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["checkpoint"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_checkpoint__only_required_params__default_values_used_for_optional_params(self): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint created successfully" + expected_call_with_default_values = mock.call(self.any_checkpoint_name, False, False) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["checkpoint"], [self.any_checkpoint_name], catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.checkpoint.assert_called_once() + mock_generic_updater.checkpoint.assert_has_calls([expected_call_with_default_values]) + + def test_checkpoint__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint created successfully" + expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True, True) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["checkpoint"], + [self.any_checkpoint_name, + "--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.checkpoint.assert_called_once() + mock_generic_updater.checkpoint.assert_has_calls([expected_call_with_non_default_values]) + + def test_checkpoint__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.checkpoint.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["checkpoint"], + [self.any_checkpoint_name], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_checkpoint__optional_parameters_passed_correctly(self): + self.validate_checkpoint_optional_parameter( + ["--verbose"], + mock.call(self.any_checkpoint_name, True, False)) + self.validate_checkpoint_optional_parameter( + ["--dry-run"], + mock.call(self.any_checkpoint_name, False, True)) + + def validate_checkpoint_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint created successfully" + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["checkpoint"], + [self.any_checkpoint_name] + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.checkpoint.assert_called_once() + mock_generic_updater.checkpoint.assert_has_calls([expected_call]) + + def test_delete_checkpoint__no_params__get_required_params_error_msg(self): + # Arrange + unexpected_exit_code = 0 + expected_output = "Error: Missing argument \"CHECKPOINT_NAME\"" + + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"]) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_delete_checkpoint__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_delete_checkpoint__only_required_params__default_values_used_for_optional_params(self): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint deleted successfully" + expected_call_with_default_values = mock.call(self.any_checkpoint_name, False, False) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"], [self.any_checkpoint_name], catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.delete_checkpoint.assert_called_once() + mock_generic_updater.delete_checkpoint.assert_has_calls([expected_call_with_default_values]) + + def test_delete_checkpoint__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint deleted successfully" + expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True, True) + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"], + [self.any_checkpoint_name, + "--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.delete_checkpoint.assert_called_once() + mock_generic_updater.delete_checkpoint.assert_has_calls([expected_call_with_non_default_values]) + + def test_delete_checkpoint__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.delete_checkpoint.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"], + [self.any_checkpoint_name], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_delete_checkpoint__optional_parameters_passed_correctly(self): + self.validate_delete_checkpoint_optional_parameter( + ["--verbose"], + mock.call(self.any_checkpoint_name, True, False)) + self.validate_delete_checkpoint_optional_parameter( + ["--dry-run"], + mock.call(self.any_checkpoint_name, False, True)) + + def validate_delete_checkpoint_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = "Checkpoint deleted successfully" + mock_generic_updater = mock.Mock() + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["delete-checkpoint"], + [self.any_checkpoint_name] + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.delete_checkpoint.assert_called_once() + mock_generic_updater.delete_checkpoint.assert_has_calls([expected_call]) + + def test_list_checkpoints__help__gets_help_msg(self): + # Arrange + expected_exit_code = 0 + expected_output = "Options:" # this indicates the options are listed + + # Act + result = self.runner.invoke(config.config.commands["list-checkpoints"], ['--help']) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + + def test_list_checkpoints__all_optional_params_non_default__non_default_values_used(self): + # Arrange + expected_exit_code = 0 + expected_output = self.any_checkpoints_list_as_text + expected_call_with_non_default_values = mock.call(True, True) + mock_generic_updater = mock.Mock() + mock_generic_updater.list_checkpoints.return_value = self.any_checkpoints_list + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["list-checkpoints"], + ["--dry-run", + "--verbose"], + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.list_checkpoints.assert_called_once() + mock_generic_updater.list_checkpoints.assert_has_calls([expected_call_with_non_default_values]) + + def test_list_checkpoints__exception_thrown__error_displayed_error_code_returned(self): + # Arrange + unexpected_exit_code = 0 + any_error_message = "any_error_message" + mock_generic_updater = mock.Mock() + mock_generic_updater.list_checkpoints.side_effect = Exception(any_error_message) + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + + # Act + result = self.runner.invoke(config.config.commands["list-checkpoints"], + catch_exceptions=False) + + # Assert + self.assertNotEqual(unexpected_exit_code, result.exit_code) + self.assertTrue(any_error_message in result.output) + + def test_list_checkpoints__optional_parameters_passed_correctly(self): + self.validate_list_checkpoints_optional_parameter( + ["--verbose"], + mock.call(True, False)) + self.validate_list_checkpoints_optional_parameter( + ["--dry-run"], + mock.call(False, True)) + + def validate_list_checkpoints_optional_parameter(self, param_args, expected_call): + # Arrange + expected_exit_code = 0 + expected_output = self.any_checkpoints_list_as_text + mock_generic_updater = mock.Mock() + mock_generic_updater.list_checkpoints.return_value = self.any_checkpoints_list + with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): + # Act + result = self.runner.invoke(config.config.commands["list-checkpoints"], + param_args, + catch_exceptions=False) + + # Assert + self.assertEqual(expected_exit_code, result.exit_code) + self.assertTrue(expected_output in result.output) + mock_generic_updater.list_checkpoints.assert_called_once() + mock_generic_updater.list_checkpoints.assert_has_calls([expected_call]) From a3ca658e5ab61534d01ddf35b99b258a7d34ed18 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:39:44 -0700 Subject: [PATCH 15/32] removing dry-run option from checkpoint, delete-checkpoint, list-checkpoints --- config/main.py | 15 ++++------ generic_config_updater/generic_updater.py | 14 ++++----- tests/config_test.py | 30 ++++++------------- .../generic_updater_test.py | 12 ++++---- 4 files changed, 28 insertions(+), 43 deletions(-) diff --git a/config/main.py b/config/main.py index 916dbfe7ae..093730b360 100644 --- a/config/main.py +++ b/config/main.py @@ -1069,15 +1069,14 @@ def rollback(ctx, checkpoint_name, dry_run, verbose): @config.command() @click.argument('checkpoint-name', type=str, required=True) -@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def checkpoint(ctx, checkpoint_name, dry_run, verbose): +def checkpoint(ctx, checkpoint_name, verbose): """Take a checkpoint of the whole current config with the specified checkpoint name. : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().checkpoint(checkpoint_name, verbose, dry_run) + GenericUpdater().checkpoint(checkpoint_name, verbose) click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1086,15 +1085,14 @@ def checkpoint(ctx, checkpoint_name, dry_run, verbose): @config.command('delete-checkpoint') @click.argument('checkpoint-name', type=str, required=True) -@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def delete_checkpoint(ctx, checkpoint_name, dry_run, verbose): +def delete_checkpoint(ctx, checkpoint_name, verbose): """Delete a checkpoint with the specified checkpoint name. : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().delete_checkpoint(checkpoint_name, verbose, dry_run) + GenericUpdater().delete_checkpoint(checkpoint_name, verbose) click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1102,13 +1100,12 @@ def delete_checkpoint(ctx, checkpoint_name, dry_run, verbose): ctx.fail(ex) @config.command('list-checkpoints') -@click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config/disk states') @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def list_checkpoints(ctx, dry_run, verbose): +def list_checkpoints(ctx, verbose): """List the config checkpoints available.""" try: - checkpoints_list = GenericUpdater().list_checkpoints(verbose, dry_run) + checkpoints_list = GenericUpdater().list_checkpoints(verbose) formatted_output = json.dumps(checkpoints_list, indent=4) click.echo(formatted_output) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index d320ba30c8..fbe2526cdc 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -446,7 +446,7 @@ def create_config_replacer(self, config_format, verbose, dry_run): return config_replacer - def create_config_rollbacker(self, verbose, dry_run): + def create_config_rollbacker(self, verbose, dry_run=False): self.init_verbose_logging(verbose) config_wrapper = self.get_config_wrapper(dry_run) @@ -490,14 +490,14 @@ def rollback(self, checkpoint_name, verbose, dry_run): config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) config_rollbacker.rollback(checkpoint_name) - def checkpoint(self, checkpoint_name, verbose, dry_run): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + def checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.checkpoint(checkpoint_name) - def delete_checkpoint(self, checkpoint_name, verbose, dry_run): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + def delete_checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.delete_checkpoint(checkpoint_name) - def list_checkpoints(self, verbose, dry_run): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run) + def list_checkpoints(self, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) return config_rollbacker.list_checkpoints() diff --git a/tests/config_test.py b/tests/config_test.py index 8c5be79e35..5b921e81fc 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -535,7 +535,7 @@ def test_checkpoint__only_required_params__default_values_used_for_optional_para # Arrange expected_exit_code = 0 expected_output = "Checkpoint created successfully" - expected_call_with_default_values = mock.call(self.any_checkpoint_name, False, False) + expected_call_with_default_values = mock.call(self.any_checkpoint_name, False) mock_generic_updater = mock.Mock() with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): # Act @@ -551,14 +551,13 @@ def test_checkpoint__all_optional_params_non_default__non_default_values_used(se # Arrange expected_exit_code = 0 expected_output = "Checkpoint created successfully" - expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True, True) + expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True) mock_generic_updater = mock.Mock() with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): # Act result = self.runner.invoke(config.config.commands["checkpoint"], [self.any_checkpoint_name, - "--dry-run", "--verbose"], catch_exceptions=False) @@ -588,10 +587,7 @@ def test_checkpoint__exception_thrown__error_displayed_error_code_returned(self) def test_checkpoint__optional_parameters_passed_correctly(self): self.validate_checkpoint_optional_parameter( ["--verbose"], - mock.call(self.any_checkpoint_name, True, False)) - self.validate_checkpoint_optional_parameter( - ["--dry-run"], - mock.call(self.any_checkpoint_name, False, True)) + mock.call(self.any_checkpoint_name, True)) def validate_checkpoint_optional_parameter(self, param_args, expected_call): # Arrange @@ -638,7 +634,7 @@ def test_delete_checkpoint__only_required_params__default_values_used_for_option # Arrange expected_exit_code = 0 expected_output = "Checkpoint deleted successfully" - expected_call_with_default_values = mock.call(self.any_checkpoint_name, False, False) + expected_call_with_default_values = mock.call(self.any_checkpoint_name, False) mock_generic_updater = mock.Mock() with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): # Act @@ -654,14 +650,13 @@ def test_delete_checkpoint__all_optional_params_non_default__non_default_values_ # Arrange expected_exit_code = 0 expected_output = "Checkpoint deleted successfully" - expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True, True) + expected_call_with_non_default_values = mock.call(self.any_checkpoint_name, True) mock_generic_updater = mock.Mock() with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): # Act result = self.runner.invoke(config.config.commands["delete-checkpoint"], [self.any_checkpoint_name, - "--dry-run", "--verbose"], catch_exceptions=False) @@ -691,10 +686,7 @@ def test_delete_checkpoint__exception_thrown__error_displayed_error_code_returne def test_delete_checkpoint__optional_parameters_passed_correctly(self): self.validate_delete_checkpoint_optional_parameter( ["--verbose"], - mock.call(self.any_checkpoint_name, True, False)) - self.validate_delete_checkpoint_optional_parameter( - ["--dry-run"], - mock.call(self.any_checkpoint_name, False, True)) + mock.call(self.any_checkpoint_name, True)) def validate_delete_checkpoint_optional_parameter(self, param_args, expected_call): # Arrange @@ -729,15 +721,14 @@ def test_list_checkpoints__all_optional_params_non_default__non_default_values_u # Arrange expected_exit_code = 0 expected_output = self.any_checkpoints_list_as_text - expected_call_with_non_default_values = mock.call(True, True) + expected_call_with_non_default_values = mock.call(True) mock_generic_updater = mock.Mock() mock_generic_updater.list_checkpoints.return_value = self.any_checkpoints_list with mock.patch('config.main.GenericUpdater', return_value=mock_generic_updater): # Act result = self.runner.invoke(config.config.commands["list-checkpoints"], - ["--dry-run", - "--verbose"], + ["--verbose"], catch_exceptions=False) # Assert @@ -765,10 +756,7 @@ def test_list_checkpoints__exception_thrown__error_displayed_error_code_returned def test_list_checkpoints__optional_parameters_passed_correctly(self): self.validate_list_checkpoints_optional_parameter( ["--verbose"], - mock.call(True, False)) - self.validate_list_checkpoints_optional_parameter( - ["--dry-run"], - mock.call(False, True)) + mock.call(True)) def validate_list_checkpoints_optional_parameter(self, param_args, expected_call): # Arrange diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 430197499f..268155e0a3 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -927,12 +927,12 @@ def test_checkpoint__creates_rollbacker_and_checkpoint(self): factory = Mock() factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + create_side_effect_dict({(str(self.any_verbose),): config_rollbacker}) generic_updater = gu.GenericUpdater(factory) # Act - generic_updater.checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + generic_updater.checkpoint(self.any_checkpoint_name, self.any_verbose) # Assert config_rollbacker.checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) @@ -944,12 +944,12 @@ def test_delete_checkpoint__creates_rollbacker_and_deletes_checkpoint(self): factory = Mock() factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + create_side_effect_dict({(str(self.any_verbose),): config_rollbacker}) generic_updater = gu.GenericUpdater(factory) # Act - generic_updater.delete_checkpoint(self.any_checkpoint_name, self.any_verbose, self.any_dry_run) + generic_updater.delete_checkpoint(self.any_checkpoint_name, self.any_verbose) # Assert config_rollbacker.delete_checkpoint.assert_has_calls([call(self.any_checkpoint_name)]) @@ -961,14 +961,14 @@ def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): factory = Mock() factory.create_config_rollbacker.side_effect = \ - create_side_effect_dict({(str(self.any_verbose), str(self.any_dry_run),): config_rollbacker}) + create_side_effect_dict({(str(self.any_verbose),): config_rollbacker}) generic_updater = gu.GenericUpdater(factory) expected = self.any_checkpoints_list # Act - actual = generic_updater.list_checkpoints(self.any_verbose, self.any_dry_run) + actual = generic_updater.list_checkpoints(self.any_verbose) # Assert self.assertListEqual(expected, actual) From e0d6d43ccaf229208863f9b097924295c22a90fe Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:47:46 -0700 Subject: [PATCH 16/32] minor fix --- tests/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config_test.py b/tests/config_test.py index 5b921e81fc..f5e959b5f3 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -4,6 +4,7 @@ import traceback import json import jsonpatch +import sys import unittest from unittest import mock @@ -13,7 +14,6 @@ from sonic_py_common import device_info from utilities_common.db import Db -import sys test_path = os.path.dirname(os.path.abspath(__file__)) modules_path = os.path.dirname(test_path) generic_config_updater_path = os.path.join(modules_path, "generic_config_updater") From 99b6796260f4bbb232488754811f6ad73ca2d6b3 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:48:54 -0700 Subject: [PATCH 17/32] minor fix --- config/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/main.py b/config/main.py index 093730b360..09ea746820 100644 --- a/config/main.py +++ b/config/main.py @@ -4540,9 +4540,9 @@ def delete(ctx): # Load plugins and register them -# helper = util_base.UtilHelper() -# for plugin in helper.load_plugins(plugins): -# helper.register_plugin(plugin, config) +helper = util_base.UtilHelper() +for plugin in helper.load_plugins(plugins): + helper.register_plugin(plugin, config) if __name__ == '__main__': From d42e16f6583c49ece1e45e5aaaa18fc5c71f851b Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:52:10 -0700 Subject: [PATCH 18/32] adding sonic-cfggen to install_requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d7676e8d34..6e44f2eb29 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ 'pexpect==4.8.0', 'pyroute2==0.5.14', 'requests==2.25.0', + 'sonic-cfggen', 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', From ecd651968357dc562373c4df0c48e19a476595c9 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 17:59:50 -0700 Subject: [PATCH 19/32] fix generic_config_updater path in test file --- tests/generic_config_updater/generic_updater_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 268155e0a3..e589f71817 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -6,7 +6,13 @@ import unittest from imp import load_source from unittest.mock import Mock, call -sys.path.insert(1, '../../generic_config_updater') + +generic_config_updater_test_path = os.path.dirname(os.path.abspath(__file__)) +test_path = os.path.dirname(generic_config_updater_test_path) +modules_path = os.path.dirname(test_path) +generic_config_updater_path = os.path.join(modules_path, "generic_config_updater") +sys.path.insert(0, generic_config_updater_path) + import generic_updater as gu class MockSideEffectDict: From f50498048c04f0007815c2b343d563a472284459 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 18:13:28 -0700 Subject: [PATCH 20/32] trying sonic_cfggen instead of sonic-cfggen --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e44f2eb29..85a40d0ebb 100644 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ 'pexpect==4.8.0', 'pyroute2==0.5.14', 'requests==2.25.0', - 'sonic-cfggen', + 'sonic_cfggen', 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', From 9e0519d9392842887a3f7cb37d20f00f53bc7d86 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 18:19:54 -0700 Subject: [PATCH 21/32] removing sonic-cfggen from install_requires --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 85a40d0ebb..d7676e8d34 100644 --- a/setup.py +++ b/setup.py @@ -165,7 +165,6 @@ 'pexpect==4.8.0', 'pyroute2==0.5.14', 'requests==2.25.0', - 'sonic_cfggen', 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', From 757d306fbc16e5be3ab1307a81fa7beed48f4531 Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 18:39:40 -0700 Subject: [PATCH 22/32] make generic_config_updater a pkg --- config/main.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/main.py b/config/main.py index 09ea746820..e6208f83ff 100644 --- a/config/main.py +++ b/config/main.py @@ -12,6 +12,7 @@ import sys import time +from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat from socket import AF_INET, AF_INET6 from minigraph import parse_device_desc_xml from portconfig import get_child_ports @@ -36,7 +37,6 @@ from . import vxlan from . import plugins from .config_mgmt import ConfigMgmtDPB -from .generic_updater import GenericUpdater, ConfigFormat # mock masic APIs for unit test try: diff --git a/setup.py b/setup.py index d7676e8d34..9bbf908f9b 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ 'counterpoll', 'crm', 'debug', + 'generic_config_updater', 'pfcwd', 'sfputil', 'ssdutil', From 038ec673ff8f8b8417dd5489427e63326f1330cb Mon Sep 17 00:00:00 2001 From: ghooo Date: Thu, 15 Apr 2021 19:08:59 -0700 Subject: [PATCH 23/32] use generic_config_updater pkg in uts --- tests/config_test.py | 8 +------- tests/generic_config_updater/generic_updater_test.py | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index f5e959b5f3..0e13e0f6e8 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -14,13 +14,7 @@ from sonic_py_common import device_info from utilities_common.db import Db -test_path = os.path.dirname(os.path.abspath(__file__)) -modules_path = os.path.dirname(test_path) -generic_config_updater_path = os.path.join(modules_path, "generic_config_updater") -sys.path.insert(0, modules_path) -sys.path.insert(0, generic_config_updater_path) - -from generic_updater import ConfigFormat +from generic_config_updater.generic_updater import ConfigFormat import config.main as config diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index e589f71817..c2f3b3c3b7 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -7,13 +7,7 @@ from imp import load_source from unittest.mock import Mock, call -generic_config_updater_test_path = os.path.dirname(os.path.abspath(__file__)) -test_path = os.path.dirname(generic_config_updater_test_path) -modules_path = os.path.dirname(test_path) -generic_config_updater_path = os.path.join(modules_path, "generic_config_updater") -sys.path.insert(0, generic_config_updater_path) - -import generic_updater as gu +import generic_config_updater.generic_updater as gu class MockSideEffectDict: def __init__(self, map): From cf6be42985626728ede33417a57631d879f7d3cc Mon Sep 17 00:00:00 2001 From: ghooo Date: Fri, 16 Apr 2021 23:05:24 -0700 Subject: [PATCH 24/32] removing load_source of sonic-cfggen --- config/main.py | 10 +- generic_config_updater/generic_updater.py | 39 +++--- tests/config_test.py | 10 +- .../files/config_db_as_dict.py-dict | 121 ------------------ .../generic_updater_test.py | 109 ++++++---------- 5 files changed, 74 insertions(+), 215 deletions(-) delete mode 100644 tests/generic_config_updater/files/config_db_as_dict.py-dict diff --git a/config/main.py b/config/main.py index e6208f83ff..fbbb73e71a 100644 --- a/config/main.py +++ b/config/main.py @@ -828,7 +828,7 @@ def cache_arp_entries(): if filter_err: click.echo("Could not filter FDB entries prior to reloading") success = False - + # If we are able to successfully cache ARP table info, signal SWSS to restore from our cache # by creating /host/config-reload/needs-restore if success: @@ -1008,7 +1008,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, verbose): text = fh.read() patch_as_json = json.loads(text) patch = jsonpatch.JsonPatch(patch_as_json) - + config_format = ConfigFormat[format.upper()] GenericUpdater().apply_patch(patch, config_format, verbose, dry_run) @@ -1038,7 +1038,7 @@ def replace(ctx, target_file_path, format, dry_run, verbose): with open(target_file_path, 'r') as fh: target_config_as_text = fh.read() target_config = json.loads(target_config_as_text) - + config_format = ConfigFormat[format.upper()] GenericUpdater().replace(target_config, config_format, verbose, dry_run) @@ -2705,8 +2705,8 @@ def add(ctx, interface_name, ip_addr, gw): if interface_name is None: ctx.fail("'interface_name' is None!") - # Add a validation to check this interface is not a member in vlan before - # changing it to a router port + # Add a validation to check this interface is not a member in vlan before + # changing it to a router port vlan_member_table = config_db.get_table('VLAN_MEMBER') if (interface_is_in_vlan(vlan_member_table, interface_name)): click.echo("Interface {} is a member of vlan\nAborting!".format(interface_name)) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index fbe2526cdc..c64aefb0f7 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -3,18 +3,20 @@ import sonic_yang import os import copy +import subprocess from enum import Enum -from swsssdk import ConfigDBConnector -from imp import load_source -# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. -load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') -from sonic_cfggen import deep_update, FormatConverter YANG_DIR = "/usr/local/yang-models" CHECKPOINTS_DIR = "/etc/sonic/checkpoints" CHECKPOINT_EXT = ".cp.json" -class ConfigNotCompletelyUpdatedError(Exception): +class GenericUpdaterException(Exception): + pass + +class ConfigNotCompletelyUpdatedError(GenericUpdaterException): + pass + +class FailedToGetRunningConfigError(GenericUpdaterException): pass class JsonChange: @@ -41,15 +43,22 @@ def apply(self, change): pass class ConfigWrapper: - def __init__(self, default_config_db_connector = None, yang_dir = YANG_DIR): - self.default_config_db_connector = default_config_db_connector + def __init__(self, yang_dir = YANG_DIR): self.yang_dir = YANG_DIR def get_config_db_as_json(self): - config_db = self._create_and_connect_config_db() - data = dict() - deep_update(data, FormatConverter.db_to_output(config_db.get_config())) - return FormatConverter.to_serialized(data) + text = self._get_config_db_as_text() + return json.loads(text) + + def _get_config_db_as_text(self): + # TODO: Getting configs from CLI is very slow, need to get it from sonic-cffgen directly + cmd = "show runningconfiguration all" + result = subprocess.Popen(cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + text, err = result.communicate() + return_code = result.returncode + if return_code: # non-zero means failure + raise FailedToGetRunningConfig(f"Return code: {return_code}, Error: {err}") + return text def get_sonic_yang_as_json(self): config_db_json = self.get_config_db_as_json() @@ -194,8 +203,8 @@ def convert_sonic_yang_patch_to_config_db_patch(self, patch): return self.generate_patch(current_config_db, target_config_db) class ConfigFormat(Enum): - SONICYANG = 1 - CONFIGDB = 2 + CONFIGDB = 1 + SONICYANG = 2 class PatchApplier: def __init__(self, @@ -385,7 +394,7 @@ def __init__(self, def apply(self, patch): self.execute_write_action(Decorator.apply, self, patch) - + def replace(self, target_config): self.execute_write_action(Decorator.replace, self, target_config) diff --git a/tests/config_test.py b/tests/config_test.py index 0e13e0f6e8..e8744bd28a 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -265,7 +265,7 @@ def test_apply_patch__optional_parameters_passed_correctly(self): self.validate_apply_patch_optional_parameter( ["--dry-run"], mock.call(self.any_patch, ConfigFormat.CONFIGDB, False, True)) - + def validate_apply_patch_optional_parameter(self, param_args, expected_call): # Arrange expected_exit_code = 0 @@ -378,7 +378,7 @@ def test_replace__optional_parameters_passed_correctly(self): self.validate_replace_optional_parameter( ["--dry-run"], mock.call(self.any_target_config, ConfigFormat.CONFIGDB, False, True)) - + def validate_replace_optional_parameter(self, param_args, expected_call): # Arrange expected_exit_code = 0 @@ -483,7 +483,7 @@ def test_rollback__optional_parameters_passed_correctly(self): self.validate_rollback_optional_parameter( ["--dry-run"], mock.call(self.any_checkpoint_name, False, True)) - + def validate_rollback_optional_parameter(self, param_args, expected_call): # Arrange expected_exit_code = 0 @@ -582,7 +582,7 @@ def test_checkpoint__optional_parameters_passed_correctly(self): self.validate_checkpoint_optional_parameter( ["--verbose"], mock.call(self.any_checkpoint_name, True)) - + def validate_checkpoint_optional_parameter(self, param_args, expected_call): # Arrange expected_exit_code = 0 @@ -681,7 +681,7 @@ def test_delete_checkpoint__optional_parameters_passed_correctly(self): self.validate_delete_checkpoint_optional_parameter( ["--verbose"], mock.call(self.any_checkpoint_name, True)) - + def validate_delete_checkpoint_optional_parameter(self, param_args, expected_call): # Arrange expected_exit_code = 0 diff --git a/tests/generic_config_updater/files/config_db_as_dict.py-dict b/tests/generic_config_updater/files/config_db_as_dict.py-dict deleted file mode 100644 index f83ee09f4f..0000000000 --- a/tests/generic_config_updater/files/config_db_as_dict.py-dict +++ /dev/null @@ -1,121 +0,0 @@ -{ - "VLAN_MEMBER": { - ("Vlan1000", "Ethernet0"): { - "tagging_mode": "untagged" - }, - ("Vlan1000", "Ethernet4"): { - "tagging_mode": "untagged" - }, - ("Vlan1000", "Ethernet8"): { - "tagging_mode": "untagged" - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "ACL_TABLE": { - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - }, - "DATAACL": { - "policy_desc": "DATAACL", - "ports": [ - "Ethernet4" - ], - "stage": "ingress", - "type": "L3" - }, - "EVERFLOW": { - "policy_desc": "EVERFLOW", - "ports": [ - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRROR" - }, - "EVERFLOWV6": { - "policy_desc": "EVERFLOWV6", - "ports": [ - "Ethernet4", - "Ethernet8" - ], - "stage": "ingress", - "type": "MIRRORV6" - }, - "NO-NSW-PACL-V4": { - "type": "L3", - "policy_desc": "NO-NSW-PACL-V4", - "ports": [ - "Ethernet0" - ] - } - }, - "VLAN": { - "Vlan1000": { - "vlanid": "1000", - "dhcp_servers": [ - "192.0.0.1", - "192.0.0.2", - "192.0.0.3", - "192.0.0.4" - ] - } - }, - "PORT": { - "Ethernet0": { - "alias": "Eth1", - "lanes": "65, 66, 67, 68", - "description": "Ethernet0 100G link", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "fortyGigE0/4", - "description": "Servers0:eth0", - "index": "1", - "lanes": "29,30,31,32", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "fortyGigE0/8", - "description": "Servers1:eth0", - "index": "2", - "lanes": "33,34,35,36", - "mtu": "9100", - "pfc_asym": "off", - "speed": "40000" - } - }, - "WRED_PROFILE": { - "AZURE_LOSSLESS": { - "wred_green_enable": "true", - "wred_yellow_enable": "true", - "wred_red_enable": "true", - "ecn": "ecn_all", - "green_max_threshold": "2097152", - "green_min_threshold": "1048576", - "yellow_max_threshold": "2097152", - "yellow_min_threshold": "1048576", - "red_max_threshold": "2097152", - "red_min_threshold": "1048576", - "green_drop_probability": "5", - "yellow_drop_probability": "5", - "red_drop_probability": "5" - } - } -} diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index c2f3b3c3b7..f430e2579a 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -4,8 +4,7 @@ import shutil import sys import unittest -from imp import load_source -from unittest.mock import Mock, call +from unittest.mock import MagicMock, Mock, call import generic_config_updater.generic_updater as gu @@ -27,63 +26,47 @@ def create_side_effect_dict(map): class FilesLoader: def __init__(self): - self.test_path = os.path.dirname(os.path.abspath(__file__)) + self.files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + self.cache = {} def __getattr__(self, attr): - return self.__load(attr) + return self._load(attr) - def __load(self, file_name): + def _load(self, file_name): normalized_file_name = file_name.lower() - # Try load dict file - json_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.py-dict") - if os.path.isfile(json_file_path): - with open(json_file_path) as fh: - text = fh.read() - return eval(text) - # Try load json file - json_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.json") + json_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json") if os.path.isfile(json_file_path): with open(json_file_path) as fh: text = fh.read() return json.loads(text) # Try load json-patch file - jsonpatch_file_path = os.path.join(self.test_path, "files", f"{normalized_file_name}.json-patch") + jsonpatch_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json-patch") if os.path.isfile(jsonpatch_file_path): with open(jsonpatch_file_path) as fh: text = fh.read() return jsonpatch.JsonPatch(json.loads(text)) - + raise ValueError(f"There is no file called '{file_name}' in 'files/' directory") - + # Files.File_Name will look for a file called "file_name" in the "files/" directory Files = FilesLoader() class TestConfigWrapper(unittest.TestCase): + def setUp(self): + self.config_wrapper_mock = gu.ConfigWrapper() + self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) + def test_ctor__default_values_set(self): config_wrapper = gu.ConfigWrapper() - self.assertEqual(None, config_wrapper.default_config_db_connector) self.assertEqual("/usr/local/yang-models", gu.YANG_DIR) - def test_get_config_db_as_json__returns_config_db_as_json(self): - # Arrange - config_db_connector_mock = self.__get_config_db_connector_mock(Files.CONFIG_DB_AS_DICT) - config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) - expected = Files.CONFIG_DB_AS_JSON - - # Act - actual = config_wrapper.get_config_db_as_json() - - # Assert - self.assertDictEqual(expected, actual) - def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): # Arrange - config_db_connector_mock = self.__get_config_db_connector_mock(Files.CONFIG_DB_AS_DICT) - config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) + config_wrapper = self.config_wrapper_mock expected = Files.SONIC_YANG_AS_JSON # Act @@ -212,12 +195,11 @@ def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): # Assert self.assertDictEqual(expected, actual) - def __get_config_db_connector_mock(self, config_db_as_dict): - mock_connector = Mock() - mock_connector.get_config.return_value = config_db_as_dict - return mock_connector - class TestPatchWrapper(unittest.TestCase): + def setUp(self): + self.config_wrapper_mock = gu.ConfigWrapper() + self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) + def test_validate_config_db_patch_has_yang_models__table_without_yang_model__returns_false(self): # Arrange patch_wrapper = gu.PatchWrapper() @@ -313,8 +295,7 @@ def test_generate_patch__diff__non_empty_patch(self): def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) patch = jsonpatch.JsonPatch([]) expected = jsonpatch.JsonPatch([]) @@ -326,8 +307,7 @@ def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH expected = Files.SINGLE_OPERATION_SONIC_YANG_PATCH @@ -339,7 +319,7 @@ def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__re def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + config_wrapper = self.config_wrapper_mock patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) config_db_patch = Files.MULTI_OPERATION_CONFIG_DB_PATCH @@ -351,8 +331,7 @@ def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch_ def test_convert_sonic_yang_patch_to_config_db_patch__empty_patch__returns_empty_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) patch = jsonpatch.JsonPatch([]) expected = jsonpatch.JsonPatch([]) @@ -364,8 +343,7 @@ def test_convert_sonic_yang_patch_to_config_db_patch__empty_patch__returns_empty def test_convert_sonic_yang_patch_to_config_db_patch__single_operation_patch__returns_config_db_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) + patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) patch = Files.SINGLE_OPERATION_SONIC_YANG_PATCH expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH @@ -377,7 +355,7 @@ def test_convert_sonic_yang_patch_to_config_db_patch__single_operation_patch__re def test_convert_sonic_yang_patch_to_config_db_patch__multiple_operations_patch__returns_config_db_patch(self): # Arrange - config_wrapper = self.__get_config_wrapper_mock(Files.CONFIG_DB_AS_DICT) + config_wrapper = self.config_wrapper_mock patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) sonic_yang_patch = Files.MULTI_OPERATION_SONIC_YANG_PATCH @@ -400,16 +378,6 @@ def __assert_same_patch(self, config_db_patch, sonic_yang_patch, config_wrapper, self.assertTrue(patch_wrapper.verify_same_json(after_update_config_db_cropped, after_update_sonic_yang_as_config_db)) - def __get_config_wrapper_mock(self, config_db_as_dict): - config_db_connector_mock = self.__get_config_db_connector_mock(config_db_as_dict) - config_wrapper = gu.ConfigWrapper(default_config_db_connector = config_db_connector_mock) - return config_wrapper - - def __get_config_db_connector_mock(self, config_db_as_dict): - mock_connector = Mock() - mock_connector.get_config.return_value = config_db_as_dict - return mock_connector - class TestPatchApplier(unittest.TestCase): def test_apply__invalid_patch_updating_tables_without_yang_models__failure(self): # Arrange @@ -610,8 +578,8 @@ def test_list_checkpoints__checkpoints_dir_does_not_exist__empty_list(self): actual = rollbacker.list_checkpoints() # Assert - self.assertListEqual(expected, actual) - + # 'assertCountEqual' does check same count, same elements ignoring order + self.assertCountEqual(expected, actual) def test_list_checkpoints__checkpoints_dir_exist_but_no_files__empty_list(self): # Arrange @@ -623,7 +591,8 @@ def test_list_checkpoints__checkpoints_dir_exist_but_no_files__empty_list(self): actual = rollbacker.list_checkpoints() # Assert - self.assertListEqual(expected, actual) + # 'assertCountEqual' does check same count, same elements ignoring order + self.assertCountEqual(expected, actual) def test_list_checkpoints__checkpoints_dir_has_multiple_files__multiple_files(self): # Arrange @@ -637,7 +606,8 @@ def test_list_checkpoints__checkpoints_dir_has_multiple_files__multiple_files(se actual = rollbacker.list_checkpoints() # Assert - self.assertListEqual(expected, actual) + # 'assertCountEqual' does check same count, same elements ignoring order + self.assertCountEqual(expected, actual) def test_list_checkpoints__checkpoints_names_have_special_characters__multiple_files(self): # Arrange @@ -677,24 +647,25 @@ def test_delete_checkpoint__checkpoint_exist__success(self): def test_multiple_operations(self): rollbacker = self.create_rollbacker() - self.assertListEqual([], rollbacker.list_checkpoints()) + # 'assertCountEqual' does check same count, same elements ignoring order + self.assertCountEqual([], rollbacker.list_checkpoints()) rollbacker.checkpoint(self.any_checkpoint_name) - self.assertListEqual([self.any_checkpoint_name], rollbacker.list_checkpoints()) + self.assertCountEqual([self.any_checkpoint_name], rollbacker.list_checkpoints()) self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) rollbacker.rollback(self.any_checkpoint_name) rollbacker.config_replacer.replace.assert_has_calls([call(self.any_config)]) rollbacker.checkpoint(self.any_other_checkpoint_name) - self.assertListEqual([self.any_checkpoint_name, self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + self.assertCountEqual([self.any_checkpoint_name, self.any_other_checkpoint_name], rollbacker.list_checkpoints()) self.assertEqual(self.any_config, self.get_checkpoint(self.any_other_checkpoint_name)) rollbacker.delete_checkpoint(self.any_checkpoint_name) - self.assertListEqual([self.any_other_checkpoint_name], rollbacker.list_checkpoints()) + self.assertCountEqual([self.any_other_checkpoint_name], rollbacker.list_checkpoints()) rollbacker.delete_checkpoint(self.any_other_checkpoint_name) - self.assertListEqual([], rollbacker.list_checkpoints()) + self.assertCountEqual([], rollbacker.list_checkpoints()) def clean_up(self): if os.path.isdir(self.checkpoints_dir): @@ -971,7 +942,7 @@ def test_list_checkpoints__creates_rollbacker_and_list_checkpoints(self): actual = generic_updater.list_checkpoints(self.any_verbose) # Assert - self.assertListEqual(expected, actual) + self.assertCountEqual(expected, actual) class TestDecorator(unittest.TestCase): def setUp(self): @@ -983,10 +954,10 @@ def setUp(self): self.any_other_checkpoint_name = "anyothercheckpoint" self.any_checkpoints_list = [self.any_checkpoint_name, self.any_other_checkpoint_name] self.decorated_config_rollbacker.list_checkpoints.return_value = self.any_checkpoints_list - + self.decorator = gu.Decorator( self.decorated_patch_applier, self.decorated_config_replacer, self.decorated_config_rollbacker) - + def test_apply__calls_decorated_applier(self): # Act self.decorator.apply(Files.SINGLE_OPERATION_SONIC_YANG_PATCH) @@ -1058,7 +1029,7 @@ def test_replace__converts_to_config_db_and_calls_decorated_class(self): sonic_yang_decorator.config_wrapper.convert_sonic_yang_to_config_db.assert_has_calls( [call(Files.SONIC_YANG_AS_JSON)]) sonic_yang_decorator.decorated_config_replacer.replace.assert_has_calls([call(Files.CONFIG_DB_AS_JSON)]) - + def __create_sonic_yang_decorator(self): patch_applier = Mock() patch_applier.apply.side_effect = create_side_effect_dict({(str(Files.SINGLE_OPERATION_CONFIG_DB_PATCH),): 0}) From 4a2a96447d2c870dcf01428ef6c615a3f78bd2f5 Mon Sep 17 00:00:00 2001 From: ghooo Date: Sat, 17 Apr 2021 10:03:00 -0700 Subject: [PATCH 25/32] unifying exceptions --- generic_config_updater/generic_updater.py | 16 +++++----------- .../generic_updater_test.py | 6 +++--- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index c64aefb0f7..f545ed4024 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -10,13 +10,7 @@ CHECKPOINTS_DIR = "/etc/sonic/checkpoints" CHECKPOINT_EXT = ".cp.json" -class GenericUpdaterException(Exception): - pass - -class ConfigNotCompletelyUpdatedError(GenericUpdaterException): - pass - -class FailedToGetRunningConfigError(GenericUpdaterException): +class GenericConfigUpdaterError(Exception): pass class JsonChange: @@ -57,7 +51,7 @@ def _get_config_db_as_text(self): text, err = result.communicate() return_code = result.returncode if return_code: # non-zero means failure - raise FailedToGetRunningConfig(f"Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError(f"Failed to get running config, Return code: {return_code}, Error: {err}") return text def get_sonic_yang_as_json(self): @@ -242,7 +236,7 @@ def apply(self, patch): # Validate config updated successfully new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise ConfigNotCompletelyUpdatedError(f"After applying patch to config, there are still some parts not updated") + raise GenericConfigUpdaterError(f"After applying patch to config, there are still some parts not updated") class ConfigReplacer: def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None): @@ -261,7 +255,7 @@ def replace(self, target_config): new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise ConfigNotCompletelyUpdatedError(f"After replacing config, there is still some parts not updated") + raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") class FileSystemConfigRollbacker: def __init__(self, @@ -281,7 +275,7 @@ def rollback(self, checkpoint_name): self.config_replacer.replace(target_config) def checkpoint(self, checkpoint_name): - json_content = self.config_wrapper.get_sonic_yang_as_json() + json_content = self.config_wrapper.get_config_db_as_json() path = self._get_checkpoint_full_path(checkpoint_name) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index f430e2579a..03f5660116 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -398,7 +398,7 @@ def test_apply__json_not_fully_updated__failure(self): patch_applier = self.__create_patch_applier(verified_same_config=False) # Act and assert - self.assertRaises(gu.ConfigNotCompletelyUpdatedError, patch_applier.apply, Files.MULTI_OPERATION_CONFIG_DB_PATCH) + self.assertRaises(gu.GenericConfigUpdaterError, patch_applier.apply, Files.MULTI_OPERATION_CONFIG_DB_PATCH) def test_apply__no_errors__update_successful(self): # Arrange @@ -468,7 +468,7 @@ def test_replace__json_not_fully_updated__failure(self): config_replacer = self.__create_config_replacer(verified_same_config=False) # Act and assert - self.assertRaises(gu.ConfigNotCompletelyUpdatedError, config_replacer.replace, Files.CONFIG_DB_AFTER_MULTI_PATCH) + self.assertRaises(gu.GenericConfigUpdaterError, config_replacer.replace, Files.CONFIG_DB_AFTER_MULTI_PATCH) def test_replace__no_errors__update_successful(self): # Arrange @@ -694,7 +694,7 @@ def create_rollbacker(self): replacer.replace.side_effect = create_side_effect_dict({(str(self.any_config),): 0}) config_wrapper = Mock() - config_wrapper.get_sonic_yang_as_json.return_value = self.any_config + config_wrapper.get_config_db_as_json.return_value = self.any_config return gu.FileSystemConfigRollbacker(checkpoints_dir=self.checkpoints_dir, config_replacer=replacer, From fc026fbcfddc9e880390525afcf8b3eceeeac8ec Mon Sep 17 00:00:00 2001 From: ghooo Date: Sat, 17 Apr 2021 10:42:51 -0700 Subject: [PATCH 26/32] moving common functionalities to common file --- generic_config_updater/generic_updater.py | 173 +------- generic_config_updater/gu_common.py | 173 ++++++++ .../generic_updater_test.py | 375 +----------------- .../generic_config_updater/gu_common_test.py | 334 ++++++++++++++++ .../generic_config_updater/gutest_helpers.py | 53 +++ 5 files changed, 566 insertions(+), 542 deletions(-) create mode 100644 generic_config_updater/gu_common.py create mode 100644 tests/generic_config_updater/gu_common_test.py create mode 100644 tests/generic_config_updater/gutest_helpers.py diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index f545ed4024..6b6f452912 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,22 +1,13 @@ import json -import jsonpatch import sonic_yang import os -import copy -import subprocess from enum import Enum +from .gu_common import GenericConfigUpdaterError, JsonChange, \ + ConfigWrapper, DryRunConfigWrapper, PatchWrapper -YANG_DIR = "/usr/local/yang-models" CHECKPOINTS_DIR = "/etc/sonic/checkpoints" CHECKPOINT_EXT = ".cp.json" -class GenericConfigUpdaterError(Exception): - pass - -class JsonChange: - # TODO: Implement JsonChange - pass - class ConfigLock: def acquire_lock(self): # TODO: Implement ConfigLock @@ -36,166 +27,6 @@ def apply(self, change): # TODO: Implement change applier pass -class ConfigWrapper: - def __init__(self, yang_dir = YANG_DIR): - self.yang_dir = YANG_DIR - - def get_config_db_as_json(self): - text = self._get_config_db_as_text() - return json.loads(text) - - def _get_config_db_as_text(self): - # TODO: Getting configs from CLI is very slow, need to get it from sonic-cffgen directly - cmd = "show runningconfiguration all" - result = subprocess.Popen(cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - text, err = result.communicate() - return_code = result.returncode - if return_code: # non-zero means failure - raise GenericConfigUpdaterError(f"Failed to get running config, Return code: {return_code}, Error: {err}") - return text - - def get_sonic_yang_as_json(self): - config_db_json = self.get_config_db_as_json() - return self.convert_config_db_to_sonic_yang(config_db_json) - - def convert_config_db_to_sonic_yang(self, config_db_as_json): - sy = sonic_yang.SonicYang(self.yang_dir) - sy.loadYangModel() - - # Crop config_db tables that do not have sonic yang models - cropped_config_db_as_json = self.crop_tables_without_yang(config_db_as_json) - - sonic_yang_as_json = dict() - - sy._xlateConfigDBtoYang(cropped_config_db_as_json, sonic_yang_as_json) - - return sonic_yang_as_json - - def convert_sonic_yang_to_config_db(self, sonic_yang_as_json): - sy = sonic_yang.SonicYang(self.yang_dir) - sy.loadYangModel() - - # replace container of the format 'module:table' with just 'table' - new_sonic_yang_json = {} - for module_top in sonic_yang_as_json: - new_sonic_yang_json[module_top] = {} - for container in sonic_yang_as_json[module_top]: - tokens = container.split(':') - if len(tokens) > 2: - raise ValueError(f"Expecting ':
' or '
', found {container}") - table = container if len(tokens) == 1 else tokens[1] - new_sonic_yang_json[module_top][table] = sonic_yang_as_json[module_top][container] - - config_db_as_json = dict() - sy.xlateJson = new_sonic_yang_json - sy.revXlateJson = config_db_as_json - sy._revXlateYangtoConfigDB(new_sonic_yang_json, config_db_as_json) - - return config_db_as_json - - def validate_sonic_yang_config(self, sonic_yang_as_json): - config_db_as_json = self.convert_sonic_yang_to_config_db(sonic_yang_as_json) - - sy = sonic_yang.SonicYang(self.yang_dir) - sy.loadYangModel() - - try: - sy.loadData(config_db_as_json) - - sy.validate_data_tree() - return True - except sonic_yang.SonicYangException as ex: - return False - - def validate_config_db_config(self, config_db_as_json): - sy = sonic_yang.SonicYang(self.yang_dir) - sy.loadYangModel() - - try: - sy.loadData(config_db_as_json) - - sy.validate_data_tree() - return True - except sonic_yang.SonicYangException as ex: - return False - - def crop_tables_without_yang(self, config_db_as_json): - sy = sonic_yang.SonicYang(self.yang_dir) - sy.loadYangModel() - - sy.jIn = copy.deepcopy(config_db_as_json) - - sy.tablesWithOutYang = dict() - - sy._cropConfigDB() - - return sy.jIn - - def _create_and_connect_config_db(self): - if self.default_config_db_connector != None: - return self.default_config_db_connector - - config_db = ConfigDBConnector() - config_db.connect() - return config_db - -class DryRunConfigWrapper(ConfigWrapper): - # TODO: implement DryRunConfigWrapper - # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. - pass - -class PatchWrapper: - def __init__(self, config_wrapper=None): - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() - - def validate_config_db_patch_has_yang_models(self, patch): - config_db = {} - for operation in patch: - tokens = operation['path'].split('/')[1:] - if len(tokens) == 0: # Modifying whole config_db - tables_dict = {table_name: {} for table_name in operation['value']} - config_db.update(tables_dict) - elif not tokens[0]: # Not empty - raise ValueError("Table name in patch cannot be empty") - else: - config_db[tokens[0]] = {} - - cropped_config_db = self.config_wrapper.crop_tables_without_yang(config_db) - - # valid if no tables dropped during cropping - return len(cropped_config_db.keys()) == len(config_db.keys()) - - def verify_same_json(self, expected, actual): - # patch will be [] if no diff, [] evaluates to False - return not jsonpatch.make_patch(expected, actual) - - def generate_patch(self, current, target): - return jsonpatch.make_patch(current, target) - - def simulate_patch(self, patch, jsonconfig): - return patch.apply(jsonconfig) - - def convert_config_db_patch_to_sonic_yang_patch(self, patch): - if not(self.validate_config_db_patch_has_yang_models(patch)): - raise ValueError(f"Given patch is not valid") - - current_config_db = self.config_wrapper.get_config_db_as_json() - target_config_db = self.simulate_patch(patch, current_config_db) - - current_yang = self.config_wrapper.convert_config_db_to_sonic_yang(current_config_db) - target_yang = self.config_wrapper.convert_config_db_to_sonic_yang(target_config_db) - - return self.generate_patch(current_yang, target_yang) - - def convert_sonic_yang_patch_to_config_db_patch(self, patch): - current_yang = self.config_wrapper.get_sonic_yang_as_json() - target_yang = self.simulate_patch(patch, current_yang) - - current_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(current_yang) - target_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(target_yang) - - return self.generate_patch(current_config_db, target_config_db) - class ConfigFormat(Enum): CONFIGDB = 1 SONICYANG = 2 diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py new file mode 100644 index 0000000000..5c45a18ca6 --- /dev/null +++ b/generic_config_updater/gu_common.py @@ -0,0 +1,173 @@ +import json +import jsonpatch +import sonic_yang +import copy + +YANG_DIR = "/usr/local/yang-models" + +class GenericConfigUpdaterError(Exception): + pass + +class JsonChange: + # TODO: Implement JsonChange + pass + +class ConfigWrapper: + def __init__(self, yang_dir = YANG_DIR): + self.yang_dir = YANG_DIR + + def get_config_db_as_json(self): + text = self._get_config_db_as_text() + return json.loads(text) + + def _get_config_db_as_text(self): + # TODO: Getting configs from CLI is very slow, need to get it from sonic-cffgen directly + cmd = "show runningconfiguration all" + result = subprocess.Popen(cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + text, err = result.communicate() + return_code = result.returncode + if return_code: # non-zero means failure + raise GenericConfigUpdaterError(f"Failed to get running config, Return code: {return_code}, Error: {err}") + return text + + def get_sonic_yang_as_json(self): + config_db_json = self.get_config_db_as_json() + return self.convert_config_db_to_sonic_yang(config_db_json) + + def convert_config_db_to_sonic_yang(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + # Crop config_db tables that do not have sonic yang models + cropped_config_db_as_json = self.crop_tables_without_yang(config_db_as_json) + + sonic_yang_as_json = dict() + + sy._xlateConfigDBtoYang(cropped_config_db_as_json, sonic_yang_as_json) + + return sonic_yang_as_json + + def convert_sonic_yang_to_config_db(self, sonic_yang_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + # replace container of the format 'module:table' with just 'table' + new_sonic_yang_json = {} + for module_top in sonic_yang_as_json: + new_sonic_yang_json[module_top] = {} + for container in sonic_yang_as_json[module_top]: + tokens = container.split(':') + if len(tokens) > 2: + raise ValueError(f"Expecting ':
' or '
', found {container}") + table = container if len(tokens) == 1 else tokens[1] + new_sonic_yang_json[module_top][table] = sonic_yang_as_json[module_top][container] + + config_db_as_json = dict() + sy.xlateJson = new_sonic_yang_json + sy.revXlateJson = config_db_as_json + sy._revXlateYangtoConfigDB(new_sonic_yang_json, config_db_as_json) + + return config_db_as_json + + def validate_sonic_yang_config(self, sonic_yang_as_json): + config_db_as_json = self.convert_sonic_yang_to_config_db(sonic_yang_as_json) + + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + try: + sy.loadData(config_db_as_json) + + sy.validate_data_tree() + return True + except sonic_yang.SonicYangException as ex: + return False + + def validate_config_db_config(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + try: + sy.loadData(config_db_as_json) + + sy.validate_data_tree() + return True + except sonic_yang.SonicYangException as ex: + return False + + def crop_tables_without_yang(self, config_db_as_json): + sy = sonic_yang.SonicYang(self.yang_dir) + sy.loadYangModel() + + sy.jIn = copy.deepcopy(config_db_as_json) + + sy.tablesWithOutYang = dict() + + sy._cropConfigDB() + + return sy.jIn + + def _create_and_connect_config_db(self): + if self.default_config_db_connector != None: + return self.default_config_db_connector + + config_db = ConfigDBConnector() + config_db.connect() + return config_db + +class DryRunConfigWrapper(ConfigWrapper): + # TODO: implement DryRunConfigWrapper + # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. + pass + +class PatchWrapper: + def __init__(self, config_wrapper=None): + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() + + def validate_config_db_patch_has_yang_models(self, patch): + config_db = {} + for operation in patch: + tokens = operation['path'].split('/')[1:] + if len(tokens) == 0: # Modifying whole config_db + tables_dict = {table_name: {} for table_name in operation['value']} + config_db.update(tables_dict) + elif not tokens[0]: # Not empty + raise ValueError("Table name in patch cannot be empty") + else: + config_db[tokens[0]] = {} + + cropped_config_db = self.config_wrapper.crop_tables_without_yang(config_db) + + # valid if no tables dropped during cropping + return len(cropped_config_db.keys()) == len(config_db.keys()) + + def verify_same_json(self, expected, actual): + # patch will be [] if no diff, [] evaluates to False + return not jsonpatch.make_patch(expected, actual) + + def generate_patch(self, current, target): + return jsonpatch.make_patch(current, target) + + def simulate_patch(self, patch, jsonconfig): + return patch.apply(jsonconfig) + + def convert_config_db_patch_to_sonic_yang_patch(self, patch): + if not(self.validate_config_db_patch_has_yang_models(patch)): + raise ValueError(f"Given patch is not valid") + + current_config_db = self.config_wrapper.get_config_db_as_json() + target_config_db = self.simulate_patch(patch, current_config_db) + + current_yang = self.config_wrapper.convert_config_db_to_sonic_yang(current_config_db) + target_yang = self.config_wrapper.convert_config_db_to_sonic_yang(target_config_db) + + return self.generate_patch(current_yang, target_yang) + + def convert_sonic_yang_patch_to_config_db_patch(self, patch): + current_yang = self.config_wrapper.get_sonic_yang_as_json() + target_yang = self.simulate_patch(patch, current_yang) + + current_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(current_yang) + target_config_db = self.config_wrapper.convert_sonic_yang_to_config_db(target_yang) + + return self.generate_patch(current_config_db, target_config_db) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 03f5660116..f1f577dc84 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -1,382 +1,15 @@ import json -import jsonpatch import os import shutil -import sys import unittest from unittest.mock import MagicMock, Mock, call +from .gutest_helpers import create_side_effect_dict, Files import generic_config_updater.generic_updater as gu -class MockSideEffectDict: - def __init__(self, map): - self.map = map - - def side_effect_func(self, *args): - l = [str(arg) for arg in args] - key = tuple(l) - value = self.map.get(key) - if value is None: - raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") - - return value - -def create_side_effect_dict(map): - return MockSideEffectDict(map).side_effect_func - -class FilesLoader: - def __init__(self): - self.files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") - self.cache = {} - - def __getattr__(self, attr): - return self._load(attr) - - def _load(self, file_name): - normalized_file_name = file_name.lower() - - # Try load json file - json_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json") - if os.path.isfile(json_file_path): - with open(json_file_path) as fh: - text = fh.read() - return json.loads(text) - - # Try load json-patch file - jsonpatch_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json-patch") - if os.path.isfile(jsonpatch_file_path): - with open(jsonpatch_file_path) as fh: - text = fh.read() - return jsonpatch.JsonPatch(json.loads(text)) - - raise ValueError(f"There is no file called '{file_name}' in 'files/' directory") - -# Files.File_Name will look for a file called "file_name" in the "files/" directory -Files = FilesLoader() - -class TestConfigWrapper(unittest.TestCase): - def setUp(self): - self.config_wrapper_mock = gu.ConfigWrapper() - self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) - - def test_ctor__default_values_set(self): - config_wrapper = gu.ConfigWrapper() - - self.assertEqual("/usr/local/yang-models", gu.YANG_DIR) - - def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): - # Arrange - config_wrapper = self.config_wrapper_mock - expected = Files.SONIC_YANG_AS_JSON - - # Act - actual = config_wrapper.get_sonic_yang_as_json() - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_config_db_to_sonic_yang__empty_config_db__returns_empty_sonic_yang(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = {} - - # Act - actual = config_wrapper.convert_config_db_to_sonic_yang({}) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_config_db_to_sonic_yang__non_empty_config_db__returns_sonic_yang_as_json(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = Files.SONIC_YANG_AS_JSON - - # Act - actual = config_wrapper.convert_config_db_to_sonic_yang(Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__empty_sonic_yang__returns_empty_config_db(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = {} - - # Act - actual = config_wrapper.convert_sonic_yang_to_config_db({}) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_db_as_json(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = Files.CROPPED_CONFIG_DB_AS_JSON - - # Act - actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__table_name_without_colons__returns_config_db_as_json(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = Files.CROPPED_CONFIG_DB_AS_JSON - - # Act - actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON_WITHOUT_COLONS) - - # Assert - self.assertDictEqual(expected, actual) - - def test_convert_sonic_yang_to_config_db__table_name_with_unexpected_colons__returns_config_db_as_json(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = Files.CROPPED_CONFIG_DB_AS_JSON - - # Act and assert - self.assertRaises(ValueError, - config_wrapper.convert_sonic_yang_to_config_db, - Files.SONIC_YANG_AS_JSON_WITH_UNEXPECTED_COLONS) - - def test_validate_sonic_yang_config__valid_config__returns_true(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = True - - # Act - actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON) - - # Assert - self.assertEqual(expected, actual) - - def test_validate_sonic_yang_config__invvalid_config__returns_false(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = False - - # Act - actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON_INVALID) - - # Assert - self.assertEqual(expected, actual) - - def test_validate_config_db_config__valid_config__returns_true(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = True - - # Act - actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertEqual(expected, actual) - - def test_config_db_config__invvalid_config__returns_false(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = False - - # Act - actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON_INVALID) - - # Assert - self.assertEqual(expected, actual) - - def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): - # Arrange - config_wrapper = gu.ConfigWrapper() - expected = Files.CROPPED_CONFIG_DB_AS_JSON - - # Act - actual = config_wrapper.crop_tables_without_yang(Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - -class TestPatchWrapper(unittest.TestCase): - def setUp(self): - self.config_wrapper_mock = gu.ConfigWrapper() - self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) - - def test_validate_config_db_patch_has_yang_models__table_without_yang_model__returns_false(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] - expected = False - - # Act - actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_validate_config_db_patch_has_yang_models__table_with_yang_model__returns_true(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] - expected = True - - # Act - actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__failure(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] - - # Act and Assert - self.assertRaises(ValueError, patch_wrapper.convert_config_db_patch_to_sonic_yang_patch, patch) - - def test_same_patch__no_diff__returns_true(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - - # Act and Assert - self.assertTrue(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON)) - - def test_same_patch__diff__returns_false(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - - # Act and Assert - self.assertFalse(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CROPPED_CONFIG_DB_AS_JSON)) - - def test_generate_patch__no_diff__empty_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - - # Act - patch = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertFalse(patch) - - def test_simulate_patch__empty_patch__no_changes(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - patch = jsonpatch.JsonPatch([]) - expected = Files.CONFIG_DB_AS_JSON - - # Act - actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_simulate_patch__non_empty_patch__changes_applied(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH - expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) - - # Act - actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) - - # Assert - self.assertDictEqual(expected, actual) - - def test_generate_patch__diff__non_empty_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper() - after_update_json = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) - expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH - - # Act - actual = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, after_update_json) - - # Assert - self.assertTrue(actual) - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) - patch = jsonpatch.JsonPatch([]) - expected = jsonpatch.JsonPatch([]) - - # Act - actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) - patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH - expected = Files.SINGLE_OPERATION_SONIC_YANG_PATCH - - # Act - actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): - # Arrange - config_wrapper = self.config_wrapper_mock - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) - config_db_patch = Files.MULTI_OPERATION_CONFIG_DB_PATCH - - # Act - sonic_yang_patch = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(config_db_patch) - - # Assert - self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) - - def test_convert_sonic_yang_patch_to_config_db_patch__empty_patch__returns_empty_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) - patch = jsonpatch.JsonPatch([]) - expected = jsonpatch.JsonPatch([]) - - # Act - actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_sonic_yang_patch_to_config_db_patch__single_operation_patch__returns_config_db_patch(self): - # Arrange - patch_wrapper = gu.PatchWrapper(config_wrapper = self.config_wrapper_mock) - patch = Files.SINGLE_OPERATION_SONIC_YANG_PATCH - expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH - - # Act - actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) - - # Assert - self.assertEqual(expected, actual) - - def test_convert_sonic_yang_patch_to_config_db_patch__multiple_operations_patch__returns_config_db_patch(self): - # Arrange - config_wrapper = self.config_wrapper_mock - patch_wrapper = gu.PatchWrapper(config_wrapper = config_wrapper) - sonic_yang_patch = Files.MULTI_OPERATION_SONIC_YANG_PATCH - - # Act - config_db_patch = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(sonic_yang_patch) - - # Assert - self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) - - def __assert_same_patch(self, config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper): - sonic_yang = config_wrapper.get_sonic_yang_as_json() - config_db = config_wrapper.get_config_db_as_json() - - after_update_sonic_yang = patch_wrapper.simulate_patch(sonic_yang_patch, sonic_yang) - after_update_config_db = patch_wrapper.simulate_patch(config_db_patch, config_db) - after_update_config_db_cropped = config_wrapper.crop_tables_without_yang(after_update_config_db) - - after_update_sonic_yang_as_config_db = \ - config_wrapper.convert_sonic_yang_to_config_db(after_update_sonic_yang) - - self.assertTrue(patch_wrapper.verify_same_json(after_update_config_db_cropped, after_update_sonic_yang_as_config_db)) +# import sys +# sys.path.insert(0,'../../generic_config_updater') +# import generic_updater as gu class TestPatchApplier(unittest.TestCase): def test_apply__invalid_patch_updating_tables_without_yang_models__failure(self): diff --git a/tests/generic_config_updater/gu_common_test.py b/tests/generic_config_updater/gu_common_test.py new file mode 100644 index 0000000000..f3b9303840 --- /dev/null +++ b/tests/generic_config_updater/gu_common_test.py @@ -0,0 +1,334 @@ +import jsonpatch +import unittest +from unittest.mock import MagicMock, Mock +from .gutest_helpers import create_side_effect_dict, Files + +import generic_config_updater.gu_common as gu_common + +# import sys +# sys.path.insert(0,'../../generic_config_updater') +# import gu_common + +class TestConfigWrapper(unittest.TestCase): + def setUp(self): + self.config_wrapper_mock = gu_common.ConfigWrapper() + self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) + + def test_ctor__default_values_set(self): + config_wrapper = gu_common.ConfigWrapper() + + self.assertEqual("/usr/local/yang-models", gu_common.YANG_DIR) + + def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): + # Arrange + config_wrapper = self.config_wrapper_mock + expected = Files.SONIC_YANG_AS_JSON + + # Act + actual = config_wrapper.get_sonic_yang_as_json() + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__empty_config_db__returns_empty_sonic_yang(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = {} + + # Act + actual = config_wrapper.convert_config_db_to_sonic_yang({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_config_db_to_sonic_yang__non_empty_config_db__returns_sonic_yang_as_json(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = Files.SONIC_YANG_AS_JSON + + # Act + actual = config_wrapper.convert_config_db_to_sonic_yang(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__empty_sonic_yang__returns_empty_config_db(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = {} + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db({}) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__non_empty_sonic_yang__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__table_name_without_colons__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.convert_sonic_yang_to_config_db(Files.SONIC_YANG_AS_JSON_WITHOUT_COLONS) + + # Assert + self.assertDictEqual(expected, actual) + + def test_convert_sonic_yang_to_config_db__table_name_with_unexpected_colons__returns_config_db_as_json(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act and assert + self.assertRaises(ValueError, + config_wrapper.convert_sonic_yang_to_config_db, + Files.SONIC_YANG_AS_JSON_WITH_UNEXPECTED_COLONS) + + def test_validate_sonic_yang_config__valid_config__returns_true(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = True + + # Act + actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_sonic_yang_config__invvalid_config__returns_false(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = False + + # Act + actual = config_wrapper.validate_sonic_yang_config(Files.SONIC_YANG_AS_JSON_INVALID) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_config_db_config__valid_config__returns_true(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = True + + # Act + actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertEqual(expected, actual) + + def test_config_db_config__invvalid_config__returns_false(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = False + + # Act + actual = config_wrapper.validate_config_db_config(Files.CONFIG_DB_AS_JSON_INVALID) + + # Assert + self.assertEqual(expected, actual) + + def test_crop_tables_without_yang__returns_cropped_config_db_as_json(self): + # Arrange + config_wrapper = gu_common.ConfigWrapper() + expected = Files.CROPPED_CONFIG_DB_AS_JSON + + # Act + actual = config_wrapper.crop_tables_without_yang(Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + +class TestPatchWrapper(unittest.TestCase): + def setUp(self): + self.config_wrapper_mock = gu_common.ConfigWrapper() + self.config_wrapper_mock.get_config_db_as_json=MagicMock(return_value=Files.CONFIG_DB_AS_JSON) + + def test_validate_config_db_patch_has_yang_models__table_without_yang_model__returns_false(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + expected = False + + # Act + actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_validate_config_db_patch_has_yang_models__table_with_yang_model__returns_true(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/ACL_TABLE' } ] + expected = True + + # Act + actual = patch_wrapper.validate_config_db_patch_has_yang_models(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__invalid_config_db_patch__failure(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + patch = [ { 'op': 'remove', 'path': '/TABLE_WITHOUT_YANG' } ] + + # Act and Assert + self.assertRaises(ValueError, patch_wrapper.convert_config_db_patch_to_sonic_yang_patch, patch) + + def test_same_patch__no_diff__returns_true(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + + # Act and Assert + self.assertTrue(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON)) + + def test_same_patch__diff__returns_false(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + + # Act and Assert + self.assertFalse(patch_wrapper.verify_same_json(Files.CONFIG_DB_AS_JSON, Files.CROPPED_CONFIG_DB_AS_JSON)) + + def test_generate_patch__no_diff__empty_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + + # Act + patch = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertFalse(patch) + + def test_simulate_patch__empty_patch__no_changes(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + patch = jsonpatch.JsonPatch([]) + expected = Files.CONFIG_DB_AS_JSON + + # Act + actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_simulate_patch__non_empty_patch__changes_applied(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) + + # Act + actual = patch_wrapper.simulate_patch(patch, Files.CONFIG_DB_AS_JSON) + + # Assert + self.assertDictEqual(expected, actual) + + def test_generate_patch__diff__non_empty_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper() + after_update_json = Files.SINGLE_OPERATION_CONFIG_DB_PATCH.apply(Files.CONFIG_DB_AS_JSON) + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + + # Act + actual = patch_wrapper.generate_patch(Files.CONFIG_DB_AS_JSON, after_update_json) + + # Assert + self.assertTrue(actual) + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__empty_patch__returns_empty_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper(config_wrapper = self.config_wrapper_mock) + patch = jsonpatch.JsonPatch([]) + expected = jsonpatch.JsonPatch([]) + + # Act + actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__single_operation_patch__returns_sonic_yang_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper(config_wrapper = self.config_wrapper_mock) + patch = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + expected = Files.SINGLE_OPERATION_SONIC_YANG_PATCH + + # Act + actual = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_config_db_patch_to_sonic_yang_patch__multiple_operations_patch__returns_sonic_yang_patch(self): + # Arrange + config_wrapper = self.config_wrapper_mock + patch_wrapper = gu_common.PatchWrapper(config_wrapper = config_wrapper) + config_db_patch = Files.MULTI_OPERATION_CONFIG_DB_PATCH + + # Act + sonic_yang_patch = patch_wrapper.convert_config_db_patch_to_sonic_yang_patch(config_db_patch) + + # Assert + self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) + + def test_convert_sonic_yang_patch_to_config_db_patch__empty_patch__returns_empty_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper(config_wrapper = self.config_wrapper_mock) + patch = jsonpatch.JsonPatch([]) + expected = jsonpatch.JsonPatch([]) + + # Act + actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_sonic_yang_patch_to_config_db_patch__single_operation_patch__returns_config_db_patch(self): + # Arrange + patch_wrapper = gu_common.PatchWrapper(config_wrapper = self.config_wrapper_mock) + patch = Files.SINGLE_OPERATION_SONIC_YANG_PATCH + expected = Files.SINGLE_OPERATION_CONFIG_DB_PATCH + + # Act + actual = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + + # Assert + self.assertEqual(expected, actual) + + def test_convert_sonic_yang_patch_to_config_db_patch__multiple_operations_patch__returns_config_db_patch(self): + # Arrange + config_wrapper = self.config_wrapper_mock + patch_wrapper = gu_common.PatchWrapper(config_wrapper = config_wrapper) + sonic_yang_patch = Files.MULTI_OPERATION_SONIC_YANG_PATCH + + # Act + config_db_patch = patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(sonic_yang_patch) + + # Assert + self.__assert_same_patch(config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper) + + def __assert_same_patch(self, config_db_patch, sonic_yang_patch, config_wrapper, patch_wrapper): + sonic_yang = config_wrapper.get_sonic_yang_as_json() + config_db = config_wrapper.get_config_db_as_json() + + after_update_sonic_yang = patch_wrapper.simulate_patch(sonic_yang_patch, sonic_yang) + after_update_config_db = patch_wrapper.simulate_patch(config_db_patch, config_db) + after_update_config_db_cropped = config_wrapper.crop_tables_without_yang(after_update_config_db) + + after_update_sonic_yang_as_config_db = \ + config_wrapper.convert_sonic_yang_to_config_db(after_update_sonic_yang) + + self.assertTrue(patch_wrapper.verify_same_json(after_update_config_db_cropped, after_update_sonic_yang_as_config_db)) diff --git a/tests/generic_config_updater/gutest_helpers.py b/tests/generic_config_updater/gutest_helpers.py new file mode 100644 index 0000000000..2e8984ad68 --- /dev/null +++ b/tests/generic_config_updater/gutest_helpers.py @@ -0,0 +1,53 @@ +import json +import jsonpatch +import os +import shutil +import sys +import unittest +from unittest.mock import MagicMock, Mock, call + +class MockSideEffectDict: + def __init__(self, map): + self.map = map + + def side_effect_func(self, *args): + l = [str(arg) for arg in args] + key = tuple(l) + value = self.map.get(key) + if value is None: + raise ValueError(f"Given arguments were not found in arguments map.\n Arguments: {key}\n Map: {self.map}") + + return value + +def create_side_effect_dict(map): + return MockSideEffectDict(map).side_effect_func + +class FilesLoader: + def __init__(self): + self.files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + self.cache = {} + + def __getattr__(self, attr): + return self._load(attr) + + def _load(self, file_name): + normalized_file_name = file_name.lower() + + # Try load json file + json_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json") + if os.path.isfile(json_file_path): + with open(json_file_path) as fh: + text = fh.read() + return json.loads(text) + + # Try load json-patch file + jsonpatch_file_path = os.path.join(self.files_path, f"{normalized_file_name}.json-patch") + if os.path.isfile(jsonpatch_file_path): + with open(jsonpatch_file_path) as fh: + text = fh.read() + return jsonpatch.JsonPatch(json.loads(text)) + + raise ValueError(f"There is no file called '{file_name}' in 'files/' directory") + +# Files.File_Name will look for a file called "file_name" in the "files/" directory +Files = FilesLoader() From 90f6731ab551c3366dbee5e1e69d6ade3730a296 Mon Sep 17 00:00:00 2001 From: ghooo Date: Sat, 17 Apr 2021 11:07:24 -0700 Subject: [PATCH 27/32] small fixes --- generic_config_updater/generic_updater.py | 11 +++++------ generic_config_updater/gu_common.py | 5 ++++- tests/generic_config_updater/gu_common_test.py | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 6b6f452912..8520326cb8 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,9 +1,8 @@ import json -import sonic_yang import os from enum import Enum -from .gu_common import GenericConfigUpdaterError, JsonChange, \ - ConfigWrapper, DryRunConfigWrapper, PatchWrapper +from .gu_common import GenericConfigUpdaterError, ConfigWrapper, \ + DryRunConfigWrapper, PatchWrapper CHECKPOINTS_DIR = "/etc/sonic/checkpoints" CHECKPOINT_EXT = ".cp.json" @@ -20,12 +19,12 @@ def release_lock(self): class PatchSorter: def sort(self, patch): # TODO: Implement patch sorter - pass + raise NotImplementedError("PatchSorter.sort(patch) is not implemented yet") class ChangeApplier: def apply(self, change): # TODO: Implement change applier - pass + raise NotImplementedError("ChangeApplier.apply(change) is not implemented yet") class ConfigFormat(Enum): CONFIGDB = 1 @@ -122,7 +121,7 @@ def list_checkpoints(self): def delete_checkpoint(self, checkpoint_name): if not self._check_checkpoint_exists(checkpoint_name): - raise ValueError("Checkpoint does not exist") + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") self._delete_checkpoint(checkpoint_name) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 5c45a18ca6..2aa6a36d8a 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -1,6 +1,7 @@ import json import jsonpatch import sonic_yang +import subprocess import copy YANG_DIR = "/usr/local/yang-models" @@ -88,7 +89,9 @@ def validate_config_db_config(self, config_db_as_json): sy.loadYangModel() try: - sy.loadData(config_db_as_json) + tmp_config_db_as_json = copy.deepcopy(config_db_as_json) + + sy.loadData(tmp_config_db_as_json) sy.validate_data_tree() return True diff --git a/tests/generic_config_updater/gu_common_test.py b/tests/generic_config_updater/gu_common_test.py index f3b9303840..f18ad45799 100644 --- a/tests/generic_config_updater/gu_common_test.py +++ b/tests/generic_config_updater/gu_common_test.py @@ -1,3 +1,4 @@ +import json import jsonpatch import unittest from unittest.mock import MagicMock, Mock @@ -128,7 +129,7 @@ def test_validate_config_db_config__valid_config__returns_true(self): # Assert self.assertEqual(expected, actual) - def test_config_db_config__invvalid_config__returns_false(self): + def test_validate_config_db_config__invalid_config__returns_false(self): # Arrange config_wrapper = gu_common.ConfigWrapper() expected = False From 34ee074dfd3de75b68c687dda0f98ad5e448a92e Mon Sep 17 00:00:00 2001 From: ghooo Date: Sat, 17 Apr 2021 19:53:09 -0700 Subject: [PATCH 28/32] renaming table without yant to TABLE_WITHOUT_YANG --- .../files/config_db_after_multi_patch.json | 19 ++++--------------- .../files/config_db_as_json.json | 19 ++++--------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/tests/generic_config_updater/files/config_db_after_multi_patch.json b/tests/generic_config_updater/files/config_db_after_multi_patch.json index 71313bd4e0..042bf1d51b 100644 --- a/tests/generic_config_updater/files/config_db_after_multi_patch.json +++ b/tests/generic_config_updater/files/config_db_after_multi_patch.json @@ -113,21 +113,10 @@ "speed": "10000" } }, - "WRED_PROFILE": { - "AZURE_LOSSLESS": { - "wred_green_enable": "true", - "wred_yellow_enable": "true", - "wred_red_enable": "true", - "ecn": "ecn_all", - "green_max_threshold": "2097152", - "green_min_threshold": "1048576", - "yellow_max_threshold": "2097152", - "yellow_min_threshold": "1048576", - "red_max_threshold": "2097152", - "red_min_threshold": "1048576", - "green_drop_probability": "5", - "yellow_drop_probability": "5", - "red_drop_probability": "5" + "TABLE_WITHOUT_YANG": { + "Item1": { + "key11": "value11", + "key12": "value12" } } } \ No newline at end of file diff --git a/tests/generic_config_updater/files/config_db_as_json.json b/tests/generic_config_updater/files/config_db_as_json.json index a6d8b3eb03..02fb7c7e6a 100644 --- a/tests/generic_config_updater/files/config_db_as_json.json +++ b/tests/generic_config_updater/files/config_db_as_json.json @@ -83,21 +83,10 @@ "speed": "40000" } }, - "WRED_PROFILE": { - "AZURE_LOSSLESS": { - "wred_green_enable": "true", - "wred_yellow_enable": "true", - "wred_red_enable": "true", - "ecn": "ecn_all", - "green_max_threshold": "2097152", - "green_min_threshold": "1048576", - "yellow_max_threshold": "2097152", - "yellow_min_threshold": "1048576", - "red_max_threshold": "2097152", - "red_min_threshold": "1048576", - "green_drop_probability": "5", - "yellow_drop_probability": "5", - "red_drop_probability": "5" + "TABLE_WITHOUT_YANG": { + "Item1": { + "key11": "value11", + "key12": "value12" } } } From 52883457ae0c8b58a4d10dce69cae1faef875c39 Mon Sep 17 00:00:00 2001 From: ghooo Date: Sun, 18 Apr 2021 01:07:50 -0700 Subject: [PATCH 29/32] validate running config before taking a checkpoint --- generic_config_updater/generic_updater.py | 3 +++ tests/generic_config_updater/generic_updater_test.py | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 8520326cb8..079d7ab742 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -107,6 +107,9 @@ def rollback(self, checkpoint_name): def checkpoint(self, checkpoint_name): json_content = self.config_wrapper.get_config_db_as_json() + if not self.config_wrapper.validate_config_db_config(json_content): + raise ValueError(f"Running configs on the device are not valid.") + path = self._get_checkpoint_full_path(checkpoint_name) self._ensure_checkpoints_dir_exists() diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index f1f577dc84..f201280062 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -190,6 +190,13 @@ def test_checkpoint__checkpoints_dir_does_not_exist__checkpoint_created(self): self.assertTrue(os.path.isdir(self.checkpoints_dir)) self.assertEqual(self.any_config, self.get_checkpoint(self.any_checkpoint_name)) + def test_checkpoint__config_not_valid__failure(self): + # Arrange + rollbacker = self.create_rollbacker(valid_config=False) + + # Act and assert + self.assertRaises(ValueError, rollbacker.checkpoint, self.any_checkpoint_name) + def test_checkpoint__checkpoints_dir_exists__checkpoint_created(self): # Arrange self.create_checkpoints_dir() @@ -322,12 +329,13 @@ def check_checkpoint_exists(self, name): path=os.path.join(self.checkpoints_dir, f"{name}{self.checkpoint_ext}") return os.path.isfile(path) - def create_rollbacker(self): + def create_rollbacker(self, valid_config=True): replacer = Mock() replacer.replace.side_effect = create_side_effect_dict({(str(self.any_config),): 0}) config_wrapper = Mock() config_wrapper.get_config_db_as_json.return_value = self.any_config + config_wrapper.validate_config_db_config.return_value = valid_config return gu.FileSystemConfigRollbacker(checkpoints_dir=self.checkpoints_dir, config_replacer=replacer, From e5d142c1e79222f88a998df7b84627c4efd64379 Mon Sep 17 00:00:00 2001 From: ghooo Date: Mon, 19 Apr 2021 21:07:55 -0700 Subject: [PATCH 30/32] fixing azure pipeline --- .azure-pipelines/docker-sonic-vs/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile index 4e0a50e7a4..2b3e634232 100644 --- a/.azure-pipelines/docker-sonic-vs/Dockerfile +++ b/.azure-pipelines/docker-sonic-vs/Dockerfile @@ -4,4 +4,8 @@ ARG docker_container_name ADD ["wheels", "/wheels"] -RUN pip3 install --no-deps --force-reinstall /wheels/sonic_utilities-1.2-py3-none-any.whl +# Uninstalls only sonic-utilities and does not impact its dependencies +RUN pip3 uninstall -y sonic-utilities + +# Installs sonic-utilities, adds missing dependencies, upgrades out-dated depndencies +RUN pip3 install /wheels/sonic_utilities-1.2-py3-none-any.whl From 7ebf3aef2fd787332483c69cbef8d60a0fe2d273 Mon Sep 17 00:00:00 2001 From: ghooo Date: Wed, 21 Apr 2021 16:22:08 -0700 Subject: [PATCH 31/32] making sonic-config-engine required install_required --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9bbf908f9b..d070827667 100644 --- a/setup.py +++ b/setup.py @@ -166,6 +166,7 @@ 'pexpect==4.8.0', 'pyroute2==0.5.14', 'requests==2.25.0', + 'sonic-config-engine', 'sonic-platform-common', 'sonic-py-common', 'sonic-yang-mgmt', @@ -180,7 +181,6 @@ tests_require = [ 'pytest', 'mockredispy>=2.9.3', - 'sonic-config-engine', 'deepdiff==5.2.3' ], classifiers=[ From 1a3e37ee490b3cc36cfa1725793b4f2ff96517f5 Mon Sep 17 00:00:00 2001 From: ghooo Date: Fri, 23 Apr 2021 18:23:29 -0700 Subject: [PATCH 32/32] err to stderr and format upper case --- config/main.py | 20 ++++++++++---------- tests/config_test.py | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config/main.py b/config/main.py index fbbb73e71a..6c55feed6e 100644 --- a/config/main.py +++ b/config/main.py @@ -990,8 +990,8 @@ def load(filename, yes): @config.command('apply-patch') @click.argument('patch-file-path', type=str, required=True) -@click.option('-f', '--format', type=click.Choice([e.name.lower() for e in ConfigFormat]), - default=ConfigFormat.CONFIGDB.name.lower(), +@click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), + default=ConfigFormat.CONFIGDB.name, help='format of config of the patch is either ConfigDb(ABNF) or SonicYang') @click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @@ -1015,13 +1015,13 @@ def apply_patch(ctx, patch_file_path, format, dry_run, verbose): click.secho("Patch applied successfully.", fg="cyan", underline=True) except Exception as ex: - click.secho("Failed to apply patch", fg="red", underline=True) + click.secho("Failed to apply patch", fg="red", underline=True, err=True) ctx.fail(ex) @config.command() @click.argument('target-file-path', type=str, required=True) -@click.option('-f', '--format', type=click.Choice([e.name.lower() for e in ConfigFormat]), - default=ConfigFormat.CONFIGDB.name.lower(), +@click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), + default=ConfigFormat.CONFIGDB.name, help='format of target config is either ConfigDb(ABNF) or SonicYang') @click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @@ -1045,7 +1045,7 @@ def replace(ctx, target_file_path, format, dry_run, verbose): click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: - click.secho("Failed to replace config", fg="red", underline=True) + click.secho("Failed to replace config", fg="red", underline=True, err=True) ctx.fail(ex) @config.command() @@ -1064,7 +1064,7 @@ def rollback(ctx, checkpoint_name, dry_run, verbose): click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: - click.secho("Failed to rollback config", fg="red", underline=True) + click.secho("Failed to rollback config", fg="red", underline=True, err=True) ctx.fail(ex) @config.command() @@ -1080,7 +1080,7 @@ def checkpoint(ctx, checkpoint_name, verbose): click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: - click.secho("Failed to create a config checkpoint", fg="red", underline=True) + click.secho("Failed to create a config checkpoint", fg="red", underline=True, err=True) ctx.fail(ex) @config.command('delete-checkpoint') @@ -1096,7 +1096,7 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: - click.secho("Failed to delete config checkpoint", fg="red", underline=True) + click.secho("Failed to delete config checkpoint", fg="red", underline=True, err=True) ctx.fail(ex) @config.command('list-checkpoints') @@ -1109,7 +1109,7 @@ def list_checkpoints(ctx, verbose): formatted_output = json.dumps(checkpoints_list, indent=4) click.echo(formatted_output) except Exception as ex: - click.secho("Failed to list config checkpoints", fg="red", underline=True) + click.secho("Failed to list config checkpoints", fg="red", underline=True, err=True) ctx.fail(ex) @config.command() diff --git a/tests/config_test.py b/tests/config_test.py index e8744bd28a..32ecc5bdef 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -226,7 +226,7 @@ def test_apply_patch__all_optional_params_non_default__non_default_values_used(s # Act result = self.runner.invoke(config.config.commands["apply-patch"], [self.any_path, - "--format", ConfigFormat.SONICYANG.name.lower(), + "--format", ConfigFormat.SONICYANG.name, "--dry-run", "--verbose"], catch_exceptions=False) @@ -257,7 +257,7 @@ def test_apply_patch__exception_thrown__error_displayed_error_code_returned(self def test_apply_patch__optional_parameters_passed_correctly(self): self.validate_apply_patch_optional_parameter( - ["--format", ConfigFormat.SONICYANG.name.lower()], + ["--format", ConfigFormat.SONICYANG.name], mock.call(self.any_patch, ConfigFormat.SONICYANG, False, False)) self.validate_apply_patch_optional_parameter( ["--verbose"], @@ -339,7 +339,7 @@ def test_replace__all_optional_params_non_default__non_default_values_used(self) # Act result = self.runner.invoke(config.config.commands["replace"], [self.any_path, - "--format", ConfigFormat.SONICYANG.name.lower(), + "--format", ConfigFormat.SONICYANG.name, "--dry-run", "--verbose"], catch_exceptions=False) @@ -370,7 +370,7 @@ def test_replace__exception_thrown__error_displayed_error_code_returned(self): def test_replace__optional_parameters_passed_correctly(self): self.validate_replace_optional_parameter( - ["--format", ConfigFormat.SONICYANG.name.lower()], + ["--format", ConfigFormat.SONICYANG.name], mock.call(self.any_target_config, ConfigFormat.SONICYANG, False, False)) self.validate_replace_optional_parameter( ["--verbose"],