From cb06a3b7e1e6e7ba5c898df982644e781a1f7f53 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Fri, 14 May 2021 05:10:04 +0000 Subject: [PATCH 01/22] Port Module and UT Added Signed-off-by: Vivek Reddy Karri --- dump/plugins/__init__.py | 19 +++ dump/plugins/executor.py | 15 +++ dump/plugins/port.py | 110 ++++++++++++++++ tests/dump_tests/files/copp_cfg.json | 103 +++++++++++++++ tests/dump_tests/files/port/appl_db.json | 36 +++++ tests/dump_tests/files/port/asic_db.json | 30 +++++ tests/dump_tests/files/port/config_db.json | 30 +++++ tests/dump_tests/files/port/state_db.json | 14 ++ tests/dump_tests/module_tests/__init__.py | 0 .../module_tests/mock_sonicv2connector.py | 57 ++++++++ tests/dump_tests/module_tests/port_test.py | 123 ++++++++++++++++++ 11 files changed, 537 insertions(+) create mode 100644 dump/plugins/__init__.py create mode 100644 dump/plugins/executor.py create mode 100644 dump/plugins/port.py create mode 100644 tests/dump_tests/files/copp_cfg.json create mode 100644 tests/dump_tests/files/port/appl_db.json create mode 100644 tests/dump_tests/files/port/asic_db.json create mode 100644 tests/dump_tests/files/port/config_db.json create mode 100644 tests/dump_tests/files/port/state_db.json create mode 100644 tests/dump_tests/module_tests/__init__.py create mode 100644 tests/dump_tests/module_tests/mock_sonicv2connector.py create mode 100644 tests/dump_tests/module_tests/port_test.py diff --git a/dump/plugins/__init__.py b/dump/plugins/__init__.py new file mode 100644 index 0000000000..a10e90ec92 --- /dev/null +++ b/dump/plugins/__init__.py @@ -0,0 +1,19 @@ +import os +import sys +import pkgutil +import importlib +from .executor import Executor + + +dump_modules = {} +pkg_dir = os.path.dirname(__file__) + +# import child classes automatically +for (module_loader, name, ispkg) in pkgutil.iter_modules([pkg_dir]): + importlib.import_module('.' + name, __package__) + +# Classes inheriting Executor +dump_modules = {cls.__name__.lower(): cls for cls in Executor.__subclasses__()} + + + diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py new file mode 100644 index 0000000000..13fb762ff8 --- /dev/null +++ b/dump/plugins/executor.py @@ -0,0 +1,15 @@ + +from abc import ABC, abstractmethod + +class Executor(ABC): + + ARG_NAME = "id" # Arg Identifier + CONFIG_FILE = "" # Path to config file, if any + + @abstractmethod + def execute(self): + pass + + @abstractmethod + def get_all_args(self): + pass \ No newline at end of file diff --git a/dump/plugins/port.py b/dump/plugins/port.py new file mode 100644 index 0000000000..51c00e0f1b --- /dev/null +++ b/dump/plugins/port.py @@ -0,0 +1,110 @@ +import os, sys +from .executor import Executor +from dump.redis_match import MatchEngine, MatchRequest +from dump.helper import display_template + +class Port(Executor): + + ARG_NAME = "port_name" + + def __init__(self): + self.match_engine = MatchEngine() + + def get_all_args(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "PORT" + req.key_pattern = "*" + ret = self.match_engine.fetch(req) + all_ports = [] + for key in ret["keys"]: + all_ports.append(key.split("|")[-1]) + return all_ports + + + def execute(self, params_dict): + ret_temp = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + port_name = params_dict[Port.ARG_NAME] + ret_temp = self.get_config_info(port_name, ret_temp) + ret_temp = self.get_appl_info(port_name, ret_temp) + ret_temp, port_asic_obj = self.get_asic_info_hostif(port_name, ret_temp) + ret_temp = self.get_asic_info_port(port_asic_obj, ret_temp) + ret_temp = self.get_state_info(port_name, ret_temp) + return ret_temp + + def get_config_info(self, port_name, ret_temp): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "PORT" + req.key_pattern = port_name + ret = self.match_engine.fetch(req) + if not ret["error"] and len(ret["keys"]) != 0: + ret_temp[req.db]["keys"] = ret["keys"] + else: + ret_temp[req.db]["tables_not_found"] = [req.table] + return ret_temp + + def get_appl_info(self, port_name, ret_temp): + req = MatchRequest() + req.db = "APPL_DB" + req.table = "PORT_TABLE" + req.key_pattern = port_name + ret = self.match_engine.fetch(req) + if not ret["error"] and len(ret["keys"]) != 0: + ret_temp[req.db]["keys"] = ret["keys"] + else: + ret_temp[req.db]["tables_not_found"] = [req.table] + return ret_temp + + + def get_state_info(self, port_name, ret_temp): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "PORT_TABLE" + req.key_pattern = port_name + ret = self.match_engine.fetch(req) + if not ret["error"] and len(ret["keys"]) != 0: + ret_temp[req.db]["keys"] = ret["keys"] + else: + ret_temp[req.db]["tables_not_found"] = [req.table] + return ret_temp + + def get_asic_info_hostif(self, port_name, ret_temp): + req = MatchRequest() + req.db = "ASIC_DB" + req.table = "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF" + req.key_pattern = "*" + req.field = "SAI_HOSTIF_ATTR_NAME" + req.value = port_name + req.return_fields = ["SAI_HOSTIF_ATTR_OBJ_ID"] + ret = self.match_engine.fetch(req) + + asic_port_obj_id = "" + + if not ret["error"] and len(ret["keys"]) != 0: + ret_temp[req.db]["keys"] = ret["keys"] + if ret["keys"][-1] in ret["return_values"] and "SAI_HOSTIF_ATTR_OBJ_ID" in ret["return_values"][ret["keys"][-1]]: + asic_port_obj_id = ret["return_values"][ret["keys"][-1]]["SAI_HOSTIF_ATTR_OBJ_ID"] + else: + ret_temp[req.db]["tables_not_found"] = [req.table] + return ret_temp, asic_port_obj_id + + + def get_asic_info_port(self, asic_port_obj_id, ret_temp): + if not asic_port_obj_id: + ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + return ret_temp + + req = MatchRequest() + req.db = "ASIC_DB" + req.table = "ASIC_STATE:SAI_OBJECT_TYPE_PORT" + req.key_pattern = asic_port_obj_id + ret = self.match_engine.fetch(req) + if not ret["error"] and len(ret["keys"]) != 0: + ret_temp[req.db]["keys"].append(ret["keys"][-1]) + else: + ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + return ret_temp + + + \ No newline at end of file diff --git a/tests/dump_tests/files/copp_cfg.json b/tests/dump_tests/files/copp_cfg.json new file mode 100644 index 0000000000..cae5f4e8e0 --- /dev/null +++ b/tests/dump_tests/files/copp_cfg.json @@ -0,0 +1,103 @@ +{ + "COPP_GROUP": { + "default": { + "queue": "0", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue4_group1": { + "trap_action":"trap", + "trap_priority":"4", + "queue": "4" + }, + "queue4_group2": { + "trap_action":"copy", + "trap_priority":"4", + "queue": "4", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue4_group3": { + "trap_action":"trap", + "trap_priority":"4", + "queue": "4" + }, + "queue1_group1": { + "trap_action":"trap", + "trap_priority":"1", + "queue": "1", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"6000", + "cbs":"6000", + "red_action":"drop" + }, + "queue1_group2": { + "trap_action":"trap", + "trap_priority":"1", + "queue": "1", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue2_group1": { + "cbs": "1000", + "cir": "1000", + "genetlink_mcgrp_name": "packets", + "genetlink_name": "psample", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "2", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + + } + }, + "COPP_TRAP": { + "bgp": { + "trap_ids": "bgp,bgpv6", + "trap_group": "queue4_group1" + }, + "lacp": { + "trap_ids": "lacp", + "trap_group": "queue4_group1" + }, + "arp": { + "trap_ids": "arp_req,arp_resp,neigh_discovery", + "trap_group": "queue4_group2" + }, + "lldp": { + "trap_ids": "lldp", + "trap_group": "queue4_group3" + }, + "dhcp": { + "trap_ids": "dhcp,dhcpv6", + "trap_group": "queue4_group3" + }, + "udld": { + "trap_ids": "udld", + "trap_group": "queue4_group3" + }, + "ip2me": { + "trap_ids": "ip2me", + "trap_group": "queue1_group1" + }, + "nat": { + "trap_ids": "src_nat_miss,dest_nat_miss", + "trap_group": "queue1_group2" + }, + "sflow": { + "trap_group": "queue2_group1", + "trap_ids": "sample_packet" + } + } +} diff --git a/tests/dump_tests/files/port/appl_db.json b/tests/dump_tests/files/port/appl_db.json new file mode 100644 index 0000000000..bf228e6730 --- /dev/null +++ b/tests/dump_tests/files/port/appl_db.json @@ -0,0 +1,36 @@ +{ + + "PORT_TABLE:Ethernet176": { + "index": "0", + "lanes": "0", + "alias": "etp45", + "speed": "25000", + "oper_status": "up", + "pfc_asym": "off", + "mtu": "9100", + "fec": "rs", + "admin_status": "up" + }, + "PORT_TABLE:Ethernet160": { + "index": "0", + "lanes": "0", + "alias": "etp41", + "speed": "25000", + "oper_status": "up", + "pfc_asym": "off", + "mtu": "9100", + "fec": "rs", + "admin_status": "up" + }, + "PORT_TABLE:Ethernet164": { + "index": "0", + "lanes": "0", + "alias": "etp42", + "speed": "25000", + "oper_status": "up", + "pfc_asym": "off", + "mtu": "9100", + "fec": "rs", + "admin_status": "up" + } +} diff --git a/tests/dump_tests/files/port/asic_db.json b/tests/dump_tests/files/port/asic_db.json new file mode 100644 index 0000000000..3b06b38191 --- /dev/null +++ b/tests/dump_tests/files/port/asic_db.json @@ -0,0 +1,30 @@ +{ + + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d":{ + "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", + "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x100000000036a", + "SAI_HOSTIF_ATTR_NAME" : "Ethernet176", + "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x100000000036a": { + "SAI_PORT_ATTR_ADMIN_STATE" : "true", + "SAI_PORT_ATTR_SPEED" : "25000", + "SAI_PORT_ATTR_MTU" : "9122" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a49":{ + "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", + "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x10000000002e6", + "SAI_HOSTIF_ATTR_NAME" : "Ethernet160", + "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4a":{ + "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", + "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x1000000000307", + "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x1000000000307": { + "SAI_PORT_ATTR_ADMIN_STATE" : "true", + "SAI_PORT_ATTR_SPEED" : "25000", + "SAI_PORT_ATTR_MTU" : "9122" + } +} diff --git a/tests/dump_tests/files/port/config_db.json b/tests/dump_tests/files/port/config_db.json new file mode 100644 index 0000000000..a5a5dd967b --- /dev/null +++ b/tests/dump_tests/files/port/config_db.json @@ -0,0 +1,30 @@ +{ + "PORT|Ethernet176": { + "admin_status" : "up", + "alias": "etp45", + "index": "45", + "lanes": "176", + "speed": "25000" + }, + "PORT|Ethernet164": { + "admin_status" : "up", + "alias": "etp42", + "index": "42", + "lanes": "164", + "speed": "25000" + }, + "PORT|Ethernet160": { + "admin_status" : "up", + "alias": "etp41", + "index": "41", + "lanes": "160", + "speed": "25000" + }, + "PORT|Ethernet156": { + "admin_status" : "up", + "alias": "etp40", + "index": "40", + "lanes": "156", + "speed": "25000" + } +} diff --git a/tests/dump_tests/files/port/state_db.json b/tests/dump_tests/files/port/state_db.json new file mode 100644 index 0000000000..3cd0b62de6 --- /dev/null +++ b/tests/dump_tests/files/port/state_db.json @@ -0,0 +1,14 @@ +{ + "PORT_TABLE|Ethernet176":{ + "state" : "ok", + "netdev_oper_status" : "up" + }, + "PORT_TABLE|Ethernet160":{ + "state" : "ok", + "netdev_oper_status" : "up" + }, + "PORT_TABLE|Ethernet164":{ + "state" : "ok", + "netdev_oper_status" : "up" + } +} diff --git a/tests/dump_tests/module_tests/__init__.py b/tests/dump_tests/module_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py new file mode 100644 index 0000000000..3393fd77d8 --- /dev/null +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -0,0 +1,57 @@ + +import json, re, os + +class MockSonicV2Connector(): + def __init__(self, dedicated_dbs): + self.db = None + self.db_name_connect_to = None + self.dedicated_dbs = dedicated_dbs + db_config_path = os.path.join(os.path.dirname(__file__),"../../mock_tables/database_config.json") + with open(db_config_path) as f: + self.db_cfg = json.load(f) + + def connect(self, db_name, retry=False): + if db_name not in self.dedicated_dbs: + return None + try: + with open(self.dedicated_dbs[db_name]) as f: + self.db = json.load(f) + self.db_name_connect_to = db_name + except Exception as e: + raise "Connection Failure" + + def get_db_separator(self, db_name): + return self.db_cfg["DATABASES"][db_name]["separator"] + + def keys(self, db_name, pattern): + if not self.db: + raise "MockDB Not Connected" + if self.db_name_connect_to != db_name: + raise "Failed to find {} in the MockDB".format(db_name) + + pattern = re.escape(pattern) + pattern = pattern.replace("\\*", ".*") + filtered_keys = [] + all_keys = self.db.keys() + for key in all_keys: + if re.match(pattern, key): + filtered_keys.append(key) + return filtered_keys + + def get_all(self, db_name, key): + if not self.db: + raise "MockDB Not Connected" + if self.db_name_connect_to != db_name: + raise "Failed to find {} in the MockDB".format(db_name) + if key not in self.db: + return {} + return self.db[key] + + def get(self, db_name, key, field): + if not self.db: + raise "MockDB Not Connected" + if self.db_name_connect_to != db_name: + raise "Failed to find {} in the MockDB".format(db_name) + if key not in self.db or field not in self.db[key]: + return "" + return self.db[key][field] diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py new file mode 100644 index 0000000000..cf8995b9a1 --- /dev/null +++ b/tests/dump_tests/module_tests/port_test.py @@ -0,0 +1,123 @@ +import json, os, sys +import jsonpatch +import unittest +import pytest +from deepdiff import DeepDiff +from mock import patch +from dump.helper import display_template +from dump.plugins.port import Port + +module_tests_path = os.path.dirname(__file__) +dump_tests_path = os.path.join(module_tests_path, "../") +port_files_path = os.path.join(dump_tests_path,"files","port") +sys.path.append(dump_tests_path) +sys.path.append(module_tests_path) + +from module_tests.mock_sonicv2connector import MockSonicV2Connector + +dedicated_dbs = {} +dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") +dedicated_dbs['APPL_DB'] = os.path.join(port_files_path, "appl_db.json") +dedicated_dbs['ASIC_DB'] = os.path.join(port_files_path, "asic_db.json") +dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") + +def mock_connector(host): + return MockSonicV2Connector(dedicated_dbs) + +@pytest.fixture(scope="module", autouse=True) +def verbosity_setup(): + print("SETUP") + os.environ["VERBOSE"] = "1" + yield + print("TEARDOWN") + os.environ["VERBOSE"] = "0" + +def sort_lists(ret): + for db in ret.keys(): + for key in ret[db].keys(): + if isinstance(ret[db][key], list): + ret[db][key].sort() + return ret + +@patch("dump.redis_match.SonicV2Connector", mock_connector) +class TestPortModule(unittest.TestCase): + + def test_working_state(self): + params = {} + params[Port.ARG_NAME] = "Ethernet176" + m_port = Port() + returned = m_port.execute(params) + expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect["CONFIG_DB"]["keys"].append("PORT|Ethernet176") + expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet176") + expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet176") + expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x100000000036a") + expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d") + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + assert not ddiff, ddiff + + def test_missing_asic_port(self): + params = {} + params[Port.ARG_NAME] = "Ethernet160" + m_port = Port() + returned = m_port.execute(params) + expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect["CONFIG_DB"]["keys"].append("PORT|Ethernet160") + expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet160") + expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet160") + expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a49") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + assert not ddiff, ddiff + + def test_missing_asic_hostif(self): + params = {} + params[Port.ARG_NAME] = "Ethernet164" + m_port = Port() + returned = m_port.execute(params) + expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect["CONFIG_DB"]["keys"].append("PORT|Ethernet164") + expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet164") + expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet164") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + assert not ddiff, ddiff + + def test_missing_state_and_appl(self): + params = {} + params[Port.ARG_NAME] = "Ethernet156" + m_port = Port() + returned = m_port.execute(params) + expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect["CONFIG_DB"]["keys"].append("PORT|Ethernet156") + expect["APPL_DB"]["tables_not_found"].append("PORT_TABLE") + expect["STATE_DB"]["tables_not_found"].append("PORT_TABLE") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + assert not ddiff, ddiff + + def test_no_port(self): + params = {} + params[Port.ARG_NAME] = "Ethernet152" + m_port = Port() + returned = m_port.execute(params) + expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect["CONFIG_DB"]["tables_not_found"].append("PORT") + expect["APPL_DB"]["tables_not_found"].append("PORT_TABLE") + expect["STATE_DB"]["tables_not_found"].append("PORT_TABLE") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + assert not ddiff, ddiff + + def test_all_args(self): + params = {} + m_port = Port() + returned = m_port.get_all_args() + expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] + ddiff = DeepDiff(set(expect), set(expect)) + assert not ddiff, ddiff + + \ No newline at end of file From d236a7dc6d157a22e7817eb704b4149ac998db5a Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 20 May 2021 20:34:58 +0000 Subject: [PATCH 02/22] Multi-Asic Changes Added Signed-off-by: Vivek Reddy Karri --- dump/plugins/executor.py | 7 +++-- dump/plugins/port.py | 31 ++++++++++++------- .../module_tests/mock_sonicv2connector.py | 15 ++++++++- tests/dump_tests/module_tests/port_test.py | 13 +++++--- 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py index 13fb762ff8..b1a35a6246 100644 --- a/dump/plugins/executor.py +++ b/dump/plugins/executor.py @@ -7,9 +7,10 @@ class Executor(ABC): CONFIG_FILE = "" # Path to config file, if any @abstractmethod - def execute(self): + def execute(self, params): pass @abstractmethod - def get_all_args(self): - pass \ No newline at end of file + def get_all_args(self, ns): + pass + diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 51c00e0f1b..2f582918b5 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -9,12 +9,13 @@ class Port(Executor): def __init__(self): self.match_engine = MatchEngine() - - def get_all_args(self): + + def get_all_args(self, ns): req = MatchRequest() req.db = "CONFIG_DB" req.table = "PORT" req.key_pattern = "*" + req.ns = ns ret = self.match_engine.fetch(req) all_ports = [] for key in ret["keys"]: @@ -25,18 +26,20 @@ def get_all_args(self): def execute(self, params_dict): ret_temp = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) port_name = params_dict[Port.ARG_NAME] - ret_temp = self.get_config_info(port_name, ret_temp) - ret_temp = self.get_appl_info(port_name, ret_temp) - ret_temp, port_asic_obj = self.get_asic_info_hostif(port_name, ret_temp) - ret_temp = self.get_asic_info_port(port_asic_obj, ret_temp) - ret_temp = self.get_state_info(port_name, ret_temp) + ns = params_dict["namespace"] + ret_temp = self.get_config_info(port_name, ret_temp, ns) + ret_temp = self.get_appl_info(port_name, ret_temp, ns) + ret_temp, port_asic_obj = self.get_asic_info_hostif(port_name, ret_temp, ns) + ret_temp = self.get_asic_info_port(port_asic_obj, ret_temp, ns) + ret_temp = self.get_state_info(port_name, ret_temp, ns) return ret_temp - def get_config_info(self, port_name, ret_temp): + def get_config_info(self, port_name, ret_temp, ns): req = MatchRequest() req.db = "CONFIG_DB" req.table = "PORT" req.key_pattern = port_name + req.ns = ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: ret_temp[req.db]["keys"] = ret["keys"] @@ -44,11 +47,12 @@ def get_config_info(self, port_name, ret_temp): ret_temp[req.db]["tables_not_found"] = [req.table] return ret_temp - def get_appl_info(self, port_name, ret_temp): + def get_appl_info(self, port_name, ret_temp, ns): req = MatchRequest() req.db = "APPL_DB" req.table = "PORT_TABLE" req.key_pattern = port_name + req.ns = ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: ret_temp[req.db]["keys"] = ret["keys"] @@ -57,11 +61,12 @@ def get_appl_info(self, port_name, ret_temp): return ret_temp - def get_state_info(self, port_name, ret_temp): + def get_state_info(self, port_name, ret_temp, ns): req = MatchRequest() req.db = "STATE_DB" req.table = "PORT_TABLE" req.key_pattern = port_name + req.ns = ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: ret_temp[req.db]["keys"] = ret["keys"] @@ -69,7 +74,7 @@ def get_state_info(self, port_name, ret_temp): ret_temp[req.db]["tables_not_found"] = [req.table] return ret_temp - def get_asic_info_hostif(self, port_name, ret_temp): + def get_asic_info_hostif(self, port_name, ret_temp, ns): req = MatchRequest() req.db = "ASIC_DB" req.table = "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF" @@ -77,6 +82,7 @@ def get_asic_info_hostif(self, port_name, ret_temp): req.field = "SAI_HOSTIF_ATTR_NAME" req.value = port_name req.return_fields = ["SAI_HOSTIF_ATTR_OBJ_ID"] + req.ns = ns ret = self.match_engine.fetch(req) asic_port_obj_id = "" @@ -90,7 +96,7 @@ def get_asic_info_hostif(self, port_name, ret_temp): return ret_temp, asic_port_obj_id - def get_asic_info_port(self, asic_port_obj_id, ret_temp): + def get_asic_info_port(self, asic_port_obj_id, ret_temp, ns): if not asic_port_obj_id: ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") return ret_temp @@ -99,6 +105,7 @@ def get_asic_info_port(self, asic_port_obj_id, ret_temp): req.db = "ASIC_DB" req.table = "ASIC_STATE:SAI_OBJECT_TYPE_PORT" req.key_pattern = asic_port_obj_id + req.ns = ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: ret_temp[req.db]["keys"].append(ret["keys"][-1]) diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 3393fd77d8..013e69e348 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -1,8 +1,11 @@ import json, re, os +from utilities_common.constants import DEFAULT_NAMESPACE class MockSonicV2Connector(): - def __init__(self, dedicated_dbs): + def __init__(self, dedicated_dbs, namespace): + if namespace != DEFAULT_NAMESPACE: + raise "This Mock doesn't support multi-asic configuration" self.db = None self.db_name_connect_to = None self.dedicated_dbs = dedicated_dbs @@ -55,3 +58,13 @@ def get(self, db_name, key, field): if key not in self.db or field not in self.db[key]: return "" return self.db[key][field] + + def hexists(self, db_name, key, field): + if not self.db: + raise "MockDB Not Connected" + if self.db_name_connect_to != db_name: + raise "Failed to find {} in the MockDB".format(db_name) + if key not in self.db or field not in self.db[key]: + return False + else: + return True diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index cf8995b9a1..bf26aae1f6 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -21,8 +21,8 @@ dedicated_dbs['ASIC_DB'] = os.path.join(port_files_path, "asic_db.json") dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") -def mock_connector(host): - return MockSonicV2Connector(dedicated_dbs) +def mock_connector(host, namespace): + return MockSonicV2Connector(dedicated_dbs, namespace) @pytest.fixture(scope="module", autouse=True) def verbosity_setup(): @@ -45,6 +45,7 @@ class TestPortModule(unittest.TestCase): def test_working_state(self): params = {} params[Port.ARG_NAME] = "Ethernet176" + params["namespace"] = "" m_port = Port() returned = m_port.execute(params) expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -59,6 +60,7 @@ def test_working_state(self): def test_missing_asic_port(self): params = {} params[Port.ARG_NAME] = "Ethernet160" + params["namespace"] = "" m_port = Port() returned = m_port.execute(params) expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -73,6 +75,7 @@ def test_missing_asic_port(self): def test_missing_asic_hostif(self): params = {} params[Port.ARG_NAME] = "Ethernet164" + params["namespace"] = "" m_port = Port() returned = m_port.execute(params) expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -87,6 +90,7 @@ def test_missing_asic_hostif(self): def test_missing_state_and_appl(self): params = {} params[Port.ARG_NAME] = "Ethernet156" + params["namespace"] = "" m_port = Port() returned = m_port.execute(params) expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -101,6 +105,7 @@ def test_missing_state_and_appl(self): def test_no_port(self): params = {} params[Port.ARG_NAME] = "Ethernet152" + params["namespace"] = "" m_port = Port() returned = m_port.execute(params) expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -115,9 +120,9 @@ def test_no_port(self): def test_all_args(self): params = {} m_port = Port() - returned = m_port.get_all_args() + returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] - ddiff = DeepDiff(set(expect), set(expect)) + ddiff = DeepDiff(set(expect), set(returned)) assert not ddiff, ddiff \ No newline at end of file From f5ffd27724d77d441efb6755acf50d1ced5ebb16 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Mon, 24 May 2021 19:16:59 +0000 Subject: [PATCH 03/22] Added TODO to mock Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/module_tests/mock_sonicv2connector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 013e69e348..5a97cc5622 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -3,7 +3,10 @@ from utilities_common.constants import DEFAULT_NAMESPACE class MockSonicV2Connector(): - def __init__(self, dedicated_dbs, namespace): + def __init__(self, dedicated_dbs, namespace=DEFAULT_NAMESPACE): + # TODO: Add Support for + # 1) Different namespace, i.e. asic0, asic1 etc + # 2) In the case when user-provider dedicated_dbs is empty, Read from mock_tables/*_db.json if namespace != DEFAULT_NAMESPACE: raise "This Mock doesn't support multi-asic configuration" self.db = None From bfe107ed86a17733feda83235fb6d797b21a86f0 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 27 May 2021 17:41:39 +0000 Subject: [PATCH 04/22] redis_match name updated Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 2 +- tests/dump_tests/module_tests/port_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 2f582918b5..0fafc4391b 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -1,6 +1,6 @@ import os, sys from .executor import Executor -from dump.redis_match import MatchEngine, MatchRequest +from dump.match_infra import MatchEngine, MatchRequest from dump.helper import display_template class Port(Executor): diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index bf26aae1f6..686bba2966 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -39,7 +39,7 @@ def sort_lists(ret): ret[db][key].sort() return ret -@patch("dump.redis_match.SonicV2Connector", mock_connector) +@patch("dump.match_infra.SonicV2Connector", mock_connector) class TestPortModule(unittest.TestCase): def test_working_state(self): From e6607da4cf3645712dd37a9271f40bcd3f469f4a Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 27 May 2021 17:44:40 +0000 Subject: [PATCH 05/22] copp_cfg is not required in this PR Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/files/copp_cfg.json | 103 --------------------------- 1 file changed, 103 deletions(-) delete mode 100644 tests/dump_tests/files/copp_cfg.json diff --git a/tests/dump_tests/files/copp_cfg.json b/tests/dump_tests/files/copp_cfg.json deleted file mode 100644 index cae5f4e8e0..0000000000 --- a/tests/dump_tests/files/copp_cfg.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "COPP_GROUP": { - "default": { - "queue": "0", - "meter_type":"packets", - "mode":"sr_tcm", - "cir":"600", - "cbs":"600", - "red_action":"drop" - }, - "queue4_group1": { - "trap_action":"trap", - "trap_priority":"4", - "queue": "4" - }, - "queue4_group2": { - "trap_action":"copy", - "trap_priority":"4", - "queue": "4", - "meter_type":"packets", - "mode":"sr_tcm", - "cir":"600", - "cbs":"600", - "red_action":"drop" - }, - "queue4_group3": { - "trap_action":"trap", - "trap_priority":"4", - "queue": "4" - }, - "queue1_group1": { - "trap_action":"trap", - "trap_priority":"1", - "queue": "1", - "meter_type":"packets", - "mode":"sr_tcm", - "cir":"6000", - "cbs":"6000", - "red_action":"drop" - }, - "queue1_group2": { - "trap_action":"trap", - "trap_priority":"1", - "queue": "1", - "meter_type":"packets", - "mode":"sr_tcm", - "cir":"600", - "cbs":"600", - "red_action":"drop" - }, - "queue2_group1": { - "cbs": "1000", - "cir": "1000", - "genetlink_mcgrp_name": "packets", - "genetlink_name": "psample", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "2", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - - } - }, - "COPP_TRAP": { - "bgp": { - "trap_ids": "bgp,bgpv6", - "trap_group": "queue4_group1" - }, - "lacp": { - "trap_ids": "lacp", - "trap_group": "queue4_group1" - }, - "arp": { - "trap_ids": "arp_req,arp_resp,neigh_discovery", - "trap_group": "queue4_group2" - }, - "lldp": { - "trap_ids": "lldp", - "trap_group": "queue4_group3" - }, - "dhcp": { - "trap_ids": "dhcp,dhcpv6", - "trap_group": "queue4_group3" - }, - "udld": { - "trap_ids": "udld", - "trap_group": "queue4_group3" - }, - "ip2me": { - "trap_ids": "ip2me", - "trap_group": "queue1_group1" - }, - "nat": { - "trap_ids": "src_nat_miss,dest_nat_miss", - "trap_group": "queue1_group2" - }, - "sflow": { - "trap_group": "queue2_group1", - "trap_ids": "sample_packet" - } - } -} From ee22710a88987b4f2c391ecdac9c01a9e0715d29 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Sat, 29 May 2021 04:45:02 +0000 Subject: [PATCH 06/22] Final Changes before review made Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 67 ++++++++++------------ tests/dump_tests/module_tests/port_test.py | 8 +-- 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 0fafc4391b..671469938f 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -22,59 +22,55 @@ def get_all_args(self, ns): all_ports.append(key.split("|")[-1]) return all_ports - def execute(self, params_dict): - ret_temp = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + self.ret_temp = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) port_name = params_dict[Port.ARG_NAME] - ns = params_dict["namespace"] - ret_temp = self.get_config_info(port_name, ret_temp, ns) - ret_temp = self.get_appl_info(port_name, ret_temp, ns) - ret_temp, port_asic_obj = self.get_asic_info_hostif(port_name, ret_temp, ns) - ret_temp = self.get_asic_info_port(port_asic_obj, ret_temp, ns) - ret_temp = self.get_state_info(port_name, ret_temp, ns) - return ret_temp + self.ns = params_dict["namespace"] + self.get_config_info(port_name) + self.get_appl_info(port_name) + port_asic_obj = self.get_asic_info_hostif(port_name) + self.get_asic_info_port(port_asic_obj) + self.get_state_info(port_name) + return self.ret_temp - def get_config_info(self, port_name, ret_temp, ns): + def get_config_info(self, port_name): req = MatchRequest() req.db = "CONFIG_DB" req.table = "PORT" req.key_pattern = port_name - req.ns = ns + req.ns = self.ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: - ret_temp[req.db]["keys"] = ret["keys"] + self.ret_temp[req.db]["keys"] = ret["keys"] else: - ret_temp[req.db]["tables_not_found"] = [req.table] - return ret_temp + self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_appl_info(self, port_name, ret_temp, ns): + def get_appl_info(self, port_name): req = MatchRequest() req.db = "APPL_DB" req.table = "PORT_TABLE" req.key_pattern = port_name - req.ns = ns + req.ns = self.ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: - ret_temp[req.db]["keys"] = ret["keys"] + self.ret_temp[req.db]["keys"] = ret["keys"] else: - ret_temp[req.db]["tables_not_found"] = [req.table] - return ret_temp + self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_state_info(self, port_name, ret_temp, ns): + def get_state_info(self, port_name): req = MatchRequest() req.db = "STATE_DB" req.table = "PORT_TABLE" req.key_pattern = port_name - req.ns = ns + req.ns = self.ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: - ret_temp[req.db]["keys"] = ret["keys"] + self.ret_temp[req.db]["keys"] = ret["keys"] else: - ret_temp[req.db]["tables_not_found"] = [req.table] - return ret_temp + self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_asic_info_hostif(self, port_name, ret_temp, ns): + def get_asic_info_hostif(self, port_name): req = MatchRequest() req.db = "ASIC_DB" req.table = "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF" @@ -82,36 +78,35 @@ def get_asic_info_hostif(self, port_name, ret_temp, ns): req.field = "SAI_HOSTIF_ATTR_NAME" req.value = port_name req.return_fields = ["SAI_HOSTIF_ATTR_OBJ_ID"] - req.ns = ns + req.ns = self.ns ret = self.match_engine.fetch(req) asic_port_obj_id = "" if not ret["error"] and len(ret["keys"]) != 0: - ret_temp[req.db]["keys"] = ret["keys"] + self.ret_temp[req.db]["keys"] = ret["keys"] if ret["keys"][-1] in ret["return_values"] and "SAI_HOSTIF_ATTR_OBJ_ID" in ret["return_values"][ret["keys"][-1]]: asic_port_obj_id = ret["return_values"][ret["keys"][-1]]["SAI_HOSTIF_ATTR_OBJ_ID"] else: - ret_temp[req.db]["tables_not_found"] = [req.table] - return ret_temp, asic_port_obj_id + self.ret_temp[req.db]["tables_not_found"] = [req.table] + return asic_port_obj_id - def get_asic_info_port(self, asic_port_obj_id, ret_temp, ns): + def get_asic_info_port(self, asic_port_obj_id): if not asic_port_obj_id: - ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") - return ret_temp + self.ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + return req = MatchRequest() req.db = "ASIC_DB" req.table = "ASIC_STATE:SAI_OBJECT_TYPE_PORT" req.key_pattern = asic_port_obj_id - req.ns = ns + req.ns = self.ns ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: - ret_temp[req.db]["keys"].append(ret["keys"][-1]) + self.ret_temp[req.db]["keys"].append(ret["keys"][-1]) else: - ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") - return ret_temp + self.ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") \ No newline at end of file diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 686bba2966..8c86b33156 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -4,7 +4,7 @@ import pytest from deepdiff import DeepDiff from mock import patch -from dump.helper import display_template +from dump.helper import display_template, sort_lists from dump.plugins.port import Port module_tests_path = os.path.dirname(__file__) @@ -32,12 +32,6 @@ def verbosity_setup(): print("TEARDOWN") os.environ["VERBOSE"] = "0" -def sort_lists(ret): - for db in ret.keys(): - for key in ret[db].keys(): - if isinstance(ret[db][key], list): - ret[db][key].sort() - return ret @patch("dump.match_infra.SonicV2Connector", mock_connector) class TestPortModule(unittest.TestCase): From 6d094f42da36e3890e7ee6480c104de1b850c1ea Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 9 Jun 2021 08:07:42 +0000 Subject: [PATCH 07/22] Comments Addressed Signed-off-by: Vivek Reddy Karri --- dump/plugins/executor.py | 1 + dump/plugins/port.py | 63 ++++++---------------- tests/dump_tests/module_tests/port_test.py | 24 ++++----- 3 files changed, 30 insertions(+), 58 deletions(-) diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py index b1a35a6246..d918a4e4f0 100644 --- a/dump/plugins/executor.py +++ b/dump/plugins/executor.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod class Executor(ABC): + """ Abstract Class which should be extended from in order to be included in the dump state CLI """ ARG_NAME = "id" # Arg Identifier CONFIG_FILE = "" # Path to config file, if any diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 671469938f..1f6e383169 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -1,7 +1,7 @@ import os, sys from .executor import Executor from dump.match_infra import MatchEngine, MatchRequest -from dump.helper import display_template +from dump.helper import create_template_dict class Port(Executor): @@ -11,19 +11,13 @@ def __init__(self): self.match_engine = MatchEngine() def get_all_args(self, ns): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "PORT" - req.key_pattern = "*" - req.ns = ns + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", ns=ns) ret = self.match_engine.fetch(req) - all_ports = [] - for key in ret["keys"]: - all_ports.append(key.split("|")[-1]) - return all_ports + all_ports = ret["keys"] + return [key.split("|")[-1] for key in all_ports] def execute(self, params_dict): - self.ret_temp = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) port_name = params_dict[Port.ARG_NAME] self.ns = params_dict["namespace"] self.get_config_info(port_name) @@ -34,11 +28,7 @@ def execute(self, params_dict): return self.ret_temp def get_config_info(self, port_name): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "PORT" - req.key_pattern = port_name - req.ns = self.ns + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: self.ret_temp[req.db]["keys"] = ret["keys"] @@ -46,11 +36,7 @@ def get_config_info(self, port_name): self.ret_temp[req.db]["tables_not_found"] = [req.table] def get_appl_info(self, port_name): - req = MatchRequest() - req.db = "APPL_DB" - req.table = "PORT_TABLE" - req.key_pattern = port_name - req.ns = self.ns + req = MatchRequest(db="APPL_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: self.ret_temp[req.db]["keys"] = ret["keys"] @@ -59,11 +45,7 @@ def get_appl_info(self, port_name): def get_state_info(self, port_name): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "PORT_TABLE" - req.key_pattern = port_name - req.ns = self.ns + req = MatchRequest(db="STATE_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: self.ret_temp[req.db]["keys"] = ret["keys"] @@ -71,40 +53,29 @@ def get_state_info(self, port_name): self.ret_temp[req.db]["tables_not_found"] = [req.table] def get_asic_info_hostif(self, port_name): - req = MatchRequest() - req.db = "ASIC_DB" - req.table = "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF" - req.key_pattern = "*" - req.field = "SAI_HOSTIF_ATTR_NAME" - req.value = port_name - req.return_fields = ["SAI_HOSTIF_ATTR_OBJ_ID"] - req.ns = self.ns + req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF", key_pattern="*", field="SAI_HOSTIF_ATTR_NAME", + value=port_name, return_fields=["SAI_HOSTIF_ATTR_OBJ_ID"], ns=self.ns) ret = self.match_engine.fetch(req) - asic_port_obj_id = "" if not ret["error"] and len(ret["keys"]) != 0: self.ret_temp[req.db]["keys"] = ret["keys"] - if ret["keys"][-1] in ret["return_values"] and "SAI_HOSTIF_ATTR_OBJ_ID" in ret["return_values"][ret["keys"][-1]]: - asic_port_obj_id = ret["return_values"][ret["keys"][-1]]["SAI_HOSTIF_ATTR_OBJ_ID"] + sai_hostif_obj_key = ret["keys"][-1] + if sai_hostif_obj_key in ret["return_values"] and "SAI_HOSTIF_ATTR_OBJ_ID" in ret["return_values"][sai_hostif_obj_key]: + asic_port_obj_id = ret["return_values"][sai_hostif_obj_key]["SAI_HOSTIF_ATTR_OBJ_ID"] else: self.ret_temp[req.db]["tables_not_found"] = [req.table] return asic_port_obj_id - - + def get_asic_info_port(self, asic_port_obj_id): if not asic_port_obj_id: self.ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") return - - req = MatchRequest() - req.db = "ASIC_DB" - req.table = "ASIC_STATE:SAI_OBJECT_TYPE_PORT" - req.key_pattern = asic_port_obj_id - req.ns = self.ns + req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_PORT", key_pattern=asic_port_obj_id, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: - self.ret_temp[req.db]["keys"].append(ret["keys"][-1]) + sai_port_obj_key = ret["keys"][-1] + self.ret_temp[req.db]["keys"].append(sai_port_obj_key) else: self.ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 8c86b33156..80289247b7 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -4,7 +4,7 @@ import pytest from deepdiff import DeepDiff from mock import patch -from dump.helper import display_template, sort_lists +from dump.helper import create_template_dict, sort_lists from dump.plugins.port import Port module_tests_path = os.path.dirname(__file__) @@ -42,13 +42,13 @@ def test_working_state(self): params["namespace"] = "" m_port = Port() returned = m_port.execute(params) - expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet176") expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet176") expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet176") expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x100000000036a") expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d") - ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff def test_missing_asic_port(self): @@ -57,13 +57,13 @@ def test_missing_asic_port(self): params["namespace"] = "" m_port = Port() returned = m_port.execute(params) - expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet160") expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet160") expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet160") expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a49") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") - ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff def test_missing_asic_hostif(self): @@ -72,13 +72,13 @@ def test_missing_asic_hostif(self): params["namespace"] = "" m_port = Port() returned = m_port.execute(params) - expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet164") expect["APPL_DB"]["keys"].append("PORT_TABLE:Ethernet164") expect["STATE_DB"]["keys"].append("PORT_TABLE|Ethernet164") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") - ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff def test_missing_state_and_appl(self): @@ -87,13 +87,13 @@ def test_missing_state_and_appl(self): params["namespace"] = "" m_port = Port() returned = m_port.execute(params) - expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet156") expect["APPL_DB"]["tables_not_found"].append("PORT_TABLE") expect["STATE_DB"]["tables_not_found"].append("PORT_TABLE") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") - ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff def test_no_port(self): @@ -102,13 +102,13 @@ def test_no_port(self): params["namespace"] = "" m_port = Port() returned = m_port.execute(params) - expect = display_template(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) + expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["tables_not_found"].append("PORT") expect["APPL_DB"]["tables_not_found"].append("PORT_TABLE") expect["STATE_DB"]["tables_not_found"].append("PORT_TABLE") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") - ddiff = DeepDiff(sort_lists(returned), sort_lists(expect)) + ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff def test_all_args(self): @@ -116,7 +116,7 @@ def test_all_args(self): m_port = Port() returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] - ddiff = DeepDiff(set(expect), set(returned)) + ddiff = DeepDiff(expect, returned, ignore_order=True) assert not ddiff, ddiff \ No newline at end of file From ffe61584e04026578f1b98bec23431e189fb42b1 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 9 Jun 2021 08:18:37 +0000 Subject: [PATCH 08/22] Minor changes addressed Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 1f6e383169..44523d6a29 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -9,8 +9,10 @@ class Port(Executor): def __init__(self): self.match_engine = MatchEngine() + self.ret_temp = {} + self.ns = '' - def get_all_args(self, ns): + def get_all_args(self, ns=""): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", ns=ns) ret = self.match_engine.fetch(req) all_ports = ret["keys"] From b750eba7344c34f57b160b01405576c34394154b Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 9 Jun 2021 08:23:18 +0000 Subject: [PATCH 09/22] Comments addressed Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 44523d6a29..5f71c05e06 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -22,14 +22,14 @@ def execute(self, params_dict): self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) port_name = params_dict[Port.ARG_NAME] self.ns = params_dict["namespace"] - self.get_config_info(port_name) - self.get_appl_info(port_name) - port_asic_obj = self.get_asic_info_hostif(port_name) - self.get_asic_info_port(port_asic_obj) - self.get_state_info(port_name) + self.init_port_config_info(port_name) + self.init_port_appl_info(port_name) + port_asic_obj = self.init_asic_hostif_info(port_name) + self.init_asic_port_info(port_asic_obj) + self.init_state_port_info(port_name) return self.ret_temp - def get_config_info(self, port_name): + def init_port_config_info(self, port_name): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: @@ -37,7 +37,7 @@ def get_config_info(self, port_name): else: self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_appl_info(self, port_name): + def init_port_appl_info(self, port_name): req = MatchRequest(db="APPL_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: @@ -46,7 +46,7 @@ def get_appl_info(self, port_name): self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_state_info(self, port_name): + def init_state_port_info(self, port_name): req = MatchRequest(db="STATE_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) if not ret["error"] and len(ret["keys"]) != 0: @@ -54,7 +54,7 @@ def get_state_info(self, port_name): else: self.ret_temp[req.db]["tables_not_found"] = [req.table] - def get_asic_info_hostif(self, port_name): + def init_asic_hostif_info(self, port_name): req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF", key_pattern="*", field="SAI_HOSTIF_ATTR_NAME", value=port_name, return_fields=["SAI_HOSTIF_ATTR_OBJ_ID"], ns=self.ns) ret = self.match_engine.fetch(req) @@ -69,7 +69,7 @@ def get_asic_info_hostif(self, port_name): self.ret_temp[req.db]["tables_not_found"] = [req.table] return asic_port_obj_id - def get_asic_info_port(self, asic_port_obj_id): + def init_asic_port_info(self, asic_port_obj_id): if not asic_port_obj_id: self.ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") return From b92290ee88d11be73cb9120ad1eb87ca996a3fca Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 10 Jun 2021 04:45:29 +0000 Subject: [PATCH 10/22] Changes made Signed-off-by: Vivek Reddy Karri --- dump/plugins/__init__.py | 3 --- dump/plugins/executor.py | 1 - dump/plugins/port.py | 5 +---- tests/dump_tests/files/port/appl_db.json | 3 +-- tests/dump_tests/files/port/asic_db.json | 1 - tests/dump_tests/module_tests/mock_sonicv2connector.py | 1 - tests/dump_tests/module_tests/port_test.py | 4 +--- 7 files changed, 3 insertions(+), 15 deletions(-) diff --git a/dump/plugins/__init__.py b/dump/plugins/__init__.py index a10e90ec92..3bb0cdb16d 100644 --- a/dump/plugins/__init__.py +++ b/dump/plugins/__init__.py @@ -14,6 +14,3 @@ # Classes inheriting Executor dump_modules = {cls.__name__.lower(): cls for cls in Executor.__subclasses__()} - - - diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py index d918a4e4f0..921a3613d8 100644 --- a/dump/plugins/executor.py +++ b/dump/plugins/executor.py @@ -1,4 +1,3 @@ - from abc import ABC, abstractmethod class Executor(ABC): diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 5f71c05e06..090faf9727 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -79,7 +79,4 @@ def init_asic_port_info(self, asic_port_obj_id): sai_port_obj_key = ret["keys"][-1] self.ret_temp[req.db]["keys"].append(sai_port_obj_key) else: - self.ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") - - - \ No newline at end of file + self.ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") diff --git a/tests/dump_tests/files/port/appl_db.json b/tests/dump_tests/files/port/appl_db.json index bf228e6730..03ea8d279b 100644 --- a/tests/dump_tests/files/port/appl_db.json +++ b/tests/dump_tests/files/port/appl_db.json @@ -1,5 +1,4 @@ -{ - +{ "PORT_TABLE:Ethernet176": { "index": "0", "lanes": "0", diff --git a/tests/dump_tests/files/port/asic_db.json b/tests/dump_tests/files/port/asic_db.json index 3b06b38191..83dac66043 100644 --- a/tests/dump_tests/files/port/asic_db.json +++ b/tests/dump_tests/files/port/asic_db.json @@ -1,5 +1,4 @@ { - "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d":{ "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x100000000036a", diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 5a97cc5622..df49fa7b83 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -1,4 +1,3 @@ - import json, re, os from utilities_common.constants import DEFAULT_NAMESPACE diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 80289247b7..799fe32da8 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -117,6 +117,4 @@ def test_all_args(self): returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] ddiff = DeepDiff(expect, returned, ignore_order=True) - assert not ddiff, ddiff - - \ No newline at end of file + assert not ddiff, ddiff From 6d765140365b1e511ef338c5fccf961dcf796720 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 10 Jun 2021 18:35:35 +0000 Subject: [PATCH 11/22] Moved mock files to diff dir Signed-off-by: Vivek Reddy Karri --- .../files => dump_input}/port/appl_db.json | 0 .../files => dump_input}/port/asic_db.json | 0 .../files => dump_input}/port/config_db.json | 0 .../files => dump_input}/port/state_db.json | 0 .../dump_tests/module_tests/mock_sonicv2connector.py | 9 +++------ tests/dump_tests/module_tests/port_test.py | 11 +++++++---- 6 files changed, 10 insertions(+), 10 deletions(-) rename tests/{dump_tests/files => dump_input}/port/appl_db.json (100%) rename tests/{dump_tests/files => dump_input}/port/asic_db.json (100%) rename tests/{dump_tests/files => dump_input}/port/config_db.json (100%) rename tests/{dump_tests/files => dump_input}/port/state_db.json (100%) diff --git a/tests/dump_tests/files/port/appl_db.json b/tests/dump_input/port/appl_db.json similarity index 100% rename from tests/dump_tests/files/port/appl_db.json rename to tests/dump_input/port/appl_db.json diff --git a/tests/dump_tests/files/port/asic_db.json b/tests/dump_input/port/asic_db.json similarity index 100% rename from tests/dump_tests/files/port/asic_db.json rename to tests/dump_input/port/asic_db.json diff --git a/tests/dump_tests/files/port/config_db.json b/tests/dump_input/port/config_db.json similarity index 100% rename from tests/dump_tests/files/port/config_db.json rename to tests/dump_input/port/config_db.json diff --git a/tests/dump_tests/files/port/state_db.json b/tests/dump_input/port/state_db.json similarity index 100% rename from tests/dump_tests/files/port/state_db.json rename to tests/dump_input/port/state_db.json diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index df49fa7b83..938ca47617 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -3,9 +3,6 @@ class MockSonicV2Connector(): def __init__(self, dedicated_dbs, namespace=DEFAULT_NAMESPACE): - # TODO: Add Support for - # 1) Different namespace, i.e. asic0, asic1 etc - # 2) In the case when user-provider dedicated_dbs is empty, Read from mock_tables/*_db.json if namespace != DEFAULT_NAMESPACE: raise "This Mock doesn't support multi-asic configuration" self.db = None @@ -17,13 +14,13 @@ def __init__(self, dedicated_dbs, namespace=DEFAULT_NAMESPACE): def connect(self, db_name, retry=False): if db_name not in self.dedicated_dbs: - return None + raise Exception("{} not found. Available db's: {}".fomrat(db_name, self.dedicated_dbs.keys())) try: with open(self.dedicated_dbs[db_name]) as f: self.db = json.load(f) self.db_name_connect_to = db_name - except Exception as e: - raise "Connection Failure" + except BaseException as e: + raise "Connection Failure" + str(e) def get_db_separator(self, db_name): return self.db_cfg["DATABASES"][db_name]["separator"] diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 799fe32da8..a992cb5834 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -7,13 +7,16 @@ from dump.helper import create_template_dict, sort_lists from dump.plugins.port import Port +from .mock_sonicv2connector import MockSonicV2Connector + module_tests_path = os.path.dirname(__file__) dump_tests_path = os.path.join(module_tests_path, "../") -port_files_path = os.path.join(dump_tests_path,"files","port") -sys.path.append(dump_tests_path) -sys.path.append(module_tests_path) +tests_path = os.path.join(dump_tests_path, "../") +dump_test_input = os.path.join(tests_path,"dump_input") + -from module_tests.mock_sonicv2connector import MockSonicV2Connector +# Location for dedicated db's used for UT +port_files_path = os.path.join(dump_test_input,"port") dedicated_dbs = {} dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") From 244ff65811a4c69126e3c9e9aac2ddf3a3dd5e9f Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Sun, 13 Jun 2021 00:15:00 +0000 Subject: [PATCH 12/22] LGTM issue addressed Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 090faf9727..0bf3070734 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -1,4 +1,3 @@ -import os, sys from .executor import Executor from dump.match_infra import MatchEngine, MatchRequest from dump.helper import create_template_dict From 483cc527e249ce0a1fc19011ae1f36a59a4ae1de Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 23 Jun 2021 01:09:33 +0000 Subject: [PATCH 13/22] Multi-Asic Related Changes made Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/module_tests/mock_sonicv2connector.py | 4 ++-- tests/dump_tests/module_tests/port_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 938ca47617..296b828fa6 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -2,8 +2,8 @@ from utilities_common.constants import DEFAULT_NAMESPACE class MockSonicV2Connector(): - def __init__(self, dedicated_dbs, namespace=DEFAULT_NAMESPACE): - if namespace != DEFAULT_NAMESPACE: + def __init__(self, dedicated_dbs, **kwargs): + if "namespace" in kwargs and kwargs["namespace"] != DEFAULT_NAMESPACE: raise "This Mock doesn't support multi-asic configuration" self.db = None self.db_name_connect_to = None diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index a992cb5834..956eadf3a7 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -24,8 +24,8 @@ dedicated_dbs['ASIC_DB'] = os.path.join(port_files_path, "asic_db.json") dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") -def mock_connector(host, namespace): - return MockSonicV2Connector(dedicated_dbs, namespace) +def mock_connector(namespace, use_unix_socket_path=True): + return MockSonicV2Connector(dedicated_dbs, namespace=namespace, use_unix_socket_path=use_unix_socket_path) @pytest.fixture(scope="module", autouse=True) def verbosity_setup(): From 63f9aa44f7b5ccb117ee84441551b3ed19e6a20a Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 14 Jul 2021 20:30:39 +0000 Subject: [PATCH 14/22] Minor Fixes Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/module_tests/mock_sonicv2connector.py | 4 ++-- tests/dump_tests/module_tests/port_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 296b828fa6..ed85351b65 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -8,13 +8,13 @@ def __init__(self, dedicated_dbs, **kwargs): self.db = None self.db_name_connect_to = None self.dedicated_dbs = dedicated_dbs - db_config_path = os.path.join(os.path.dirname(__file__),"../../mock_tables/database_config.json") + db_config_path = os.path.join(os.path.dirname(__file__), "../../mock_tables/database_config.json") with open(db_config_path) as f: self.db_cfg = json.load(f) def connect(self, db_name, retry=False): if db_name not in self.dedicated_dbs: - raise Exception("{} not found. Available db's: {}".fomrat(db_name, self.dedicated_dbs.keys())) + raise Exception("{} not found. Available db's: {}".format(db_name, self.dedicated_dbs.keys())) try: with open(self.dedicated_dbs[db_name]) as f: self.db = json.load(f) diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 956eadf3a7..60ee107a8f 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -12,11 +12,11 @@ module_tests_path = os.path.dirname(__file__) dump_tests_path = os.path.join(module_tests_path, "../") tests_path = os.path.join(dump_tests_path, "../") -dump_test_input = os.path.join(tests_path,"dump_input") +dump_test_input = os.path.join(tests_path, "dump_input") # Location for dedicated db's used for UT -port_files_path = os.path.join(dump_test_input,"port") +port_files_path = os.path.join(dump_test_input, "port") dedicated_dbs = {} dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") From 94eedcc358e836e8810924f69db4cc424dde8b03 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Tue, 27 Jul 2021 18:40:51 +0000 Subject: [PATCH 15/22] Refactored and Added Comments Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 40 ++++++++++------------ tests/dump_tests/module_tests/port_test.py | 19 +++++++++- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 0bf3070734..c4a1043b68 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -3,7 +3,9 @@ from dump.helper import create_template_dict class Port(Executor): - + """ + Debug Dump Plugin for PORT Module + """ ARG_NAME = "port_name" def __init__(self): @@ -17,10 +19,10 @@ def get_all_args(self, ns=""): all_ports = ret["keys"] return [key.split("|")[-1] for key in all_ports] - def execute(self, params_dict): + def execute(self, params): self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) - port_name = params_dict[Port.ARG_NAME] - self.ns = params_dict["namespace"] + port_name = params[Port.ARG_NAME] + self.ns = params["namespace"] self.init_port_config_info(port_name) self.init_port_appl_info(port_name) port_asic_obj = self.init_asic_hostif_info(port_name) @@ -28,30 +30,28 @@ def execute(self, params_dict): self.init_state_port_info(port_name) return self.ret_temp + def add_to_ret_template(self, table, db, keys, err): + if not err and keys: + self.ret_temp[db]["keys"].extend(keys) + return True + else: + self.ret_temp[db]["tables_not_found"].extend([table]) + return False + def init_port_config_info(self, port_name): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) - if not ret["error"] and len(ret["keys"]) != 0: - self.ret_temp[req.db]["keys"] = ret["keys"] - else: - self.ret_temp[req.db]["tables_not_found"] = [req.table] + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) def init_port_appl_info(self, port_name): req = MatchRequest(db="APPL_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) - if not ret["error"] and len(ret["keys"]) != 0: - self.ret_temp[req.db]["keys"] = ret["keys"] - else: - self.ret_temp[req.db]["tables_not_found"] = [req.table] + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) - def init_state_port_info(self, port_name): req = MatchRequest(db="STATE_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) - if not ret["error"] and len(ret["keys"]) != 0: - self.ret_temp[req.db]["keys"] = ret["keys"] - else: - self.ret_temp[req.db]["tables_not_found"] = [req.table] + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) def init_asic_hostif_info(self, port_name): req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF", key_pattern="*", field="SAI_HOSTIF_ATTR_NAME", @@ -74,8 +74,4 @@ def init_asic_port_info(self, asic_port_obj_id): return req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_PORT", key_pattern=asic_port_obj_id, ns=self.ns) ret = self.match_engine.fetch(req) - if not ret["error"] and len(ret["keys"]) != 0: - sai_port_obj_key = ret["keys"][-1] - self.ret_temp[req.db]["keys"].append(sai_port_obj_key) - else: - self.ret_temp[req.db]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 60ee107a8f..ae73a934c8 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -38,8 +38,10 @@ def verbosity_setup(): @patch("dump.match_infra.SonicV2Connector", mock_connector) class TestPortModule(unittest.TestCase): - def test_working_state(self): + """ + Scenario: When the config is properly applied and propagated + """ params = {} params[Port.ARG_NAME] = "Ethernet176" params["namespace"] = "" @@ -55,6 +57,9 @@ def test_working_state(self): assert not ddiff, ddiff def test_missing_asic_port(self): + """ + Scenario: When the config was applied and just the SAI_OBJECT_TYPE_PORT is missing + """ params = {} params[Port.ARG_NAME] = "Ethernet160" params["namespace"] = "" @@ -70,6 +75,9 @@ def test_missing_asic_port(self): assert not ddiff, ddiff def test_missing_asic_hostif(self): + """ + Scenario: When the config was applied and it did not propagate to ASIC DB + """ params = {} params[Port.ARG_NAME] = "Ethernet164" params["namespace"] = "" @@ -85,6 +93,9 @@ def test_missing_asic_hostif(self): assert not ddiff, ddiff def test_missing_state_and_appl(self): + """ + Scenario: When the config was applied and it did not propagate to other db's + """ params = {} params[Port.ARG_NAME] = "Ethernet156" params["namespace"] = "" @@ -100,6 +111,9 @@ def test_missing_state_and_appl(self): assert not ddiff, ddiff def test_no_port(self): + """ + Scenario: When no entry for the port is present in any of the db's + """ params = {} params[Port.ARG_NAME] = "Ethernet152" params["namespace"] = "" @@ -115,6 +129,9 @@ def test_no_port(self): assert not ddiff, ddiff def test_all_args(self): + """ + Scenario: Verify Whether the get_all_args method is working as expected + """ params = {} m_port = Port() returned = m_port.get_all_args("") From 0db4c12efc6d97c3489858c7ec242703ef4c882e Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 29 Jul 2021 02:51:21 +0000 Subject: [PATCH 16/22] Refactored the port_test Signed-off-by: Vivek Reddy Karri --- dump/plugins/port.py | 2 +- tests/dump_tests/module_tests/port_test.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/dump/plugins/port.py b/dump/plugins/port.py index c4a1043b68..b8e80fb3d8 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -74,4 +74,4 @@ def init_asic_port_info(self, asic_port_obj_id): return req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_PORT", key_pattern=asic_port_obj_id, ns=self.ns) ret = self.match_engine.fetch(req) - self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) \ No newline at end of file diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index ae73a934c8..bcc134c974 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -42,9 +42,7 @@ def test_working_state(self): """ Scenario: When the config is properly applied and propagated """ - params = {} - params[Port.ARG_NAME] = "Ethernet176" - params["namespace"] = "" + params = {Port.ARG_NAME : "Ethernet176", "namespace" : ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -60,9 +58,7 @@ def test_missing_asic_port(self): """ Scenario: When the config was applied and just the SAI_OBJECT_TYPE_PORT is missing """ - params = {} - params[Port.ARG_NAME] = "Ethernet160" - params["namespace"] = "" + params = {Port.ARG_NAME : "Ethernet160", "namespace" : ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -78,9 +74,7 @@ def test_missing_asic_hostif(self): """ Scenario: When the config was applied and it did not propagate to ASIC DB """ - params = {} - params[Port.ARG_NAME] = "Ethernet164" - params["namespace"] = "" + params = {Port.ARG_NAME : "Ethernet164", "namespace" : ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -96,9 +90,7 @@ def test_missing_state_and_appl(self): """ Scenario: When the config was applied and it did not propagate to other db's """ - params = {} - params[Port.ARG_NAME] = "Ethernet156" - params["namespace"] = "" + params = {Port.ARG_NAME : "Ethernet156", "namespace" : ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -114,9 +106,7 @@ def test_no_port(self): """ Scenario: When no entry for the port is present in any of the db's """ - params = {} - params[Port.ARG_NAME] = "Ethernet152" - params["namespace"] = "" + params = {Port.ARG_NAME : "Ethernet152", "namespace" : ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -137,4 +127,4 @@ def test_all_args(self): returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] ddiff = DeepDiff(expect, returned, ignore_order=True) - assert not ddiff, ddiff + assert not ddiff, ddiff \ No newline at end of file From 8dac306f7150b036c920ee0a7ea5491468cf6cf3 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Sat, 14 Aug 2021 10:00:30 +0000 Subject: [PATCH 17/22] pep-8 issues handled Signed-off-by: Vivek Reddy Karri --- dump/plugins/__init__.py | 1 - dump/plugins/executor.py | 17 +++++----- dump/plugins/port.py | 31 ++++++++++--------- tests/dump_input/port/appl_db.json | 30 +++++++++--------- tests/dump_input/port/asic_db.json | 28 ++++++++--------- tests/dump_input/port/config_db.json | 20 ++++++------ tests/dump_input/port/state_db.json | 12 +++---- .../module_tests/mock_sonicv2connector.py | 15 ++++----- tests/dump_tests/module_tests/port_test.py | 25 ++++++++------- 9 files changed, 92 insertions(+), 87 deletions(-) diff --git a/dump/plugins/__init__.py b/dump/plugins/__init__.py index 3bb0cdb16d..8bbcb2372f 100644 --- a/dump/plugins/__init__.py +++ b/dump/plugins/__init__.py @@ -4,7 +4,6 @@ import importlib from .executor import Executor - dump_modules = {} pkg_dir = os.path.dirname(__file__) diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py index 921a3613d8..c6cbbb9342 100644 --- a/dump/plugins/executor.py +++ b/dump/plugins/executor.py @@ -1,16 +1,19 @@ from abc import ABC, abstractmethod + class Executor(ABC): - """ Abstract Class which should be extended from in order to be included in the dump state CLI """ - - ARG_NAME = "id" # Arg Identifier - CONFIG_FILE = "" # Path to config file, if any - + """ + Abstract Class which should be extended from in + order to be included in the dump state CLI + """ + + ARG_NAME = "id" # Arg Identifier + CONFIG_FILE = "" # Path to config file, if any + @abstractmethod def execute(self, params): pass - + @abstractmethod def get_all_args(self, ns): pass - diff --git a/dump/plugins/port.py b/dump/plugins/port.py index b8e80fb3d8..547e27bc20 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -1,24 +1,25 @@ -from .executor import Executor from dump.match_infra import MatchEngine, MatchRequest from dump.helper import create_template_dict +from .executor import Executor + class Port(Executor): """ Debug Dump Plugin for PORT Module """ ARG_NAME = "port_name" - + def __init__(self): self.match_engine = MatchEngine() self.ret_temp = {} self.ns = '' - + def get_all_args(self, ns=""): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", ns=ns) ret = self.match_engine.fetch(req) all_ports = ret["keys"] return [key.split("|")[-1] for key in all_ports] - + def execute(self, params): self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) port_name = params[Port.ARG_NAME] @@ -29,7 +30,7 @@ def execute(self, params): self.init_asic_port_info(port_asic_obj) self.init_state_port_info(port_name) return self.ret_temp - + def add_to_ret_template(self, table, db, keys, err): if not err and keys: self.ret_temp[db]["keys"].extend(keys) @@ -37,28 +38,28 @@ def add_to_ret_template(self, table, db, keys, err): else: self.ret_temp[db]["tables_not_found"].extend([table]) return False - + def init_port_config_info(self, port_name): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) - + def init_port_appl_info(self, port_name): req = MatchRequest(db="APPL_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) - + def init_state_port_info(self, port_name): req = MatchRequest(db="STATE_DB", table="PORT_TABLE", key_pattern=port_name, ns=self.ns) ret = self.match_engine.fetch(req) self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) - + def init_asic_hostif_info(self, port_name): - req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF", key_pattern="*", field="SAI_HOSTIF_ATTR_NAME", + req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF", key_pattern="*", field="SAI_HOSTIF_ATTR_NAME", value=port_name, return_fields=["SAI_HOSTIF_ATTR_OBJ_ID"], ns=self.ns) ret = self.match_engine.fetch(req) asic_port_obj_id = "" - + if not ret["error"] and len(ret["keys"]) != 0: self.ret_temp[req.db]["keys"] = ret["keys"] sai_hostif_obj_key = ret["keys"][-1] @@ -67,11 +68,11 @@ def init_asic_hostif_info(self, port_name): else: self.ret_temp[req.db]["tables_not_found"] = [req.table] return asic_port_obj_id - + def init_asic_port_info(self, asic_port_obj_id): - if not asic_port_obj_id: + if not asic_port_obj_id: self.ret_temp["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") - return + return None req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_PORT", key_pattern=asic_port_obj_id, ns=self.ns) ret = self.match_engine.fetch(req) - self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) \ No newline at end of file + self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"]) diff --git a/tests/dump_input/port/appl_db.json b/tests/dump_input/port/appl_db.json index 03ea8d279b..6dca414473 100644 --- a/tests/dump_input/port/appl_db.json +++ b/tests/dump_input/port/appl_db.json @@ -1,16 +1,16 @@ -{ - "PORT_TABLE:Ethernet176": { - "index": "0", - "lanes": "0", - "alias": "etp45", - "speed": "25000", - "oper_status": "up", - "pfc_asym": "off", - "mtu": "9100", - "fec": "rs", - "admin_status": "up" - }, - "PORT_TABLE:Ethernet160": { +{ + "PORT_TABLE:Ethernet176": { + "index": "0", + "lanes": "0", + "alias": "etp45", + "speed": "25000", + "oper_status": "up", + "pfc_asym": "off", + "mtu": "9100", + "fec": "rs", + "admin_status": "up" + }, + "PORT_TABLE:Ethernet160": { "index": "0", "lanes": "0", "alias": "etp41", @@ -21,7 +21,7 @@ "fec": "rs", "admin_status": "up" }, - "PORT_TABLE:Ethernet164": { + "PORT_TABLE:Ethernet164": { "index": "0", "lanes": "0", "alias": "etp42", @@ -32,4 +32,4 @@ "fec": "rs", "admin_status": "up" } -} +} \ No newline at end of file diff --git a/tests/dump_input/port/asic_db.json b/tests/dump_input/port/asic_db.json index 83dac66043..db9e86128b 100644 --- a/tests/dump_input/port/asic_db.json +++ b/tests/dump_input/port/asic_db.json @@ -1,27 +1,27 @@ { - "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d":{ - "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", - "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x100000000036a", - "SAI_HOSTIF_ATTR_NAME" : "Ethernet176", - "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" - }, - "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x100000000036a": { - "SAI_PORT_ATTR_ADMIN_STATE" : "true", - "SAI_PORT_ATTR_SPEED" : "25000", - "SAI_PORT_ATTR_MTU" : "9122" - }, - "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a49":{ + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d":{ + "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", + "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x100000000036a", + "SAI_HOSTIF_ATTR_NAME" : "Ethernet176", + "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x100000000036a": { + "SAI_PORT_ATTR_ADMIN_STATE" : "true", + "SAI_PORT_ATTR_SPEED" : "25000", + "SAI_PORT_ATTR_MTU" : "9122" + }, + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a49":{ "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x10000000002e6", "SAI_HOSTIF_ATTR_NAME" : "Ethernet160", "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" }, - "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4a":{ + "ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4a":{ "SAI_HOSTIF_ATTR_TYPE" : "SAI_HOSTIF_TYPE_NETDEV", "SAI_HOSTIF_ATTR_OBJ_ID": "oid:0x1000000000307", "SAI_HOSTIF_ATTR_OPER_STATUS" : "true" }, - "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x1000000000307": { + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x1000000000307": { "SAI_PORT_ATTR_ADMIN_STATE" : "true", "SAI_PORT_ATTR_SPEED" : "25000", "SAI_PORT_ATTR_MTU" : "9122" diff --git a/tests/dump_input/port/config_db.json b/tests/dump_input/port/config_db.json index a5a5dd967b..890520f4b1 100644 --- a/tests/dump_input/port/config_db.json +++ b/tests/dump_input/port/config_db.json @@ -1,26 +1,26 @@ { - "PORT|Ethernet176": { - "admin_status" : "up", - "alias": "etp45", - "index": "45", - "lanes": "176", - "speed": "25000" - }, - "PORT|Ethernet164": { + "PORT|Ethernet176": { + "admin_status" : "up", + "alias": "etp45", + "index": "45", + "lanes": "176", + "speed": "25000" + }, + "PORT|Ethernet164": { "admin_status" : "up", "alias": "etp42", "index": "42", "lanes": "164", "speed": "25000" }, - "PORT|Ethernet160": { + "PORT|Ethernet160": { "admin_status" : "up", "alias": "etp41", "index": "41", "lanes": "160", "speed": "25000" }, - "PORT|Ethernet156": { + "PORT|Ethernet156": { "admin_status" : "up", "alias": "etp40", "index": "40", diff --git a/tests/dump_input/port/state_db.json b/tests/dump_input/port/state_db.json index 3cd0b62de6..9db2d8f8c2 100644 --- a/tests/dump_input/port/state_db.json +++ b/tests/dump_input/port/state_db.json @@ -1,13 +1,13 @@ { - "PORT_TABLE|Ethernet176":{ - "state" : "ok", - "netdev_oper_status" : "up" - }, - "PORT_TABLE|Ethernet160":{ + "PORT_TABLE|Ethernet176":{ "state" : "ok", "netdev_oper_status" : "up" }, - "PORT_TABLE|Ethernet164":{ + "PORT_TABLE|Ethernet160":{ + "state" : "ok", + "netdev_oper_status" : "up" + }, + "PORT_TABLE|Ethernet164":{ "state" : "ok", "netdev_oper_status" : "up" } diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index ed85351b65..052d5e2351 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -1,6 +1,7 @@ import json, re, os from utilities_common.constants import DEFAULT_NAMESPACE + class MockSonicV2Connector(): def __init__(self, dedicated_dbs, **kwargs): if "namespace" in kwargs and kwargs["namespace"] != DEFAULT_NAMESPACE: @@ -11,26 +12,26 @@ def __init__(self, dedicated_dbs, **kwargs): db_config_path = os.path.join(os.path.dirname(__file__), "../../mock_tables/database_config.json") with open(db_config_path) as f: self.db_cfg = json.load(f) - + def connect(self, db_name, retry=False): if db_name not in self.dedicated_dbs: raise Exception("{} not found. Available db's: {}".format(db_name, self.dedicated_dbs.keys())) try: with open(self.dedicated_dbs[db_name]) as f: self.db = json.load(f) - self.db_name_connect_to = db_name + self.db_name_connect_to = db_name except BaseException as e: raise "Connection Failure" + str(e) - + def get_db_separator(self, db_name): return self.db_cfg["DATABASES"][db_name]["separator"] - + def keys(self, db_name, pattern): if not self.db: raise "MockDB Not Connected" if self.db_name_connect_to != db_name: raise "Failed to find {} in the MockDB".format(db_name) - + pattern = re.escape(pattern) pattern = pattern.replace("\\*", ".*") filtered_keys = [] @@ -39,7 +40,7 @@ def keys(self, db_name, pattern): if re.match(pattern, key): filtered_keys.append(key) return filtered_keys - + def get_all(self, db_name, key): if not self.db: raise "MockDB Not Connected" @@ -57,7 +58,7 @@ def get(self, db_name, key, field): if key not in self.db or field not in self.db[key]: return "" return self.db[key][field] - + def hexists(self, db_name, key, field): if not self.db: raise "MockDB Not Connected" diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index bcc134c974..9ca892661a 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -14,7 +14,6 @@ tests_path = os.path.join(dump_tests_path, "../") dump_test_input = os.path.join(tests_path, "dump_input") - # Location for dedicated db's used for UT port_files_path = os.path.join(dump_test_input, "port") @@ -24,9 +23,11 @@ dedicated_dbs['ASIC_DB'] = os.path.join(port_files_path, "asic_db.json") dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") + def mock_connector(namespace, use_unix_socket_path=True): return MockSonicV2Connector(dedicated_dbs, namespace=namespace, use_unix_socket_path=use_unix_socket_path) + @pytest.fixture(scope="module", autouse=True) def verbosity_setup(): print("SETUP") @@ -42,7 +43,7 @@ def test_working_state(self): """ Scenario: When the config is properly applied and propagated """ - params = {Port.ARG_NAME : "Ethernet176", "namespace" : ""} + params = {Port.ARG_NAME: "Ethernet176", "namespace": ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -53,12 +54,12 @@ def test_working_state(self): expect["ASIC_DB"]["keys"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF:oid:0xd000000000a4d") ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff - + def test_missing_asic_port(self): """ Scenario: When the config was applied and just the SAI_OBJECT_TYPE_PORT is missing """ - params = {Port.ARG_NAME : "Ethernet160", "namespace" : ""} + params = {Port.ARG_NAME: "Ethernet160", "namespace": ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -69,12 +70,12 @@ def test_missing_asic_port(self): expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_PORT") ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff - + def test_missing_asic_hostif(self): """ Scenario: When the config was applied and it did not propagate to ASIC DB """ - params = {Port.ARG_NAME : "Ethernet164", "namespace" : ""} + params = {Port.ARG_NAME: "Ethernet164", "namespace": ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -85,12 +86,12 @@ def test_missing_asic_hostif(self): expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - + def test_missing_state_and_appl(self): """ Scenario: When the config was applied and it did not propagate to other db's """ - params = {Port.ARG_NAME : "Ethernet156", "namespace" : ""} + params = {Port.ARG_NAME: "Ethernet156", "namespace": ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -101,12 +102,12 @@ def test_missing_state_and_appl(self): expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - + def test_no_port(self): """ Scenario: When no entry for the port is present in any of the db's """ - params = {Port.ARG_NAME : "Ethernet152", "namespace" : ""} + params = {Port.ARG_NAME: "Ethernet152", "namespace": ""} m_port = Port() returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) @@ -117,7 +118,7 @@ def test_no_port(self): expect["ASIC_DB"]["tables_not_found"].append("ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF") ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - + def test_all_args(self): """ Scenario: Verify Whether the get_all_args method is working as expected @@ -127,4 +128,4 @@ def test_all_args(self): returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] ddiff = DeepDiff(expect, returned, ignore_order=True) - assert not ddiff, ddiff \ No newline at end of file + assert not ddiff, ddiff \ No newline at end of file From dc4333945bb140b3732b1a6fe5512c11f11c2bc8 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Tue, 17 Aug 2021 05:45:25 +0000 Subject: [PATCH 18/22] Remaining pep8 fixes Signed-off-by: Vivek Reddy Karri --- dump/plugins/__init__.py | 4 +- tests/dump_tests/match_engine_test.py | 100 +++++++++--------- .../module_tests/mock_sonicv2connector.py | 4 +- tests/dump_tests/module_tests/port_test.py | 10 +- 4 files changed, 62 insertions(+), 56 deletions(-) diff --git a/dump/plugins/__init__.py b/dump/plugins/__init__.py index 8bbcb2372f..2141e4fec8 100644 --- a/dump/plugins/__init__.py +++ b/dump/plugins/__init__.py @@ -6,8 +6,8 @@ dump_modules = {} pkg_dir = os.path.dirname(__file__) - -# import child classes automatically + +# import child classes automatically for (module_loader, name, ispkg) in pkgutil.iter_modules([pkg_dir]): importlib.import_module('.' + name, __package__) diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index a4d4330b9b..5575bc4127 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -1,4 +1,5 @@ -import os, sys +import os +import sys import unittest import pytest from dump.match_infra import MatchEngine, EXCEP_DICT, MatchRequest @@ -10,6 +11,7 @@ sys.path.append(test_path) + @pytest.fixture(scope="module", autouse=True) def mock_setup(): print("SETUP") @@ -20,11 +22,11 @@ def mock_setup(): class TestMatchRequestValidation(unittest.TestCase): - + def __init__(self, *args, **kwargs): super(TestMatchRequestValidation, self).__init__(*args, **kwargs) self.match_engine = MatchEngine() - + def assertRaisesWithMessage(self, msg, func, *args, **kwargs): try: func(*args, **kwargs) @@ -32,111 +34,111 @@ def assertRaisesWithMessage(self, msg, func, *args, **kwargs): except Exception as inst: print(inst) assert msg in str(inst) - + def test_bad_request(self): req = [] ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["INV_REQ"] - + def test_no_source(self): self.assertRaisesWithMessage(EXCEP_DICT["NO_SRC"], MatchRequest) - - def test_vague_source(self): + + def test_vague_source(self): self.assertRaisesWithMessage(EXCEP_DICT["SRC_VAGUE"], MatchRequest, db="CONFIG_DB", file="/etc/sonic/copp_cfg.json") - - def test_no_file(self): + + def test_no_file(self): self.assertRaisesWithMessage(EXCEP_DICT["FILE_R_EXEP"], MatchRequest, file=os.path.join(test_path, "random_db.json")) - def test_invalid_db(self): + def test_invalid_db(self): self.assertRaisesWithMessage(EXCEP_DICT["INV_DB"], MatchRequest, db="CONFIGURATION_DB") - + def test_invalid_namespace(self): - self.assertRaisesWithMessage(EXCEP_DICT["INV_NS"], MatchRequest, db="APPL_DB", table="PORT_TABLE", + self.assertRaisesWithMessage(EXCEP_DICT["INV_NS"], MatchRequest, db="APPL_DB", table="PORT_TABLE", field="lanes", value="202", ns="asic4") - + def test_bad_key_pattern(self): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="") ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["NO_KEY"] - + def test_no_value(self): self.assertRaisesWithMessage(EXCEP_DICT["NO_VALUE"], MatchRequest, db="APPL_DB", table="COPP_TABLE", key_pattern="*", field="trap_ids", value="") - + def test_no_table(self): self.assertRaisesWithMessage(EXCEP_DICT["NO_TABLE"], MatchRequest, db="APPL_DB", table="", key_pattern="*", field="trap_ids", value="bgpv6") - + def test_just_keys_return_fields_compat(self): - self.assertRaisesWithMessage(EXCEP_DICT["JUST_KEYS_COMPAT"], MatchRequest, db="APPL_DB", return_fields=["trap_group"], table="COPP_TABLE", + self.assertRaisesWithMessage(EXCEP_DICT["JUST_KEYS_COMPAT"], MatchRequest, db="APPL_DB", return_fields=["trap_group"], table="COPP_TABLE", key_pattern="*", field="trap_ids", value="", just_keys=False) - + def test_invalid_combination(self): req = MatchRequest(db="CONFIG_DB", table="COPP_TRAP", key_pattern="*", field="trap_ids", value="sample_packet") ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["NO_MATCHES"] - + def test_return_fields_bad_format(self): self.assertRaisesWithMessage(EXCEP_DICT["BAD_FORMAT_RE_FIELDS"], MatchRequest, db="STATE_DB", table="REBOOT_CAUSE", key_pattern="*", return_fields="cause") - + def test_valid_match_request(self): try: req = MatchRequest(db="APPL_DB", table="PORT_TABLE", field="lanes", value="202") except Exception as e: assert False, "Exception Raised for a Valid MatchRequest" + str(e) - + class TestMatchEngine(unittest.TestCase): - + def __init__(self, *args, **kwargs): super(TestMatchEngine, self).__init__(*args, **kwargs) self.match_engine = MatchEngine() - + def test_key_pattern_wildcard(self): req = MatchRequest(db="CONFIG_DB", table="SFLOW_COLLECTOR", key_pattern="*") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 2 assert "SFLOW_COLLECTOR|ser5" in ret['keys'] - assert "SFLOW_COLLECTOR|prod" in ret['keys'] - + assert "SFLOW_COLLECTOR|prod" in ret['keys'] + def test_key_pattern_complex(self): req = MatchRequest(db="CONFIG_DB", table="ACL_RULE", key_pattern="EVERFLOW*") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 2 assert "ACL_RULE|EVERFLOW|RULE_6" in ret['keys'] - assert "ACL_RULE|EVERFLOW|RULE_08" in ret['keys'] - + assert "ACL_RULE|EVERFLOW|RULE_08" in ret['keys'] + def test_field_value_match(self): req = MatchRequest(db="CONFIG_DB", table="ACL_TABLE", field="policy_desc", value="SSH_ONLY") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "ACL_TABLE|SSH_ONLY" in ret['keys'] - + def test_field_value_match_list_type(self): req = MatchRequest(db="APPL_DB", table="PORT_TABLE", field="lanes", value="202") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "PORT_TABLE:Ethernet200" in ret['keys'] - + def test_for_no_match(self): req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_SWITCH", field="SAI_SWITCH_ATTR_SRC_MAC_ADDRESS", value="DE:AD:EE:EE:EE") ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["NO_ENTRIES"] assert len(ret["keys"]) == 0 - + def test_for_no_key_match(self): req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_SWITCH", key_pattern="oid:0x22*") ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["NO_MATCHES"] - + def test_field_value_no_match(self): req = MatchRequest(db="STATE_DB", table="FAN_INFO", key_pattern="*", field="led_status", value="yellow") ret = self.match_engine.fetch(req) assert ret["error"] == EXCEP_DICT["NO_ENTRIES"] assert len(ret["keys"]) == 0 - + def test_return_keys(self): req = MatchRequest(db="STATE_DB", table="REBOOT_CAUSE", return_fields=["cause"]) ret = self.match_engine.fetch(req) @@ -144,21 +146,21 @@ def test_return_keys(self): assert len(ret["keys"]) == 2 assert "warm-reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_04_53_58"]["cause"] assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] - + def test_return_fields_with_key_filtering(self): req = MatchRequest(db="STATE_DB", table="REBOOT_CAUSE", key_pattern="2020_10_09_02*", return_fields=["cause"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] - + def test_return_fields_with_field_value_filtering(self): req = MatchRequest(db="STATE_DB", table="CHASSIS_MODULE_TABLE", field="oper_status", value="Offline", return_fields=["slot"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "18" == ret["return_values"]["CHASSIS_MODULE_TABLE|FABRIC-CARD1"]["slot"] - + def test_return_fields_with_all_filtering(self): req = MatchRequest(db="STATE_DB", table="VXLAN_TUNNEL_TABLE", key_pattern="EVPN_25.25.25.2*", field="operstatus", value="down", return_fields=["src_ip"]) ret = self.match_engine.fetch(req) @@ -167,7 +169,7 @@ def test_return_fields_with_all_filtering(self): assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.25"]["src_ip"] assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.26"]["src_ip"] assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.27"]["src_ip"] - + def test_just_keys_false(self): req = MatchRequest(db="CONFIG_DB", table="SFLOW", key_pattern="global", just_keys=False) ret = self.match_engine.fetch(req) @@ -178,15 +180,15 @@ def test_just_keys_false(self): exp_dict = {"SFLOW|global": {"admin_state": "up", "polling_interval": "0"}} ddiff = DeepDiff(exp_dict, recv_dict) assert not ddiff, ddiff - + def test_file_source(self): file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_TRAP", field="trap_ids", value="arp_req") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 - assert "COPP_TRAP|arp" in ret["keys"] - + assert "COPP_TRAP|arp" in ret["keys"] + def test_file_source_with_key_ptrn(self): file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop") @@ -194,7 +196,7 @@ def test_file_source_with_key_ptrn(self): assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "COPP_GROUP|queue4_group2" in ret["keys"] - + def test_file_source_with_not_only_return_keys(self): file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop", just_keys=False) @@ -205,17 +207,17 @@ def test_file_source_with_not_only_return_keys(self): exp_dict = {"COPP_GROUP|queue4_group2": {"trap_action": "copy", "trap_priority": "4", "queue": "4", "meter_type": "packets", "mode": "sr_tcm", "cir": "600", "cbs": "600", "red_action": "drop"}} ddiff = DeepDiff(exp_dict, recv_dict) assert not ddiff, ddiff - + def test_match_entire_list(self): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="lanes", value="61,62,63,64", match_entire_list=True, just_keys=True) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 - assert "PORT|Ethernet60" in ret["keys"] - + assert "PORT|Ethernet60" in ret["keys"] + class TestNonDefaultNameSpace(unittest.TestCase): - + @classmethod def setup_class(cls): print("SETUP") @@ -225,12 +227,12 @@ def setup_class(cls): reload(mock_multi_asic) from ..mock_tables import dbconnector dbconnector.load_namespace_config() - + def teardown_class(cls): print("TEARDOWN") os.environ["UTILITIES_UNIT_TESTING"] = "0" - os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "" - + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "" + def test_namespace_asic0(self): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="asic_port_name", value="Eth0-ASIC0", ns="asic0") match_engine = MatchEngine() @@ -238,11 +240,11 @@ def test_namespace_asic0(self): assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "PORT|Ethernet0" in ret["keys"] - + def test_namespace_asic1(self): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="Ethernet-BP256", ns="asic1") match_engine = MatchEngine() ret = match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 - assert "PORT|Ethernet-BP256" in ret["keys"] + assert "PORT|Ethernet-BP256" in ret["keys"] diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py index 052d5e2351..f6ba18981e 100644 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ b/tests/dump_tests/module_tests/mock_sonicv2connector.py @@ -1,4 +1,6 @@ -import json, re, os +import json +import re +import os from utilities_common.constants import DEFAULT_NAMESPACE diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 9ca892661a..9886df014a 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -1,4 +1,6 @@ -import json, os, sys +import json +import os +import sys import jsonpatch import unittest import pytest @@ -18,8 +20,8 @@ port_files_path = os.path.join(dump_test_input, "port") dedicated_dbs = {} -dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") -dedicated_dbs['APPL_DB'] = os.path.join(port_files_path, "appl_db.json") +dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") +dedicated_dbs['APPL_DB'] = os.path.join(port_files_path, "appl_db.json") dedicated_dbs['ASIC_DB'] = os.path.join(port_files_path, "asic_db.json") dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") @@ -128,4 +130,4 @@ def test_all_args(self): returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] ddiff = DeepDiff(expect, returned, ignore_order=True) - assert not ddiff, ddiff \ No newline at end of file + assert not ddiff, ddiff From 1eb3be805d628cb16f056e36e0eb2ce7d6433675 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 19 Aug 2021 11:05:12 +0000 Subject: [PATCH 19/22] pep-8 fixes and introduced connection pool optimization Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 190 +++++++++++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 72 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index fe0d8126a9..31f22058a8 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -1,4 +1,5 @@ -import json, fnmatch +import json +import fnmatch from abc import ABC, abstractmethod from dump.helper import verbose_print from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig @@ -12,9 +13,9 @@ "NO_SRC": "Either one of db or file in the request should be non-empty", "NO_TABLE": "No 'table' name provided", "NO_KEY": "'key_pattern' cannot be empty", - "NO_VALUE" : "Field is provided, but no value is provided to compare with", + "NO_VALUE": "Field is provided, but no value is provided to compare with", "SRC_VAGUE": "Only one of db or file should be provided", - "CONN_ERR" : "Connection Error", + "CONN_ERR": "Connection Error", "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty", "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type", "NO_ENTRIES": "No Keys found after applying the filtering criteria", @@ -22,14 +23,15 @@ "INV_NS": "Namespace is invalid" } + class MatchRequest: - """ - Request Object which should be passed to the MatchEngine - + """ + Request Object which should be passed to the MatchEngine + Attributes: "table" : A Valid Table Name "key_pattern" : Pattern of the redis-key to match. Defaults to "*". Eg: "*" will match all the keys. - Supports these glob style patterns. https://redis.io/commands/KEYS + Supports these glob style patterns. https://redis.io/commands/KEYS "field" : Field to check for a match,Defaults to None "value" : Value to match, Defaults to None "return_fields" : An iterable type, where each element woudld imply a field to return from all the filtered keys @@ -38,9 +40,10 @@ class MatchRequest: Only one of the db/file fields should have a non-empty string. "just_keys" : If true, Only Returns the keys matched. Does not return field-value pairs. Defaults to True "ns" : namespace argument, if nothing is provided, default namespace is used - "match_entire_list" : When this arg is set to true, entire list is matched incluing the ",". + "match_entire_list" : When this arg is set to true, entire list is matched incluing the ",". When False, the values are split based on "," and individual items are matched with """ + def __init__(self, **kwargs): self.table = kwargs["table"] if "table" in kwargs else None self.key_pattern = kwargs["key_pattern"] if "key_pattern" in kwargs else "*" @@ -56,16 +59,15 @@ def __init__(self, **kwargs): verbose_print(str(err)) if err: raise Exception("Static Checks for the MatchRequest Failed, Reason: \n" + err) - - + def __static_checks(self): - + if not self.db and not self.file: return EXCEP_DICT["NO_SRC"] - + if self.db and self.file: return EXCEP_DICT["SRC_VAGUE"] - + if not self.db: try: with open(self.file) as f: @@ -75,32 +77,32 @@ def __static_checks(self): if not self.file and self.db not in SonicDBConfig.getDbList(): return EXCEP_DICT["INV_DB"] - + if not self.table: return EXCEP_DICT["NO_TABLE"] - + if not isinstance(self.return_fields, list): return EXCEP_DICT["BAD_FORMAT_RE_FIELDS"] - + if not self.just_keys and self.return_fields: return EXCEP_DICT["JUST_KEYS_COMPAT"] - + if self.field and not self.value: return EXCEP_DICT["NO_VALUE"] - + if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic.get_namespace_list(): return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic.get_namespace_list()) - + verbose_print("MatchRequest Checks Passed") - + return "" - + def __str__(self): str = "----------------------- \n MatchRequest: \n" if self.db: str += "db:{} , ".format(self.db) if self.file: - str += "file:{} , ".format(self.file) + str += "file:{} , ".format(self.file) if self.table: str += "table:{} , ".format(self.table) if self.key_pattern: @@ -116,78 +118,76 @@ def __str__(self): if len(self.return_fields) > 0: str += "return_fields: " + ",".join(self.return_fields) + " " if self.ns: - str += "namespace: , " + self.ns + str += "namespace: , " + self.ns if self.match_entire_list: str += "match_list: True , " else: str += "match_list: False , " return str - + + class SourceAdapter(ABC): """ Source Adaptor offers unified interface to Data Sources """ - + def __init__(self): pass - + @abstractmethod def connect(self, db, ns): """ Return True for Success, False for failure """ return False - + @abstractmethod def getKeys(self, db, table, key_pattern): return [] - + @abstractmethod def get(self, db, key): return {} - + @abstractmethod def hget(self, db, key, field): return "" - + @abstractmethod def get_separator(self, db): return "" - + + class RedisSource(SourceAdapter): """ Concrete Adaptor Class for connecting to Redis Data Sources """ - - def __init__(self): - self.conn = None - + + def __init__(self, conn_pool): + self.conn = None + self.pool = conn_pool + def connect(self, db, ns): try: - if not SonicDBConfig.isInit(): - if multi_asic.is_multi_asic(): - SonicDBConfig.load_sonic_global_db_config() - else: - SonicDBConfig.load_sonic_db_config() - self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True) - self.conn.connect(db) + self.conn = self.pool.get(db, ns) except Exception as e: verbose_print("RedisSource: Connection Failed\n" + str(e)) return False return True - + def get_separator(self, db): return self.conn.get_db_separator(db) - - def getKeys(self, db, table, key_pattern): + + def getKeys(self, db, table, key_pattern): return self.conn.keys(db, table + self.get_separator(db) + key_pattern) - + def get(self, db, key): return self.conn.get_all(db, key) - + def hget(self, db, key, field): return self.conn.get(db, key, field) + class JsonSource(SourceAdapter): """ Concrete Adaptor Class for connecting to JSON Data Sources """ - + def __init__(self): self.json_data = None - + def connect(self, db, ns): try: with open(db) as f: @@ -196,10 +196,10 @@ def connect(self, db, ns): verbose_print("JsonSource: Loading the JSON file failed" + str(e)) return False return True - + def get_separator(self, db): return SonicDBConfig.getSeparator("CONFIG_DB") - + def getKeys(self, db, table, key_pattern): if table not in self.json_data: return [] @@ -207,48 +207,95 @@ def getKeys(self, db, table, key_pattern): kp = key_pattern.replace("[^", "[!") kys = fnmatch.filter(self.json_data[table].keys(), kp) return [table + self.get_separator(db) + ky for ky in kys] - + def get(self, db, key): sep = self.get_separator(db) table, key = key.split(sep, 1) return self.json_data.get(table, {}).get(key, {}) - + def hget(self, db, key, field): sep = self.get_separator(db) table, key = key.split(sep, 1) return self.json_data.get(table, "").get(key, "").get(field, "") - + + +class ConnectionPool: + """ Caches SonicV2Connector objects for effective reuse """ + def __init__(self): + self.cache = dict() # Pool of SonicV2Connector objects + + def initialize_connector(self, ns): + if not SonicDBConfig.isInit(): + if multi_asic.is_multi_asic(): + SonicDBConfig.load_sonic_global_db_config() + else: + SonicDBConfig.load_sonic_db_config() + return SonicV2Connector(namespace=ns, use_unix_socket_path=True) + + def get(self, db_name, ns, update=False): + """ Returns a SonicV2Connector Object and caches it for further requests """ + if ns not in self.cache: + self.cache[ns] = {} + self.cache[ns]["conn"] = self.initialize_connector(ns) + self.cache[ns]["connected_to"] = set() + if update or db_name not in self.cache[ns]["connected_to"]: + self.cache[ns]["conn"].connect(db_name) + self.cache[ns]["connected_to"].add(db_name) + return self.cache[ns]["conn"] + + def clear(namespace=None): + if not namespace: + self.cache.clear() + elif namespace in self.cache: + del self.cache[namespace] + + class MatchEngine: - """ Pass in a MatchRequest, to fetch the Matched dump from the Data sources """ - + """ + Provide a MatchRequest to fetch the relevant keys/fv's from the data source + Usage Guidelines: + 1) Instantiate the class once for the entire execution, + to effectively use the caching of redis connection objects + """ + def __init__(self, pool=None): + if not isinstance(pool, ConnectionPool): + self.conn_pool = ConnectionPool() + else: + self.conn_pool = pool + + def clear_cache(ns): + self.conn_pool(ns) + def __get_source_adapter(self, req): src = None d_src = "" if req.db: d_src = req.db - src = RedisSource() + src = RedisSource(self.conn_pool) else: d_src = req.file src = JsonSource() return d_src, src - + def __create_template(self): - return {"error" : "", "keys" : [], "return_values" : {}} - + return {"error": "", "keys": [], "return_values": {}} + def __display_error(self, err): template = self.__create_template() template['error'] = err verbose_print("MatchEngine: \n" + template['error']) return template - + def __filter_out_keys(self, src, req, all_matched_keys): # TODO: Custom Callbacks for Complex Matching Criteria if not req.field: return all_matched_keys - + filtered_keys = [] for key in all_matched_keys: f_values = src.hget(req.db, key, req.field) + if not f_values: + continue if "," in f_values and not req.match_entire_list: f_value = f_values.split(",") else: @@ -256,7 +303,7 @@ def __filter_out_keys(self, src, req, all_matched_keys): if req.value in f_value: filtered_keys.append(key) return filtered_keys - + def __fill_template(self, src, req, filtered_keys, template): for key in filtered_keys: temp = {} @@ -266,35 +313,34 @@ def __fill_template(self, src, req, filtered_keys, template): elif len(req.return_fields) > 0: template["keys"].append(key) template["return_values"][key] = {} - for field in req.return_fields: + for field in req.return_fields: template["return_values"][key][field] = src.hget(req.db, key, field) else: template["keys"].append(key) verbose_print("Return Values:" + str(template["return_values"])) return template - + def fetch(self, req): """ Given a request obj, find its match in the data source provided """ if not isinstance(req, MatchRequest): return self.__display_error(EXCEP_DICT["INV_REQ"]) - + verbose_print(str(req)) - + if not req.key_pattern: return self.__display_error(EXCEP_DICT["NO_KEY"]) - + d_src, src = self.__get_source_adapter(req) if not src.connect(d_src, req.ns): return self.__display_error(EXCEP_DICT["CONN_ERR"]) - + template = self.__create_template() all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern) if not all_matched_keys: return self.__display_error(EXCEP_DICT["NO_MATCHES"]) - + filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) verbose_print("Filtered Keys:" + str(filtered_keys)) if not filtered_keys: return self.__display_error(EXCEP_DICT["NO_ENTRIES"]) - return self.__fill_template(src, req, filtered_keys, template) - + return self.__fill_template(src, req, filtered_keys, template) \ No newline at end of file From 663f46becf97c4af7e20f5f42e930650cc765175 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 19 Aug 2021 11:07:26 +0000 Subject: [PATCH 20/22] Minor changes owing to connection pool addition Signed-off-by: Vivek Reddy Karri --- dump/plugins/executor.py | 7 +++++++ dump/plugins/port.py | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dump/plugins/executor.py b/dump/plugins/executor.py index c6cbbb9342..73c324de80 100644 --- a/dump/plugins/executor.py +++ b/dump/plugins/executor.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from dump.match_infra import MatchEngine class Executor(ABC): @@ -10,6 +11,12 @@ class Executor(ABC): ARG_NAME = "id" # Arg Identifier CONFIG_FILE = "" # Path to config file, if any + def __init__(self, match_engine=None): + if not isinstance(match_engine, MatchEngine): + self.match_engine = MatchEngine(None) + else: + self.match_engine = match_engine + @abstractmethod def execute(self, params): pass diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 547e27bc20..7b078505ee 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -9,10 +9,8 @@ class Port(Executor): """ ARG_NAME = "port_name" - def __init__(self): - self.match_engine = MatchEngine() - self.ret_temp = {} - self.ns = '' + def __init__(self, match_engine=None): + super().__init__(match_engine) def get_all_args(self, ns=""): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", ns=ns) From 0082da29aee382dfb4fbb9988d99913a6a88ab19 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 19 Aug 2021 11:09:31 +0000 Subject: [PATCH 21/22] Removed the entra mock and added a match_engine fixture Signed-off-by: Vivek Reddy Karri --- .../module_tests/mock_sonicv2connector.py | 72 ------------------ tests/dump_tests/module_tests/port_test.py | 75 +++++++++++++------ 2 files changed, 52 insertions(+), 95 deletions(-) delete mode 100644 tests/dump_tests/module_tests/mock_sonicv2connector.py diff --git a/tests/dump_tests/module_tests/mock_sonicv2connector.py b/tests/dump_tests/module_tests/mock_sonicv2connector.py deleted file mode 100644 index f6ba18981e..0000000000 --- a/tests/dump_tests/module_tests/mock_sonicv2connector.py +++ /dev/null @@ -1,72 +0,0 @@ -import json -import re -import os -from utilities_common.constants import DEFAULT_NAMESPACE - - -class MockSonicV2Connector(): - def __init__(self, dedicated_dbs, **kwargs): - if "namespace" in kwargs and kwargs["namespace"] != DEFAULT_NAMESPACE: - raise "This Mock doesn't support multi-asic configuration" - self.db = None - self.db_name_connect_to = None - self.dedicated_dbs = dedicated_dbs - db_config_path = os.path.join(os.path.dirname(__file__), "../../mock_tables/database_config.json") - with open(db_config_path) as f: - self.db_cfg = json.load(f) - - def connect(self, db_name, retry=False): - if db_name not in self.dedicated_dbs: - raise Exception("{} not found. Available db's: {}".format(db_name, self.dedicated_dbs.keys())) - try: - with open(self.dedicated_dbs[db_name]) as f: - self.db = json.load(f) - self.db_name_connect_to = db_name - except BaseException as e: - raise "Connection Failure" + str(e) - - def get_db_separator(self, db_name): - return self.db_cfg["DATABASES"][db_name]["separator"] - - def keys(self, db_name, pattern): - if not self.db: - raise "MockDB Not Connected" - if self.db_name_connect_to != db_name: - raise "Failed to find {} in the MockDB".format(db_name) - - pattern = re.escape(pattern) - pattern = pattern.replace("\\*", ".*") - filtered_keys = [] - all_keys = self.db.keys() - for key in all_keys: - if re.match(pattern, key): - filtered_keys.append(key) - return filtered_keys - - def get_all(self, db_name, key): - if not self.db: - raise "MockDB Not Connected" - if self.db_name_connect_to != db_name: - raise "Failed to find {} in the MockDB".format(db_name) - if key not in self.db: - return {} - return self.db[key] - - def get(self, db_name, key, field): - if not self.db: - raise "MockDB Not Connected" - if self.db_name_connect_to != db_name: - raise "Failed to find {} in the MockDB".format(db_name) - if key not in self.db or field not in self.db[key]: - return "" - return self.db[key][field] - - def hexists(self, db_name, key, field): - if not self.db: - raise "MockDB Not Connected" - if self.db_name_connect_to != db_name: - raise "Failed to find {} in the MockDB".format(db_name) - if key not in self.db or field not in self.db[key]: - return False - else: - return True diff --git a/tests/dump_tests/module_tests/port_test.py b/tests/dump_tests/module_tests/port_test.py index 9886df014a..ebed245e31 100644 --- a/tests/dump_tests/module_tests/port_test.py +++ b/tests/dump_tests/module_tests/port_test.py @@ -8,17 +8,17 @@ from mock import patch from dump.helper import create_template_dict, sort_lists from dump.plugins.port import Port +from dump.match_infra import MatchEngine, ConnectionPool +from swsscommon.swsscommon import SonicV2Connector -from .mock_sonicv2connector import MockSonicV2Connector - +# Location for dedicated db's used for UT module_tests_path = os.path.dirname(__file__) dump_tests_path = os.path.join(module_tests_path, "../") tests_path = os.path.join(dump_tests_path, "../") dump_test_input = os.path.join(tests_path, "dump_input") - -# Location for dedicated db's used for UT port_files_path = os.path.join(dump_test_input, "port") +# Define the mock files to read from dedicated_dbs = {} dedicated_dbs['CONFIG_DB'] = os.path.join(port_files_path, "config_db.json") dedicated_dbs['APPL_DB'] = os.path.join(port_files_path, "appl_db.json") @@ -26,27 +26,56 @@ dedicated_dbs['STATE_DB'] = os.path.join(port_files_path, "state_db.json") -def mock_connector(namespace, use_unix_socket_path=True): - return MockSonicV2Connector(dedicated_dbs, namespace=namespace, use_unix_socket_path=use_unix_socket_path) +def populate_mock(db, db_names): + for db_name in db_names: + db.connect(db_name) + # Delete any default data + db.delete_all_by_pattern(db_name, "*") + with open(dedicated_dbs[db_name]) as f: + mock_json = json.load(f) + for key in mock_json: + for field, value in mock_json[key].items(): + db.set(db_name, key, field, value) + +@pytest.fixture(scope="class", autouse=True) +def match_engine(): -@pytest.fixture(scope="module", autouse=True) -def verbosity_setup(): print("SETUP") os.environ["VERBOSE"] = "1" - yield + + # Monkey Patch the SonicV2Connector Object + from ...mock_tables import dbconnector + db = SonicV2Connector() + + # popualate the db with mock data + db_names = list(dedicated_dbs.keys()) + try: + populate_mock(db, db_names) + except Exception as e: + assert False, "Mock initialization failed: " + str(e) + + # Initialize connection pool + conn_pool = ConnectionPool() + DEF_NS = '' # Default Namespace + conn_pool.cache = {DEF_NS: {'conn': db, + 'connected_to': set(db_names)}} + + # Initialize match_engine + match_engine = MatchEngine(conn_pool) + yield match_engine print("TEARDOWN") os.environ["VERBOSE"] = "0" -@patch("dump.match_infra.SonicV2Connector", mock_connector) -class TestPortModule(unittest.TestCase): - def test_working_state(self): +@pytest.mark.usefixtures("match_engine") +class TestPortModule: + def test_working_state(self, match_engine): """ Scenario: When the config is properly applied and propagated """ params = {Port.ARG_NAME: "Ethernet176", "namespace": ""} - m_port = Port() + m_port = Port(match_engine) returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet176") @@ -57,12 +86,12 @@ def test_working_state(self): ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff - def test_missing_asic_port(self): + def test_missing_asic_port(self, match_engine): """ Scenario: When the config was applied and just the SAI_OBJECT_TYPE_PORT is missing """ params = {Port.ARG_NAME: "Ethernet160", "namespace": ""} - m_port = Port() + m_port = Port(match_engine) returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet160") @@ -73,12 +102,12 @@ def test_missing_asic_port(self): ddiff = DeepDiff(sort_lists(returned), sort_lists(expect), ignore_order=True) assert not ddiff, ddiff - def test_missing_asic_hostif(self): + def test_missing_asic_hostif(self, match_engine): """ Scenario: When the config was applied and it did not propagate to ASIC DB """ params = {Port.ARG_NAME: "Ethernet164", "namespace": ""} - m_port = Port() + m_port = Port(match_engine) returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet164") @@ -89,12 +118,12 @@ def test_missing_asic_hostif(self): ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - def test_missing_state_and_appl(self): + def test_missing_state_and_appl(self, match_engine): """ Scenario: When the config was applied and it did not propagate to other db's """ params = {Port.ARG_NAME: "Ethernet156", "namespace": ""} - m_port = Port() + m_port = Port(match_engine) returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["keys"].append("PORT|Ethernet156") @@ -105,12 +134,12 @@ def test_missing_state_and_appl(self): ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - def test_no_port(self): + def test_no_port(self, match_engine): """ Scenario: When no entry for the port is present in any of the db's """ params = {Port.ARG_NAME: "Ethernet152", "namespace": ""} - m_port = Port() + m_port = Port(match_engine) returned = m_port.execute(params) expect = create_template_dict(dbs=["CONFIG_DB", "APPL_DB", "ASIC_DB", "STATE_DB"]) expect["CONFIG_DB"]["tables_not_found"].append("PORT") @@ -121,12 +150,12 @@ def test_no_port(self): ddiff = DeepDiff(returned, expect, ignore_order=True) assert not ddiff, ddiff - def test_all_args(self): + def test_all_args(self, match_engine): """ Scenario: Verify Whether the get_all_args method is working as expected """ params = {} - m_port = Port() + m_port = Port(match_engine) returned = m_port.get_all_args("") expect = ["Ethernet156", "Ethernet160", "Ethernet164", "Ethernet176"] ddiff = DeepDiff(expect, returned, ignore_order=True) From 2ea3725d3e24e7f438d07ab9798360d5b7a303b4 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 19 Aug 2021 11:29:28 +0000 Subject: [PATCH 22/22] LGTM issues handled Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 4 ++-- dump/plugins/port.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index 31f22058a8..0b4ca0490a 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -243,7 +243,7 @@ def get(self, db_name, ns, update=False): self.cache[ns]["connected_to"].add(db_name) return self.cache[ns]["conn"] - def clear(namespace=None): + def clear(self, namespace=None): if not namespace: self.cache.clear() elif namespace in self.cache: @@ -263,7 +263,7 @@ def __init__(self, pool=None): else: self.conn_pool = pool - def clear_cache(ns): + def clear_cache(self, ns): self.conn_pool(ns) def __get_source_adapter(self, req): diff --git a/dump/plugins/port.py b/dump/plugins/port.py index 7b078505ee..f8422c9c69 100644 --- a/dump/plugins/port.py +++ b/dump/plugins/port.py @@ -1,4 +1,4 @@ -from dump.match_infra import MatchEngine, MatchRequest +from dump.match_infra import MatchRequest from dump.helper import create_template_dict from .executor import Executor