From 5bec449798ed0b301f76248e12d76876c7413b31 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Fri, 14 May 2021 05:02:53 +0000 Subject: [PATCH 01/17] Match Infra Added Signed-off-by: Vivek Reddy Karri --- dump/__init__.py | 0 dump/helper.py | 64 ++++++ dump/redis_match.py | 278 ++++++++++++++++++++++++ tests/dump_tests/files/copp_cfg.json | 103 +++++++++ tests/dump_tests/match_engine_test.py | 297 ++++++++++++++++++++++++++ 5 files changed, 742 insertions(+) create mode 100644 dump/__init__.py create mode 100644 dump/helper.py create mode 100644 dump/redis_match.py create mode 100644 tests/dump_tests/files/copp_cfg.json create mode 100644 tests/dump_tests/match_engine_test.py diff --git a/dump/__init__.py b/dump/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dump/helper.py b/dump/helper.py new file mode 100644 index 0000000000..672b9826fa --- /dev/null +++ b/dump/helper.py @@ -0,0 +1,64 @@ +import os,json,re,click +import redis +from swsscommon.swsscommon import SonicDBConfig, ASIC_DB + + +# Generate a Template which will be returned by Executor Classes +def display_template(dbs): + template = {} + for db in dbs: + template[db] = {} + template[db]['keys'] = [] + template[db]['tables_not_found'] = [] + return template + +def verbose_print(str): + if "VERBOSE" in os.environ and os.environ["VERBOSE"] == "1": + print(str) + +# Get a vid:rid for the input vid +def vid_to_rid(vid, r): + rid = r.hget("VIDTORID", vid) + if not rid: + rid = "Real ID Not Found" + return rid + +def get_v_r_map(r, single_dict): + v_r_map = {} + asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}" + + if "ASIC_DB" in single_dict and 'keys' in single_dict["ASIC_DB"]: + for redis_key in single_dict["ASIC_DB"]['keys']: + if re.match(asic_obj_ptrn, redis_key): + matches = re.findall(r"oid:0x\w{1,14}", redis_key) + if matches: + vid = matches[0] + v_r_map[vid] = vid_to_rid(vid, r) + return v_r_map + +def extract_rid(info): + r = redis.Redis(unix_socket_path=SonicDBConfig.getDbSock("ASIC_DB"), db=ASIC_DB, encoding="utf-8", decode_responses=True) + vidtorid = {} + for arg in info.keys(): + vidtorid[arg] = get_v_r_map(r, info[arg]) + return vidtorid + +# Filter dbs which are not required +def filter_out_dbs(db_list, collected_info): + args_ = list(collected_info.keys()) + for arg in args_: + dbs = list(collected_info[arg].keys()) + for db in dbs: + if db not in db_list: + del collected_info[arg][db] + return collected_info + + + + + + + + + + diff --git a/dump/redis_match.py b/dump/redis_match.py new file mode 100644 index 0000000000..5dc936f7c6 --- /dev/null +++ b/dump/redis_match.py @@ -0,0 +1,278 @@ +import re, json, os, sys +from dump.helper import verbose_print +from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig + +error_dict = { + "INV_REQ": "Argument should be of type MatchRequest", + "INV_DB": "DB provided is not valid", + "INV_JSON": "Not a properly formatted JSON file", + "INV_PTTRN": "No Entries found for Table|key_pattern provided", + "NO_FILE": "JSON File not found", + "NO_SRC": "Either one of db or file in the request should be non-empty", + "NO_KEY": "'key_pattern' cannot be empty", + "NO_TABLE": "No 'table' name provided", + "NO_VALUE" : "Field is provided, but no value is provided to compare with", + "SRC_VAGUE": "Only one of db or file should be provided", + "CONN_ERR" : "Connection Error", + "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty", + "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type" +} + +class MatchRequest: + def __init__(self): + self.table = None + self.key_pattern = "*" + self.field = None + self.value = None + self.return_fields = [] + self.db = "" + self.file = "" + self.just_keys = True + + def __str__(self): + str = "MatchRequest: \n" + if self.db: + str += "db:{} , ".format(self.db) + if self.file: + str += "file:{} , ".format(self.file) + if self.table: + str += "table:{} , ".format(self.table) + if self.key_pattern: + str += "key_regx:{} , ".format(self.key_pattern) + if self.field: + str += "field:{} , ".format(self.field) + if self.value: + str += "value:{} , ".format(self.value) + if self.just_keys: + str += "just_keys:True " + else: + str += "just_keys:False " + if len(self.return_fields) > 0: + str += "Return Fields: " + ",".join(self.return_fields) + return str + +class SourceAdapter: + def __init__(self): + pass + + def connect(self, db): + return False + + def getKeys(self, db, table, key_pattern): + return [] + + def get(self, db, key): + return {} + + def hget(self, db, key, field): + return "" + + def sep(self, db): + return "" + +class RedisSource(SourceAdapter): + def __init__(self): + self.db_driver = None + + def connect(self, db): + self.db_driver = SonicV2Connector(host="127.0.0.1") + try: + self.db_driver.connect(db) + except Exception as e: + verbose_print("RedisSource: Connection Failed\n" + str(e)) + return False + return True + + def sep(self, db): + return self.db_driver.get_db_separator(db) + + def getKeys(self, db, table, key_pattern): + try: + keys = self.db_driver.keys(db, table + self.sep(db) + key_pattern) + except Exception as e: + verbose_print("RedisSource: {}|{}|{} Keys fetch Request Failed for DB {}\n".format(table, self.sep(db), key_pattern, db) + str(e)) + return [] + return keys + + def get(self, db, key): + try: + fv_pairs = self.db_driver.get_all(db, key) + except Exception as e: + verbose_print("RedisSource: hgetall {} request failed for DB {}\n".format(key, db) + str(e)) + return {} + return fv_pairs + + def hget(self, db, key, field): + try: + value = self.db_driver.get(db, key, field) + except Exception as e: + verbose_print("RedisSource: hget {} {} request failed for DB {}\n".format(key, field) + str(e)) + return "" + return value + +class JsonSource(SourceAdapter): + + def __init__(self): + self.db_driver = None + + def connect(self, db): + try: + with open(db) as f: + self.db_driver = json.load(f) + except Exception as e: + verbose_print("JsonSource: Loading the JSON file failed" + str(e)) + return False + return True + + def sep(self, db): + return SonicDBConfig.getSeparator("CONFIG_DB") + + def getKeys(self, db, table, key_pattern): + if table not in self.db_driver: + return [] + + all_keys = self.db_driver[table].keys() + key_ptrn = key_pattern + key_ptrn = re.escape(key_ptrn) + key_ptrn = key_ptrn.replace("\\*", ".*") + filtered_keys = [] + for key in all_keys: + if re.match(key_ptrn, key): + filtered_keys.append(table+self.sep(db)+key) + return filtered_keys + + def get(self, db, key): + sp = self.sep(db) + tokens = key.split(sp) + key_ptrn = tokens[-1] + tokens.pop() + table = sp.join(tokens) + if table in self.db_driver and key_ptrn in self.db_driver[table]: + return self.db_driver[table][key_ptrn] + return {} + + def hget(self, db, key, field): + sp = self.sep(db) + tokens = key.split(sp) + key_ptrn = tokens[-1] + tokens.pop() + table = sp.join(tokens) + print(table, key_ptrn) + if table in self.db_driver and key_ptrn in self.db_driver[table] and field in self.db_driver[table][key_ptrn]: + return self.db_driver[table][key_ptrn][field] + return "" + +class MatchEngine: + + # Given a request obj, find its match in the redis + def fetch(self, req): + verbose_print(str(req)) + template = self.__ret_template() + template['error'] = self.__validate_request(req) + if template['error']: + return self.__return_error(template) + + src = None + if req.db: + src = RedisSource() + else: + req.db = req.file + src = JsonSource() + + if not src.connect(req.db): + template['error'] = error_dict["CONN_ERR"] + return self.__return_error(template) + verbose_print("MatchRequest Checks Passed") + all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern) + if not all_matched_keys or len(all_matched_keys) == 0: + template['error'] = error_dict["INV_PTTRN"] + return self.__return_error(template) + verbose_print("Keys Matched before Filtering:" + str(all_matched_keys)) + + filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) + verbose_print("Filtered Keys:" + str(filtered_keys)) + return self.__fill(src, req, filtered_keys) + + def __ret_template(self): + return {"error" : "", "keys" : [], "return_values" : {}} + + + def __return_error(self, template): + verbose_print("MatchEngine: \n" + template['error']) + return template + + def __validate_request(self, req): + + if not isinstance(req, MatchRequest): + return error_dict["INV_REQ"] + + if not(req.db) and not(req.file): + return error_dict["NO_SRC"] + + if req.db and req.file: + return error_dict["SRC_VAGUE"] + + if not req.db and os.path.exists(req.file): + try: + with open(req.file) as f: + json.load(f) + except ValueError as e: + return error_dict["INV_JSON"] + elif not req.db: + return error_dict["NO_FILE"] + + if not(req.file) and req.db not in SonicDBConfig.getDbList(): + return error_dict["INV_DB"] + + if not req.table: + return error_dict["NO_TABLE"] + + if not req.key_pattern: + return error_dict["NO_KEY"] + + if not isinstance(req.return_fields, list): + return error_dict["BAD_FORMAT_RE_FIELDS"] + + if not req.just_keys and len(req.return_fields) > 0: + return error_dict["JUST_KEYS_COMPAT"] + + if req.field and not req.value: + return error_dict["NO_VALUE"] + + return "" + + def __filter_out_keys(self, src, req, all_matched_keys): + if not (req.field): + return all_matched_keys + + filtered_keys = [] + for key in all_matched_keys: + f_values = src.hget(req.db, key, req.field) + if "," in f_values: # Fields Containing Multile Values + f_value = f_values.split(",") + else: + f_value = [f_values] + if req.value in f_value: + filtered_keys.append(key) + return filtered_keys + + def __fill(self, src, req, filtered_keys): + + template = self.__ret_template() + for key in filtered_keys: + temp = {} + if not req.just_keys: + temp[key] = src.get(req.db, key) + template["keys"].append(temp) + elif len(req.return_fields) > 0: + template["keys"].append(key) + template["return_values"][key] = {} + for field in req.return_fields: + template["return_values"][key][field] = src.hget(req.db, key, field) + else: + template["keys"].append(key) + return template + + + + diff --git a/tests/dump_tests/files/copp_cfg.json b/tests/dump_tests/files/copp_cfg.json new file mode 100644 index 0000000000..cae5f4e8e0 --- /dev/null +++ b/tests/dump_tests/files/copp_cfg.json @@ -0,0 +1,103 @@ +{ + "COPP_GROUP": { + "default": { + "queue": "0", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue4_group1": { + "trap_action":"trap", + "trap_priority":"4", + "queue": "4" + }, + "queue4_group2": { + "trap_action":"copy", + "trap_priority":"4", + "queue": "4", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue4_group3": { + "trap_action":"trap", + "trap_priority":"4", + "queue": "4" + }, + "queue1_group1": { + "trap_action":"trap", + "trap_priority":"1", + "queue": "1", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"6000", + "cbs":"6000", + "red_action":"drop" + }, + "queue1_group2": { + "trap_action":"trap", + "trap_priority":"1", + "queue": "1", + "meter_type":"packets", + "mode":"sr_tcm", + "cir":"600", + "cbs":"600", + "red_action":"drop" + }, + "queue2_group1": { + "cbs": "1000", + "cir": "1000", + "genetlink_mcgrp_name": "packets", + "genetlink_name": "psample", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "2", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + + } + }, + "COPP_TRAP": { + "bgp": { + "trap_ids": "bgp,bgpv6", + "trap_group": "queue4_group1" + }, + "lacp": { + "trap_ids": "lacp", + "trap_group": "queue4_group1" + }, + "arp": { + "trap_ids": "arp_req,arp_resp,neigh_discovery", + "trap_group": "queue4_group2" + }, + "lldp": { + "trap_ids": "lldp", + "trap_group": "queue4_group3" + }, + "dhcp": { + "trap_ids": "dhcp,dhcpv6", + "trap_group": "queue4_group3" + }, + "udld": { + "trap_ids": "udld", + "trap_group": "queue4_group3" + }, + "ip2me": { + "trap_ids": "ip2me", + "trap_group": "queue1_group1" + }, + "nat": { + "trap_ids": "src_nat_miss,dest_nat_miss", + "trap_group": "queue1_group2" + }, + "sflow": { + "trap_group": "queue2_group1", + "trap_ids": "sample_packet" + } + } +} diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py new file mode 100644 index 0000000000..c2f0b7519a --- /dev/null +++ b/tests/dump_tests/match_engine_test.py @@ -0,0 +1,297 @@ +import json, os, sys +import jsonpatch +import unittest +import pytest +from dump.redis_match import MatchEngine, error_dict, MatchRequest +from deepdiff import DeepDiff + +test_path = os.path.join(os.path.abspath(__file__),"../") + +sys.path.append(test_path) + +@pytest.fixture(scope="module", autouse=True) +def mock_setup(): + print("SETUP") + os.environ["VERBOSE"] = "1" + yield + print("TEARDOWN") + os.environ["VERBOSE"] = "0" + + +class TestInvalidRequest(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(TestInvalidRequest, self).__init__(*args, **kwargs) + self.match_engine = MatchEngine() + + def test_bad_request(self): + req = [] + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["INV_REQ"] + + def test_no_source(self): + req = MatchRequest() + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["NO_SRC"] + + def test_vague_source(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.file = "/etc/sonic/copp_cfg.json" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["SRC_VAGUE"] + + def test_no_file(self): + req = MatchRequest() + req.file = os.path.join(test_path, "random_db.json") + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["NO_FILE"] + + def test_invalid_db(self): + req = MatchRequest() + req.db = "CONFIGURATION_DB" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["INV_DB"] + + def test_bad_key_pattern(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "PORT" + req.key_pattern = "" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["NO_KEY"] + + def test_no_value(self): + req = MatchRequest() + req.db = "APPL_DB" + req.table = "COPP_TABLE" + req.key_pattern = "*" + req.field = "trap_ids" + req.value = "" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["NO_VALUE"] + + def test_no_table(self): + req = MatchRequest() + req.db = "APPL_DB" + req.table = "" + req.key_pattern = "*" + req.field = "trap_ids" + req.value = "bgpv6" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["NO_TABLE"] + + def test_just_keys_return_fields_compat(self): + req = MatchRequest() + req.db = "APPL_DB" + req.table = "COPP_TABLE" + req.key_pattern = "*" + req.field = "trap_ids" + req.value = "" + req.just_keys = False + req.return_fields = ["trap_group"] + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["JUST_KEYS_COMPAT"] + + def test_invalid_combination(self): + + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "COPP_TRAP" + req.key_pattern = "*" + req.field = "trap_ids" + req.value = "sample_packet" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["INV_PTTRN"] + + def test_return_fields_bad_format(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "REBOOT_CAUSE" + req.return_fields = "cause" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["BAD_FORMAT_RE_FIELDS"] + +class TestMatchEngine(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(TestMatchEngine, self).__init__(*args, **kwargs) + self.match_engine = MatchEngine() + + def test_key_pattern_wildcard(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "SFLOW_COLLECTOR" + req.key_pattern = "*" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 2 + assert "SFLOW_COLLECTOR|ser5" in ret['keys'] + assert "SFLOW_COLLECTOR|prod" in ret['keys'] + + def test_key_pattern_complex(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "ACL_RULE" + req.key_pattern = "EVERFLOW*" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 2 + assert "ACL_RULE|EVERFLOW|RULE_6" in ret['keys'] + assert "ACL_RULE|EVERFLOW|RULE_08" in ret['keys'] + + def test_field_value_match(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "ACL_TABLE" + req.field = "policy_desc" + req.value = "SSH_ONLY" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "ACL_TABLE|SSH_ONLY" in ret['keys'] + + def test_field_value_match_list_type(self): + req = MatchRequest() + req.db = "APPL_DB" + req.table = "PORT_TABLE" + req.field = "lanes" + req.value = "202" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "PORT_TABLE:Ethernet200" in ret['keys'] + + def test_for_no_match(self): + req = MatchRequest() + req.db = "ASIC_DB" + req.table = "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH" + req.field = "SAI_SWITCH_ATTR_SRC_MAC_ADDRESS" + req.value = "DE:AD:EE:EE:EE" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 0 + + def test_for_no_key_match(self): + req = MatchRequest() + req.db = "ASIC_DB" + req.table = "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH" + req.key_pattern = "oid:0x22*" + ret = self.match_engine.fetch(req) + assert ret["error"] == error_dict["INV_PTTRN"] + + def test_field_value_no_match(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "FAN_INFO" + req.key_pattern = "*" + req.field = "led_status" + req.value = "yellow" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 0 + + def test_return_keys(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "REBOOT_CAUSE" + req.return_fields = ["cause"] + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 2 + assert "warm-reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_04_53_58"]["cause"] + assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] + + def test_return_fields_with_key_filtering(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "REBOOT_CAUSE" + req.key_pattern = "2020_10_09_02*" + req.return_fields = ["cause"] + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] + + def test_return_fields_with_field_value_filtering(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "CHASSIS_MODULE_TABLE" + req.field = "oper_status" + req.value = "Offline" + req.return_fields = ["slot"] + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "18" == ret["return_values"]["CHASSIS_MODULE_TABLE|FABRIC-CARD1"]["slot"] + + def test_return_fields_with_all_filtering(self): + req = MatchRequest() + req.db = "STATE_DB" + req.table = "VXLAN_TUNNEL_TABLE" + req.key_pattern = "EVPN_25.25.25.2*" + req.field = "operstatus" + req.value = "down" + req.return_fields = ["src_ip"] + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 3 + assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.25"]["src_ip"] + assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.26"]["src_ip"] + assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.27"]["src_ip"] + + def test_just_keys_false(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "SFLOW" + req.key_pattern = "global" + req.just_keys = False + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + recv_dict = ret["keys"][0] + assert isinstance(recv_dict, dict) + exp_dict = {"SFLOW|global": {"admin_state": "up", "polling_interval": "0"}} + ddiff = DeepDiff(exp_dict, recv_dict) + assert not ddiff, ddiff + + def test_file_source(self): + req = MatchRequest() + req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req.table = "COPP_TRAP" + req.field = "trap_ids" + req.value = "arp_req" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "COPP_TRAP|arp" in ret["keys"] + + def test_file_source_with_key_ptrn(self): + req = MatchRequest() + req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req.table = "COPP_GROUP" + req.key_pattern = "queue4*" + req.field = "red_action" + req.value = "drop" + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "COPP_GROUP|queue4_group2" in ret["keys"] + + def test_file_source_with_not_only_return_keys(self): + req = MatchRequest() + req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req.table = "COPP_GROUP" + req.key_pattern = "queue4*" + req.field = "red_action" + req.value = "drop" + req.just_keys = False + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + recv_dict = ret["keys"][0] + exp_dict = {"COPP_GROUP|queue4_group2": {"trap_action": "copy", "trap_priority": "4", "queue": "4", "meter_type": "packets", "mode": "sr_tcm", "cir": "600", "cbs": "600", "red_action": "drop"}} + ddiff = DeepDiff(exp_dict, recv_dict) + assert not ddiff, ddiff + + \ No newline at end of file From 4da400bf2dce722cbf83fa63d4fe12b5dc115c17 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Fri, 14 May 2021 05:16:10 +0000 Subject: [PATCH 02/17] Missing file added Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/dump_tests/__init__.py diff --git a/tests/dump_tests/__init__.py b/tests/dump_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 6c8667a4cfe44b2dc34d7ca8c17316d0a82ef06b Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 20 May 2021 20:09:46 +0000 Subject: [PATCH 03/17] Multi-Asic Changes Made Signed-off-by: Vivek Reddy Karri --- dump/helper.py | 40 +--------------------------------------- dump/redis_match.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 46 deletions(-) diff --git a/dump/helper.py b/dump/helper.py index 672b9826fa..2ed46976b2 100644 --- a/dump/helper.py +++ b/dump/helper.py @@ -1,7 +1,4 @@ -import os,json,re,click -import redis -from swsscommon.swsscommon import SonicDBConfig, ASIC_DB - +import os # Generate a Template which will be returned by Executor Classes def display_template(dbs): @@ -16,42 +13,7 @@ def verbose_print(str): if "VERBOSE" in os.environ and os.environ["VERBOSE"] == "1": print(str) -# Get a vid:rid for the input vid -def vid_to_rid(vid, r): - rid = r.hget("VIDTORID", vid) - if not rid: - rid = "Real ID Not Found" - return rid - -def get_v_r_map(r, single_dict): - v_r_map = {} - asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}" - - if "ASIC_DB" in single_dict and 'keys' in single_dict["ASIC_DB"]: - for redis_key in single_dict["ASIC_DB"]['keys']: - if re.match(asic_obj_ptrn, redis_key): - matches = re.findall(r"oid:0x\w{1,14}", redis_key) - if matches: - vid = matches[0] - v_r_map[vid] = vid_to_rid(vid, r) - return v_r_map - -def extract_rid(info): - r = redis.Redis(unix_socket_path=SonicDBConfig.getDbSock("ASIC_DB"), db=ASIC_DB, encoding="utf-8", decode_responses=True) - vidtorid = {} - for arg in info.keys(): - vidtorid[arg] = get_v_r_map(r, info[arg]) - return vidtorid -# Filter dbs which are not required -def filter_out_dbs(db_list, collected_info): - args_ = list(collected_info.keys()) - for arg in args_: - dbs = list(collected_info[arg].keys()) - for db in dbs: - if db not in db_list: - del collected_info[arg][db] - return collected_info diff --git a/dump/redis_match.py b/dump/redis_match.py index 5dc936f7c6..cbca5e92e4 100644 --- a/dump/redis_match.py +++ b/dump/redis_match.py @@ -28,6 +28,7 @@ def __init__(self): self.db = "" self.file = "" self.just_keys = True + self.ns = '' def __str__(self): str = "MatchRequest: \n" @@ -49,13 +50,15 @@ def __str__(self): str += "just_keys:False " if len(self.return_fields) > 0: str += "Return Fields: " + ",".join(self.return_fields) + if self.ns: + str += "Namespace: " + self.ns return str class SourceAdapter: def __init__(self): pass - def connect(self, db): + def connect(self, db, ns): return False def getKeys(self, db, table, key_pattern): @@ -74,9 +77,9 @@ class RedisSource(SourceAdapter): def __init__(self): self.db_driver = None - def connect(self, db): - self.db_driver = SonicV2Connector(host="127.0.0.1") + def connect(self, db, ns): try: + self.db_driver = SonicV2Connector(namespace=ns, host="127.0.0.1") self.db_driver.connect(db) except Exception as e: verbose_print("RedisSource: Connection Failed\n" + str(e)) @@ -115,7 +118,7 @@ class JsonSource(SourceAdapter): def __init__(self): self.db_driver = None - def connect(self, db): + def connect(self, db, ns): try: with open(db) as f: self.db_driver = json.load(f) @@ -173,13 +176,15 @@ def fetch(self, req): return self.__return_error(template) src = None + d_src = "" if req.db: + d_src = req.db src = RedisSource() else: - req.db = req.file + d_src = req.file src = JsonSource() - if not src.connect(req.db): + if not src.connect(d_src, req.ns): template['error'] = error_dict["CONN_ERR"] return self.__return_error(template) verbose_print("MatchRequest Checks Passed") @@ -187,7 +192,6 @@ def fetch(self, req): if not all_matched_keys or len(all_matched_keys) == 0: template['error'] = error_dict["INV_PTTRN"] return self.__return_error(template) - verbose_print("Keys Matched before Filtering:" + str(all_matched_keys)) filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) verbose_print("Filtered Keys:" + str(filtered_keys)) From 42118c030350f95b8763954a107384c2dea50007 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 20 May 2021 20:25:00 +0000 Subject: [PATCH 04/17] Minor Changes Signed-off-by: Vivek Reddy Karri --- dump/main.py | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 dump/main.py diff --git a/dump/main.py b/dump/main.py new file mode 100644 index 0000000000..6dfca6f06c --- /dev/null +++ b/dump/main.py @@ -0,0 +1,224 @@ +import os,sys,json,re +import click +from tabulate import tabulate + +sys.path.append(os.path.dirname(__file__)) +import plugins +from dump.redis_match import RedisSource, JsonSource +from swsscommon.swsscommon import SonicV2Connector +from utilities_common.multi_asic import multi_asic_ns_choices, multi_asic +from utilities_common.constants import DEFAULT_NAMESPACE + +# Autocompletion Helper +def get_available_modules(ctx, args, incomplete): + return [k for k in plugins.dump_modules.keys() if incomplete in k] + +# Display Modules Callback +def show_modules(ctx, param, value): + if not value or ctx.resilient_parsing: + return + header = ["Module", "Identifier"] + display = [] + for mod in plugins.dump_modules: + display.append((mod, plugins.dump_modules[mod].ARG_NAME)) + click.echo(tabulate(display, header)) + ctx.exit() + +@click.group() +def dump(): + pass + +@dump.command() +@click.pass_context +@click.argument('module', required=True, type=str, autocompletion=get_available_modules) +@click.argument('identifier', required=True, type=str) +@click.option('--show', '-s', is_flag=True, default=False, help='Display Modules Available', is_eager=True, expose_value=False, callback=show_modules) +@click.option('--db', '-d', multiple=True, help='Only dump from these Databases') +@click.option('--table', '-t', is_flag=True, default=False, help='Print in tabular format', show_default=True) +@click.option('--key-map', '-k', is_flag=True, default=False, help="Only fetch the keys matched, don't extract field-value dumps", show_default=True) +@click.option('--verbose', '-v', is_flag=True, default=False, help="Prints any intermediate output to stdout useful for dev & troubleshooting", show_default=True) +@click.option('--namespace', '-n', default=DEFAULT_NAMESPACE, type=str, show_default=True, help='Dump the redis-state for this namespace.') +def state(ctx, module, identifier, db, table, key_map, verbose, namespace): + """ + Dump the redis-state of the identifier for the module specified + """ + if not multi_asic.is_multi_asic() and namespace != DEFAULT_NAMESPACE: + click.echo("Namespace option is not valid for a single-ASIC device") + ctx.exit() + + if multi_asic.is_multi_asic() and (namespace != DEFAULT_NAMESPACE and namespace not in multi_asic_ns_choices()): + click.echo("Namespace option is not valid. Choose one of {}".format(multi_asic_ns_choices())) + ctx.exit() + + if module not in plugins.dump_modules: + click.echo("No Matching Plugin has been Implemented") + ctx.exit() + + if verbose: + os.environ["VERBOSE"] = "1" + else: + os.environ["VERBOSE"] = "0" + + ctx.module = module + obj = plugins.dump_modules[module]() + + if identifier == "all": + ids = obj.get_all_args(namespace) + else: + ids = identifier.split(",") + + params = {} + collected_info = {} + params['namespace'] = namespace + for arg in ids: + params[plugins.dump_modules[module].ARG_NAME] = arg + collected_info[arg] = obj.execute(params) + + if len(db) > 0: + collected_info = filter_out_dbs(db, collected_info) + + vidtorid = extract_rid(collected_info, namespace) + + if not key_map: + collected_info = populate_fv(collected_info, module, namespace) + + for id in vidtorid.keys(): + if vidtorid[id] and vidtorid: + collected_info[id]["ASIC_DB"]["vidtorid"] = vidtorid[id] + + print_dump(collected_info, table, module, identifier, key_map) + + return + +def extract_rid(info, ns): + r = SonicV2Connector(namespace=ns, host="127.0.0.1") + r.connect("ASIC_DB") + vidtorid = {} + for arg in info.keys(): + vidtorid[arg] = get_v_r_map(r, info[arg]) + return vidtorid + +def get_v_r_map(r, single_dict): + v_r_map = {} + asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}" + + if "ASIC_DB" in single_dict and 'keys' in single_dict["ASIC_DB"]: + for redis_key in single_dict["ASIC_DB"]['keys']: + if re.match(asic_obj_ptrn, redis_key): + matches = re.findall(r"oid:0x\w{1,14}", redis_key) + if matches: + vid = matches[0] + v_r_map[vid] = vid_to_rid(vid, r) + return v_r_map + +# Get a vid:rid for the input vid +def vid_to_rid(vid, r): + rid = r.get("ASIC_DB", "VIDTORID", vid) + if not rid: + rid = "Real ID Not Found" + return rid + +# Filter dbs which are not required +def filter_out_dbs(db_list, collected_info): + args_ = list(collected_info.keys()) + for arg in args_: + dbs = list(collected_info[arg].keys()) + for db in dbs: + if db not in db_list: + del collected_info[arg][db] + return collected_info + +def populate_fv(info, module, namespace): + + all_dbs = set() + for id in info.keys(): + for db_name in info[id].keys(): + all_dbs.add(db_name) + + db_dict = {} + for db_name in all_dbs: + if db_name is "CONFIG_FILE": + db_dict[db_name] = JsonSource() + db_dict[db_name].connect(plugins.dump_modules[module].CONFIG_FILE, namespace) + else: + db_dict[db_name] = RedisSource() + db_dict[db_name].connect(db_name, namespace) + + final_info = {} + for id in info.keys(): + final_info[id] = {} + for db_name in info[id].keys(): + final_info[id][db_name] = {} + final_info[id][db_name]["keys"] = [] + final_info[id][db_name]["tables_not_found"] = info[id][db_name]["tables_not_found"] + for key in info[id][db_name]["keys"]: + final_info[id][db_name]["keys"].append({key : db_dict[db_name].get(db_name, key)}) + + return final_info + +def get_dict_str(key_obj): + table = [] + for pair in key_obj.items(): + table.append(list(pair)) + return tabulate(table, headers=["field", "value"], tablefmt="psql") + +def get_keys(dump): + keys = [] + for key_ in dump: + if isinstance(key_, dict) and key_: + keys.append(list(key_.keys())[0]) + else: + keys.append(key_) + return keys + +def get_rid(redis_key, vidtorid): + matches = re.findall(r"oid:0x\w{1,14}", redis_key) + if matches: + vid = matches[0] + if vid in vidtorid: + return vidtorid[vid] + return "Not Found" + +# print dump +def print_dump(collected_info, table, module, identifier, key_map): + if not table: + click.echo(json.dumps(collected_info, indent=4)) + return + + top_header = [plugins.dump_modules[module].ARG_NAME, "DB_NAME", "DUMP"] + final_collection = [] + for ids in collected_info.keys(): + for db in collected_info[ids].keys(): + total_info = "" + + if collected_info[ids][db]["tables_not_found"]: + total_info += tabulate(collected_info[ids][db]["tables_not_found"], ["Tables Not Found"]) + total_info += "" + + if not key_map: + values = [] + hdrs = ["Keys", "field-value pairs"] + for key_obj in collected_info[ids][db]["keys"]: + if isinstance(key_obj, dict) and key_obj: + key = list(key_obj.keys())[0] + values.append([key, get_dict_str(key_obj[key])]) + total_info += str(tabulate(values, hdrs, tablefmt="grid")) + else: + temp = [] + for key_ in collected_info[ids][db]["keys"]: + temp.append([key_]) + total_info += str(tabulate(temp, headers=["Keys Collected"], tablefmt="grid")) + + total_info += "\n" + if "vidtorid" in collected_info[ids][db]: + table = [] + for pair in collected_info[ids][db]["vidtorid"].items(): + table.append(list(pair)) + total_info +=str(tabulate(table, headers=["vid", "rid"], tablefmt="grid")) + final_collection.append([ids, db, total_info]) + + click.echo(tabulate(final_collection, top_header, tablefmt="grid")) + return + +if __name__ == '__main__': + dump() From f83317a17768d0ef1b2d9d8203384f0f5f789131 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Mon, 24 May 2021 18:29:02 +0000 Subject: [PATCH 05/17] Minor Fixes Signed-off-by: Vivek Reddy Karri --- dump/helper.py | 12 --- dump/main.py | 224 -------------------------------------------- dump/redis_match.py | 4 - 3 files changed, 240 deletions(-) delete mode 100644 dump/main.py diff --git a/dump/helper.py b/dump/helper.py index 2ed46976b2..895c761421 100644 --- a/dump/helper.py +++ b/dump/helper.py @@ -12,15 +12,3 @@ def display_template(dbs): def verbose_print(str): if "VERBOSE" in os.environ and os.environ["VERBOSE"] == "1": print(str) - - - - - - - - - - - - diff --git a/dump/main.py b/dump/main.py deleted file mode 100644 index 6dfca6f06c..0000000000 --- a/dump/main.py +++ /dev/null @@ -1,224 +0,0 @@ -import os,sys,json,re -import click -from tabulate import tabulate - -sys.path.append(os.path.dirname(__file__)) -import plugins -from dump.redis_match import RedisSource, JsonSource -from swsscommon.swsscommon import SonicV2Connector -from utilities_common.multi_asic import multi_asic_ns_choices, multi_asic -from utilities_common.constants import DEFAULT_NAMESPACE - -# Autocompletion Helper -def get_available_modules(ctx, args, incomplete): - return [k for k in plugins.dump_modules.keys() if incomplete in k] - -# Display Modules Callback -def show_modules(ctx, param, value): - if not value or ctx.resilient_parsing: - return - header = ["Module", "Identifier"] - display = [] - for mod in plugins.dump_modules: - display.append((mod, plugins.dump_modules[mod].ARG_NAME)) - click.echo(tabulate(display, header)) - ctx.exit() - -@click.group() -def dump(): - pass - -@dump.command() -@click.pass_context -@click.argument('module', required=True, type=str, autocompletion=get_available_modules) -@click.argument('identifier', required=True, type=str) -@click.option('--show', '-s', is_flag=True, default=False, help='Display Modules Available', is_eager=True, expose_value=False, callback=show_modules) -@click.option('--db', '-d', multiple=True, help='Only dump from these Databases') -@click.option('--table', '-t', is_flag=True, default=False, help='Print in tabular format', show_default=True) -@click.option('--key-map', '-k', is_flag=True, default=False, help="Only fetch the keys matched, don't extract field-value dumps", show_default=True) -@click.option('--verbose', '-v', is_flag=True, default=False, help="Prints any intermediate output to stdout useful for dev & troubleshooting", show_default=True) -@click.option('--namespace', '-n', default=DEFAULT_NAMESPACE, type=str, show_default=True, help='Dump the redis-state for this namespace.') -def state(ctx, module, identifier, db, table, key_map, verbose, namespace): - """ - Dump the redis-state of the identifier for the module specified - """ - if not multi_asic.is_multi_asic() and namespace != DEFAULT_NAMESPACE: - click.echo("Namespace option is not valid for a single-ASIC device") - ctx.exit() - - if multi_asic.is_multi_asic() and (namespace != DEFAULT_NAMESPACE and namespace not in multi_asic_ns_choices()): - click.echo("Namespace option is not valid. Choose one of {}".format(multi_asic_ns_choices())) - ctx.exit() - - if module not in plugins.dump_modules: - click.echo("No Matching Plugin has been Implemented") - ctx.exit() - - if verbose: - os.environ["VERBOSE"] = "1" - else: - os.environ["VERBOSE"] = "0" - - ctx.module = module - obj = plugins.dump_modules[module]() - - if identifier == "all": - ids = obj.get_all_args(namespace) - else: - ids = identifier.split(",") - - params = {} - collected_info = {} - params['namespace'] = namespace - for arg in ids: - params[plugins.dump_modules[module].ARG_NAME] = arg - collected_info[arg] = obj.execute(params) - - if len(db) > 0: - collected_info = filter_out_dbs(db, collected_info) - - vidtorid = extract_rid(collected_info, namespace) - - if not key_map: - collected_info = populate_fv(collected_info, module, namespace) - - for id in vidtorid.keys(): - if vidtorid[id] and vidtorid: - collected_info[id]["ASIC_DB"]["vidtorid"] = vidtorid[id] - - print_dump(collected_info, table, module, identifier, key_map) - - return - -def extract_rid(info, ns): - r = SonicV2Connector(namespace=ns, host="127.0.0.1") - r.connect("ASIC_DB") - vidtorid = {} - for arg in info.keys(): - vidtorid[arg] = get_v_r_map(r, info[arg]) - return vidtorid - -def get_v_r_map(r, single_dict): - v_r_map = {} - asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}" - - if "ASIC_DB" in single_dict and 'keys' in single_dict["ASIC_DB"]: - for redis_key in single_dict["ASIC_DB"]['keys']: - if re.match(asic_obj_ptrn, redis_key): - matches = re.findall(r"oid:0x\w{1,14}", redis_key) - if matches: - vid = matches[0] - v_r_map[vid] = vid_to_rid(vid, r) - return v_r_map - -# Get a vid:rid for the input vid -def vid_to_rid(vid, r): - rid = r.get("ASIC_DB", "VIDTORID", vid) - if not rid: - rid = "Real ID Not Found" - return rid - -# Filter dbs which are not required -def filter_out_dbs(db_list, collected_info): - args_ = list(collected_info.keys()) - for arg in args_: - dbs = list(collected_info[arg].keys()) - for db in dbs: - if db not in db_list: - del collected_info[arg][db] - return collected_info - -def populate_fv(info, module, namespace): - - all_dbs = set() - for id in info.keys(): - for db_name in info[id].keys(): - all_dbs.add(db_name) - - db_dict = {} - for db_name in all_dbs: - if db_name is "CONFIG_FILE": - db_dict[db_name] = JsonSource() - db_dict[db_name].connect(plugins.dump_modules[module].CONFIG_FILE, namespace) - else: - db_dict[db_name] = RedisSource() - db_dict[db_name].connect(db_name, namespace) - - final_info = {} - for id in info.keys(): - final_info[id] = {} - for db_name in info[id].keys(): - final_info[id][db_name] = {} - final_info[id][db_name]["keys"] = [] - final_info[id][db_name]["tables_not_found"] = info[id][db_name]["tables_not_found"] - for key in info[id][db_name]["keys"]: - final_info[id][db_name]["keys"].append({key : db_dict[db_name].get(db_name, key)}) - - return final_info - -def get_dict_str(key_obj): - table = [] - for pair in key_obj.items(): - table.append(list(pair)) - return tabulate(table, headers=["field", "value"], tablefmt="psql") - -def get_keys(dump): - keys = [] - for key_ in dump: - if isinstance(key_, dict) and key_: - keys.append(list(key_.keys())[0]) - else: - keys.append(key_) - return keys - -def get_rid(redis_key, vidtorid): - matches = re.findall(r"oid:0x\w{1,14}", redis_key) - if matches: - vid = matches[0] - if vid in vidtorid: - return vidtorid[vid] - return "Not Found" - -# print dump -def print_dump(collected_info, table, module, identifier, key_map): - if not table: - click.echo(json.dumps(collected_info, indent=4)) - return - - top_header = [plugins.dump_modules[module].ARG_NAME, "DB_NAME", "DUMP"] - final_collection = [] - for ids in collected_info.keys(): - for db in collected_info[ids].keys(): - total_info = "" - - if collected_info[ids][db]["tables_not_found"]: - total_info += tabulate(collected_info[ids][db]["tables_not_found"], ["Tables Not Found"]) - total_info += "" - - if not key_map: - values = [] - hdrs = ["Keys", "field-value pairs"] - for key_obj in collected_info[ids][db]["keys"]: - if isinstance(key_obj, dict) and key_obj: - key = list(key_obj.keys())[0] - values.append([key, get_dict_str(key_obj[key])]) - total_info += str(tabulate(values, hdrs, tablefmt="grid")) - else: - temp = [] - for key_ in collected_info[ids][db]["keys"]: - temp.append([key_]) - total_info += str(tabulate(temp, headers=["Keys Collected"], tablefmt="grid")) - - total_info += "\n" - if "vidtorid" in collected_info[ids][db]: - table = [] - for pair in collected_info[ids][db]["vidtorid"].items(): - table.append(list(pair)) - total_info +=str(tabulate(table, headers=["vid", "rid"], tablefmt="grid")) - final_collection.append([ids, db, total_info]) - - click.echo(tabulate(final_collection, top_header, tablefmt="grid")) - return - -if __name__ == '__main__': - dump() diff --git a/dump/redis_match.py b/dump/redis_match.py index cbca5e92e4..e9d85cc215 100644 --- a/dump/redis_match.py +++ b/dump/redis_match.py @@ -276,7 +276,3 @@ def __fill(self, src, req, filtered_keys): else: template["keys"].append(key) return template - - - - From c9dcfefd0315601d3da9a5a44c370de62f22b1f8 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 27 May 2021 17:22:24 +0000 Subject: [PATCH 06/17] Name Change and match_list changes made Signed-off-by: Vivek Reddy Karri --- dump/{redis_match.py => match_infra.py} | 7 ++++++- tests/dump_tests/match_engine_test.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) rename dump/{redis_match.py => match_infra.py} (97%) diff --git a/dump/redis_match.py b/dump/match_infra.py similarity index 97% rename from dump/redis_match.py rename to dump/match_infra.py index e9d85cc215..6b75e3eabf 100644 --- a/dump/redis_match.py +++ b/dump/match_infra.py @@ -29,6 +29,7 @@ def __init__(self): self.file = "" self.just_keys = True self.ns = '' + self.match_entire_list = False def __str__(self): str = "MatchRequest: \n" @@ -52,6 +53,10 @@ def __str__(self): str += "Return Fields: " + ",".join(self.return_fields) if self.ns: str += "Namespace: " + self.ns + if self.match_entire_list: + str += "Match Entire List: True" + else: + str += "Match Entire List: False" return str class SourceAdapter: @@ -252,7 +257,7 @@ def __filter_out_keys(self, src, req, all_matched_keys): filtered_keys = [] for key in all_matched_keys: f_values = src.hget(req.db, key, req.field) - if "," in f_values: # Fields Containing Multile Values + if "," in f_values and not req.match_entire_list: # Fields Containing Multile Values f_value = f_values.split(",") else: f_value = [f_values] diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index c2f0b7519a..3180c7b3fa 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -2,7 +2,7 @@ import jsonpatch import unittest import pytest -from dump.redis_match import MatchEngine, error_dict, MatchRequest +from dump.match_infra import MatchEngine, error_dict, MatchRequest from deepdiff import DeepDiff test_path = os.path.join(os.path.abspath(__file__),"../") @@ -293,5 +293,20 @@ def test_file_source_with_not_only_return_keys(self): exp_dict = {"COPP_GROUP|queue4_group2": {"trap_action": "copy", "trap_priority": "4", "queue": "4", "meter_type": "packets", "mode": "sr_tcm", "cir": "600", "cbs": "600", "red_action": "drop"}} ddiff = DeepDiff(exp_dict, recv_dict) assert not ddiff, ddiff + + def test_match_entire_list(self): + req = MatchRequest() + req.db = "CONFIG_DB" + req.table = "PORT" + req.key_pattern = "*" + req.field = "lanes" + req.value = "61,62,63,64" + req.match_entire_list = True + req.just_keys = True + ret = self.match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "PORT|Ethernet60" in ret["keys"] + \ No newline at end of file From bd47df0b0346d9a9b49c580b2ca4ee046ac1a97f Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 27 May 2021 17:26:00 +0000 Subject: [PATCH 07/17] Minor Typo Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index 6b75e3eabf..e57fca10c7 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -257,7 +257,7 @@ def __filter_out_keys(self, src, req, all_matched_keys): filtered_keys = [] for key in all_matched_keys: f_values = src.hget(req.db, key, req.field) - if "," in f_values and not req.match_entire_list: # Fields Containing Multile Values + if "," in f_values and not req.match_entire_list: # Fields Containing Multiple Values f_value = f_values.split(",") else: f_value = [f_values] From b8da286837dd30ca9c45920df928ec2abd6bcf98 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Sat, 29 May 2021 04:36:27 +0000 Subject: [PATCH 08/17] Final Changes before review made Signed-off-by: Vivek Reddy Karri --- dump/helper.py | 23 +++++++++++++++++++++++ dump/match_infra.py | 16 ++++++++++------ tests/dump_tests/match_engine_test.py | 4 ++-- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/dump/helper.py b/dump/helper.py index 895c761421..dbcbe99460 100644 --- a/dump/helper.py +++ b/dump/helper.py @@ -12,3 +12,26 @@ def display_template(dbs): def verbose_print(str): if "VERBOSE" in os.environ and os.environ["VERBOSE"] == "1": print(str) + +# Handles general error conditions, if any experienced by the module, +# Set excep = True, to raise a exception +def handle_error(err_str, excep=False): + if excep: + raise Exception("ERROR : {}".format(err_str)) + else: + print("ERROR : {}".format(err_str)) + + +def handle_multiple_keys_matched_error(err_str, key_to_go_with="", excep=False): + if excep: + handle_error(err_str, True) + else: + print("ERROR (AMBIGUITY): {} \n Proceeding with the key {}".format(err_str, key_to_go_with)) + + +def sort_lists(ret): + for db in ret.keys(): + for key in ret[db].keys(): + if isinstance(ret[db][key], list): + ret[db][key].sort() + return ret \ No newline at end of file diff --git a/dump/match_infra.py b/dump/match_infra.py index e57fca10c7..90ca1f59bb 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -15,7 +15,8 @@ "SRC_VAGUE": "Only one of db or file should be provided", "CONN_ERR" : "Connection Error", "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty", - "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type" + "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type", + "NO_ENTRIES": "No Keys found after applying the filtering criteria" } class MatchRequest: @@ -32,7 +33,7 @@ def __init__(self): self.match_entire_list = False def __str__(self): - str = "MatchRequest: \n" + str = "----------------------- \n MatchRequest: \n" if self.db: str += "db:{} , ".format(self.db) if self.file: @@ -50,13 +51,13 @@ def __str__(self): else: str += "just_keys:False " if len(self.return_fields) > 0: - str += "Return Fields: " + ",".join(self.return_fields) + str += "Return Fields: " + ",".join(self.return_fields) + " " if self.ns: str += "Namespace: " + self.ns if self.match_entire_list: - str += "Match Entire List: True" + str += "Match Entire List: True " else: - str += "Match Entire List: False" + str += "Match Entire List: False " return str class SourceAdapter: @@ -165,7 +166,6 @@ def hget(self, db, key, field): key_ptrn = tokens[-1] tokens.pop() table = sp.join(tokens) - print(table, key_ptrn) if table in self.db_driver and key_ptrn in self.db_driver[table] and field in self.db_driver[table][key_ptrn]: return self.db_driver[table][key_ptrn][field] return "" @@ -200,6 +200,9 @@ def fetch(self, req): filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) verbose_print("Filtered Keys:" + str(filtered_keys)) + if not filtered_keys: + template['error'] = error_dict["NO_ENTRIES"] + return self.__return_error(template) return self.__fill(src, req, filtered_keys) def __ret_template(self): @@ -280,4 +283,5 @@ def __fill(self, src, req, filtered_keys): template["return_values"][key][field] = src.hget(req.db, key, field) else: template["keys"].append(key) + verbose_print("Return Values:" + str(template["return_values"])) return template diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index 3180c7b3fa..3c77e864ed 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -169,7 +169,7 @@ def test_for_no_match(self): req.field = "SAI_SWITCH_ATTR_SRC_MAC_ADDRESS" req.value = "DE:AD:EE:EE:EE" ret = self.match_engine.fetch(req) - assert ret["error"] == "" + assert ret["error"] == error_dict["NO_ENTRIES"] assert len(ret["keys"]) == 0 def test_for_no_key_match(self): @@ -188,7 +188,7 @@ def test_field_value_no_match(self): req.field = "led_status" req.value = "yellow" ret = self.match_engine.fetch(req) - assert ret["error"] == "" + assert ret["error"] == error_dict["NO_ENTRIES"] assert len(ret["keys"]) == 0 def test_return_keys(self): From 8cd32cd6b3bf009f0deef2eabe03083cb187a399 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 9 Jun 2021 07:48:45 +0000 Subject: [PATCH 09/17] Comments addressed Signed-off-by: Vivek Reddy Karri --- dump/helper.py | 26 +-- dump/match_infra.py | 302 +++++++++++++------------- tests/dump_tests/match_engine_test.py | 218 +++++-------------- 3 files changed, 224 insertions(+), 322 deletions(-) diff --git a/dump/helper.py b/dump/helper.py index dbcbe99460..352439f9a2 100644 --- a/dump/helper.py +++ b/dump/helper.py @@ -1,37 +1,35 @@ -import os +import os, sys -# Generate a Template which will be returned by Executor Classes -def display_template(dbs): - template = {} - for db in dbs: - template[db] = {} - template[db]['keys'] = [] - template[db]['tables_not_found'] = [] - return template +def create_template_dict(dbs): + """ Generate a Template which will be returned by Executor Classes """ + return {db: {'keys': [], 'tables_not_found': []} for db in dbs} def verbose_print(str): if "VERBOSE" in os.environ and os.environ["VERBOSE"] == "1": print(str) -# Handles general error conditions, if any experienced by the module, -# Set excep = True, to raise a exception def handle_error(err_str, excep=False): + """ + Handles general error conditions, if any experienced by the module, + Set excep = True, to raise a exception + """ if excep: raise Exception("ERROR : {}".format(err_str)) else: - print("ERROR : {}".format(err_str)) + print("ERROR : {}".format(err_str), file = sys.stderr) def handle_multiple_keys_matched_error(err_str, key_to_go_with="", excep=False): if excep: handle_error(err_str, True) else: - print("ERROR (AMBIGUITY): {} \n Proceeding with the key {}".format(err_str, key_to_go_with)) + print("ERROR (AMBIGUITY): {} \n Proceeding with the key {}".format(err_str, key_to_go_with), file = sys.stderr) def sort_lists(ret): + """ Used to sort the nested list returned by the template dict. """ for db in ret.keys(): for key in ret[db].keys(): if isinstance(ret[db][key], list): ret[db][key].sort() - return ret \ No newline at end of file + return ret diff --git a/dump/match_infra.py b/dump/match_infra.py index 90ca1f59bb..a5994a41c7 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -1,36 +1,93 @@ -import re, json, os, sys +import re, json, os, fnmatch from dump.helper import verbose_print +from abc import ABC, abstractmethod from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig -error_dict = { +EXCEP_DICT = { "INV_REQ": "Argument should be of type MatchRequest", "INV_DB": "DB provided is not valid", - "INV_JSON": "Not a properly formatted JSON file", - "INV_PTTRN": "No Entries found for Table|key_pattern provided", - "NO_FILE": "JSON File not found", + "NO_MATCHES": "No Entries found for Table|key_pattern provided", "NO_SRC": "Either one of db or file in the request should be non-empty", - "NO_KEY": "'key_pattern' cannot be empty", "NO_TABLE": "No 'table' name provided", + "NO_KEY": "'key_pattern' cannot be empty", "NO_VALUE" : "Field is provided, but no value is provided to compare with", "SRC_VAGUE": "Only one of db or file should be provided", "CONN_ERR" : "Connection Error", "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty", "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type", - "NO_ENTRIES": "No Keys found after applying the filtering criteria" + "NO_ENTRIES": "No Keys found after applying the filtering criteria", + "FILE_R_EXEP": "Exception Caught While Reading the json cfg file provided" } class MatchRequest: - def __init__(self): - self.table = None - self.key_pattern = "*" - self.field = None - self.value = None - self.return_fields = [] - self.db = "" - self.file = "" - self.just_keys = True - self.ns = '' - self.match_entire_list = False + """ + Request Object which should be passed to the MatchEngine + + Attributes: + "table" : A Valid Table Name + "key_pattern" : Pattern of the redis-key to match. Defaults to "*". Eg: "*" will match all the keys. + Supports these glob style patterns. https://redis.io/commands/KEYS + "field" : Field to check for a match,Defaults to None + "value" : Value to match, Defaults to None + "return_fields" : An iterable type, where each element woudld imply a field to return from all the filtered keys + "db" : A Valid DB name, Defaults to "". + "file" : A Valid Config JSON file, Eg: copp_cfg.json, Defaults to "". + Only one of the db/file fields should have a non-empty string. + "just_keys" : If true, Only Returns the keys matched. Does not return field-value pairs. Defaults to True + "ns" : namespace argument, if nothing is provided, default namespace is used + "match_entire_list" : When this arg is set to true, entire list is matched incluing the ",". + When False, the values are split based on "," and individual items are matched with + """ + def __init__(self, **kwargs): + self.table = kwargs["table"] if "table" in kwargs else None + self.key_pattern = kwargs["key_pattern"] if "key_pattern" in kwargs else "*" + self.field = kwargs["field"] if "field" in kwargs else None + self.value = kwargs["value"] if "value" in kwargs else None + self.return_fields = kwargs["return_fields"] if "return_fields" in kwargs else [] + self.db = kwargs["db"] if "db" in kwargs else "" + self.file = kwargs["file"] if "file" in kwargs else "" + self.just_keys = kwargs["just_keys"] if "just_keys" in kwargs else True + self.ns = kwargs["ns"] if "ns" in kwargs else "" + self.match_entire_list = kwargs["match_entire_list"] if "match_entire_list" in kwargs else False + err = self.__static_checks() + verbose_print(str(err)) + if err: + raise Exception("Static Checks for the MatchRequest Failed, Reason: \n" + err) + + + def __static_checks(self): + + if not self.db and not self.file: + return EXCEP_DICT["NO_SRC"] + + if self.db and self.file: + return EXCEP_DICT["SRC_VAGUE"] + + if not self.db: + try: + with open(self.file) as f: + json.load(f) + except Exception as e: + return EXCEP_DICT["FILE_R_EXEP"] + str(e) + + if not self.file and self.db not in SonicDBConfig.getDbList(): + return EXCEP_DICT["INV_DB"] + + if not self.table: + return EXCEP_DICT["NO_TABLE"] + + if not isinstance(self.return_fields, list): + return EXCEP_DICT["BAD_FORMAT_RE_FIELDS"] + + if not self.just_keys and self.return_fields: + return EXCEP_DICT["JUST_KEYS_COMPAT"] + + if self.field and not self.value: + return EXCEP_DICT["NO_VALUE"] + + verbose_print("MatchRequest Checks Passed") + + return "" def __str__(self): str = "----------------------- \n MatchRequest: \n" @@ -41,93 +98,89 @@ def __str__(self): if self.table: str += "table:{} , ".format(self.table) if self.key_pattern: - str += "key_regx:{} , ".format(self.key_pattern) + str += "key_pattern:{} , ".format(self.key_pattern) if self.field: str += "field:{} , ".format(self.field) if self.value: str += "value:{} , ".format(self.value) if self.just_keys: - str += "just_keys:True " + str += "just_keys:True ," else: - str += "just_keys:False " + str += "just_keys:False ," if len(self.return_fields) > 0: - str += "Return Fields: " + ",".join(self.return_fields) + " " + str += "return_fields: " + ",".join(self.return_fields) + " " if self.ns: - str += "Namespace: " + self.ns + str += "namespace: ," + self.ns if self.match_entire_list: - str += "Match Entire List: True " + str += "match_list: True ," else: - str += "Match Entire List: False " + str += "match_list: False ," return str -class SourceAdapter: +class SourceAdapter(ABC): + """ Source Adaptor offers unified interface to Data Sources """ + def __init__(self): pass + @abstractmethod def connect(self, db, ns): + """ Return True for Success, False for failure """ return False + @abstractmethod def getKeys(self, db, table, key_pattern): return [] + @abstractmethod def get(self, db, key): return {} + @abstractmethod def hget(self, db, key, field): return "" + @abstractmethod def sep(self, db): return "" class RedisSource(SourceAdapter): + """ Concrete Adaptor Class for connecting to Redis Data Sources """ + def __init__(self): - self.db_driver = None + self.conn = None def connect(self, db, ns): try: - self.db_driver = SonicV2Connector(namespace=ns, host="127.0.0.1") - self.db_driver.connect(db) + self.conn = SonicV2Connector(namespace=ns, host="127.0.0.1") + self.conn.connect(db) except Exception as e: verbose_print("RedisSource: Connection Failed\n" + str(e)) return False return True def sep(self, db): - return self.db_driver.get_db_separator(db) + return self.conn.get_db_separator(db) def getKeys(self, db, table, key_pattern): - try: - keys = self.db_driver.keys(db, table + self.sep(db) + key_pattern) - except Exception as e: - verbose_print("RedisSource: {}|{}|{} Keys fetch Request Failed for DB {}\n".format(table, self.sep(db), key_pattern, db) + str(e)) - return [] - return keys + return self.conn.keys(db, table + self.sep(db) + key_pattern) def get(self, db, key): - try: - fv_pairs = self.db_driver.get_all(db, key) - except Exception as e: - verbose_print("RedisSource: hgetall {} request failed for DB {}\n".format(key, db) + str(e)) - return {} - return fv_pairs + return self.conn.get_all(db, key) def hget(self, db, key, field): - try: - value = self.db_driver.get(db, key, field) - except Exception as e: - verbose_print("RedisSource: hget {} {} request failed for DB {}\n".format(key, field) + str(e)) - return "" - return value + return self.conn.get(db, key, field) class JsonSource(SourceAdapter): + """ Concrete Adaptor Class for connecting to JSON Data Sources """ def __init__(self): - self.db_driver = None + self.json_data = None def connect(self, db, ns): try: with open(db) as f: - self.db_driver = json.load(f) + self.json_data = json.load(f) except Exception as e: verbose_print("JsonSource: Loading the JSON file failed" + str(e)) return False @@ -137,49 +190,27 @@ def sep(self, db): return SonicDBConfig.getSeparator("CONFIG_DB") def getKeys(self, db, table, key_pattern): - if table not in self.db_driver: + if table not in self.json_data: return [] - - all_keys = self.db_driver[table].keys() - key_ptrn = key_pattern - key_ptrn = re.escape(key_ptrn) - key_ptrn = key_ptrn.replace("\\*", ".*") - filtered_keys = [] - for key in all_keys: - if re.match(key_ptrn, key): - filtered_keys.append(table+self.sep(db)+key) - return filtered_keys + # https://docs.python.org/3.7/library/fnmatch.html + kp = key_pattern.replace("[^", "[!") + kys = fnmatch.filter(self.json_data[table].keys(), kp) + return [table+self.sep(db)+ky for ky in kys] def get(self, db, key): - sp = self.sep(db) - tokens = key.split(sp) - key_ptrn = tokens[-1] - tokens.pop() - table = sp.join(tokens) - if table in self.db_driver and key_ptrn in self.db_driver[table]: - return self.db_driver[table][key_ptrn] - return {} + sep = self.sep(db) + table, key = key.split(sep, 1) + return self.json_data.get(table, {}).get(key, {}) def hget(self, db, key, field): - sp = self.sep(db) - tokens = key.split(sp) - key_ptrn = tokens[-1] - tokens.pop() - table = sp.join(tokens) - if table in self.db_driver and key_ptrn in self.db_driver[table] and field in self.db_driver[table][key_ptrn]: - return self.db_driver[table][key_ptrn][field] - return "" + sep = self.sep(db) + table, key = key.split(sep, 1) + return self.json_data.get(table, "").get(key, "").get(field, "") class MatchEngine: + """ Pass in a MatchRequest, to fetch the Matched dump from the Data sources """ - # Given a request obj, find its match in the redis - def fetch(self, req): - verbose_print(str(req)) - template = self.__ret_template() - template['error'] = self.__validate_request(req) - if template['error']: - return self.__return_error(template) - + def __get_source_adapter(self, req): src = None d_src = "" if req.db: @@ -188,79 +219,26 @@ def fetch(self, req): else: d_src = req.file src = JsonSource() - - if not src.connect(d_src, req.ns): - template['error'] = error_dict["CONN_ERR"] - return self.__return_error(template) - verbose_print("MatchRequest Checks Passed") - all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern) - if not all_matched_keys or len(all_matched_keys) == 0: - template['error'] = error_dict["INV_PTTRN"] - return self.__return_error(template) - - filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) - verbose_print("Filtered Keys:" + str(filtered_keys)) - if not filtered_keys: - template['error'] = error_dict["NO_ENTRIES"] - return self.__return_error(template) - return self.__fill(src, req, filtered_keys) + return d_src, src - def __ret_template(self): + def __create_template(self): return {"error" : "", "keys" : [], "return_values" : {}} - - def __return_error(self, template): + def __display_error(self, err): + template = self.__create_template() + template['error'] = err verbose_print("MatchEngine: \n" + template['error']) return template - def __validate_request(self, req): - - if not isinstance(req, MatchRequest): - return error_dict["INV_REQ"] - - if not(req.db) and not(req.file): - return error_dict["NO_SRC"] - - if req.db and req.file: - return error_dict["SRC_VAGUE"] - - if not req.db and os.path.exists(req.file): - try: - with open(req.file) as f: - json.load(f) - except ValueError as e: - return error_dict["INV_JSON"] - elif not req.db: - return error_dict["NO_FILE"] - - if not(req.file) and req.db not in SonicDBConfig.getDbList(): - return error_dict["INV_DB"] - - if not req.table: - return error_dict["NO_TABLE"] - - if not req.key_pattern: - return error_dict["NO_KEY"] - - if not isinstance(req.return_fields, list): - return error_dict["BAD_FORMAT_RE_FIELDS"] - - if not req.just_keys and len(req.return_fields) > 0: - return error_dict["JUST_KEYS_COMPAT"] - - if req.field and not req.value: - return error_dict["NO_VALUE"] - - return "" - def __filter_out_keys(self, src, req, all_matched_keys): - if not (req.field): + # TODO: Custom Callbacks for Complex Matching Criteria + if not req.field: return all_matched_keys filtered_keys = [] for key in all_matched_keys: f_values = src.hget(req.db, key, req.field) - if "," in f_values and not req.match_entire_list: # Fields Containing Multiple Values + if "," in f_values and not req.match_entire_list: f_value = f_values.split(",") else: f_value = [f_values] @@ -268,9 +246,7 @@ def __filter_out_keys(self, src, req, all_matched_keys): filtered_keys.append(key) return filtered_keys - def __fill(self, src, req, filtered_keys): - - template = self.__ret_template() + def __fill_template(self, src, req, filtered_keys, template): for key in filtered_keys: temp = {} if not req.just_keys: @@ -285,3 +261,31 @@ def __fill(self, src, req, filtered_keys): template["keys"].append(key) verbose_print("Return Values:" + str(template["return_values"])) return template + + def fetch(self, req): + """ Given a request obj, find its match in the data source provided """ + if not isinstance(req, MatchRequest): + return self.__display_error(EXCEP_DICT["INV_REQ"]) + + verbose_print(str(req)) + + if not req.key_pattern: + return self.__display_error(EXCEP_DICT["NO_KEY"]) + + d_src, src = self.__get_source_adapter(req) + if not src.connect(d_src, req.ns): + return self.__display_error(EXCEP_DICT["CONN_ERR"]) + + template = self.__create_template() + all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern) + if not all_matched_keys: + return self.__display_error(EXCEP_DICT["NO_MATCHES"]) + + filtered_keys = self.__filter_out_keys(src, req, all_matched_keys) + verbose_print("Filtered Keys:" + str(filtered_keys)) + if not filtered_keys: + return self.__display_error(EXCEP_DICT["NO_ENTRIES"]) + return self.__fill_template(src, req, filtered_keys, template) + + + diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index 3c77e864ed..4636e26bb2 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -2,7 +2,7 @@ import jsonpatch import unittest import pytest -from dump.match_infra import MatchEngine, error_dict, MatchRequest +from dump.match_infra import MatchEngine, EXCEP_DICT, MatchRequest from deepdiff import DeepDiff test_path = os.path.join(os.path.abspath(__file__),"../") @@ -18,99 +18,66 @@ def mock_setup(): os.environ["VERBOSE"] = "0" -class TestInvalidRequest(unittest.TestCase): +class TestMatchRequestValidation(unittest.TestCase): def __init__(self, *args, **kwargs): - super(TestInvalidRequest, self).__init__(*args, **kwargs) + super(TestMatchRequestValidation, self).__init__(*args, **kwargs) self.match_engine = MatchEngine() + def assertRaisesWithMessage(self, msg, func, *args, **kwargs): + try: + func(*args, **kwargs) + assert False, "Expected an exception with msg: " + msg + except Exception as inst: + print(inst) + assert msg in str(inst) + def test_bad_request(self): req = [] ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["INV_REQ"] + assert ret["error"] == EXCEP_DICT["INV_REQ"] def test_no_source(self): - req = MatchRequest() - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_SRC"] - + self.assertRaisesWithMessage(EXCEP_DICT["NO_SRC"], MatchRequest) + def test_vague_source(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.file = "/etc/sonic/copp_cfg.json" - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["SRC_VAGUE"] + self.assertRaisesWithMessage(EXCEP_DICT["SRC_VAGUE"], MatchRequest, db="CONFIG_DB", file="/etc/sonic/copp_cfg.json") def test_no_file(self): - req = MatchRequest() - req.file = os.path.join(test_path, "random_db.json") - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_FILE"] + self.assertRaisesWithMessage(EXCEP_DICT["FILE_R_EXEP"], MatchRequest, file=os.path.join(test_path, "random_db.json")) def test_invalid_db(self): - req = MatchRequest() - req.db = "CONFIGURATION_DB" - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["INV_DB"] - - def test_bad_key_pattern(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "PORT" - req.key_pattern = "" + self.assertRaisesWithMessage(EXCEP_DICT["INV_DB"], MatchRequest, db="CONFIGURATION_DB") + + def test_bad_key_pattern(self): + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="") ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_KEY"] + assert ret["error"] == EXCEP_DICT["NO_KEY"] - def test_no_value(self): - req = MatchRequest() - req.db = "APPL_DB" - req.table = "COPP_TABLE" - req.key_pattern = "*" - req.field = "trap_ids" - req.value = "" - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_VALUE"] + def test_no_value(self): + self.assertRaisesWithMessage(EXCEP_DICT["NO_VALUE"], MatchRequest, db="APPL_DB", table="COPP_TABLE", key_pattern="*", field="trap_ids", value="") - def test_no_table(self): - req = MatchRequest() - req.db = "APPL_DB" - req.table = "" - req.key_pattern = "*" - req.field = "trap_ids" - req.value = "bgpv6" - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_TABLE"] + def test_no_table(self): + self.assertRaisesWithMessage(EXCEP_DICT["NO_TABLE"], MatchRequest, db="APPL_DB", table="", key_pattern="*", field="trap_ids", value = "bgpv6") def test_just_keys_return_fields_compat(self): - req = MatchRequest() - req.db = "APPL_DB" - req.table = "COPP_TABLE" - req.key_pattern = "*" - req.field = "trap_ids" - req.value = "" - req.just_keys = False - req.return_fields = ["trap_group"] - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["JUST_KEYS_COMPAT"] + self.assertRaisesWithMessage(EXCEP_DICT["JUST_KEYS_COMPAT"], MatchRequest, db="APPL_DB", return_fields=["trap_group"], table="COPP_TABLE", + key_pattern="*", field="trap_ids", value="", just_keys=False) def test_invalid_combination(self): - - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "COPP_TRAP" - req.key_pattern = "*" - req.field = "trap_ids" - req.value = "sample_packet" + req = MatchRequest(db="CONFIG_DB", table="COPP_TRAP", key_pattern="*", field="trap_ids", value="sample_packet") ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["INV_PTTRN"] + assert ret["error"] == EXCEP_DICT["NO_MATCHES"] def test_return_fields_bad_format(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "REBOOT_CAUSE" - req.return_fields = "cause" - ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["BAD_FORMAT_RE_FIELDS"] + self.assertRaisesWithMessage(EXCEP_DICT["BAD_FORMAT_RE_FIELDS"], MatchRequest, db="STATE_DB", table="REBOOT_CAUSE", key_pattern="*", return_fields="cause") + + def test_valid_match_request(self): + try: + req = MatchRequest(db="APPL_DB", table="PORT_TABLE", field="lanes", value="202") + except Exception as e: + assert False, "Exception Raised for a Valid MatchRequest" + str(e) + class TestMatchEngine(unittest.TestCase): @@ -119,10 +86,7 @@ def __init__(self, *args, **kwargs): self.match_engine = MatchEngine() def test_key_pattern_wildcard(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "SFLOW_COLLECTOR" - req.key_pattern = "*" + req = MatchRequest(db="CONFIG_DB", table="SFLOW_COLLECTOR", key_pattern="*") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 2 @@ -130,10 +94,7 @@ def test_key_pattern_wildcard(self): assert "SFLOW_COLLECTOR|prod" in ret['keys'] def test_key_pattern_complex(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "ACL_RULE" - req.key_pattern = "EVERFLOW*" + req = MatchRequest(db="CONFIG_DB", table="ACL_RULE", key_pattern="EVERFLOW*") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 2 @@ -141,61 +102,38 @@ def test_key_pattern_complex(self): assert "ACL_RULE|EVERFLOW|RULE_08" in ret['keys'] def test_field_value_match(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "ACL_TABLE" - req.field = "policy_desc" - req.value = "SSH_ONLY" + req = MatchRequest(db="CONFIG_DB", table="ACL_TABLE", field="policy_desc", value="SSH_ONLY") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "ACL_TABLE|SSH_ONLY" in ret['keys'] def test_field_value_match_list_type(self): - req = MatchRequest() - req.db = "APPL_DB" - req.table = "PORT_TABLE" - req.field = "lanes" - req.value = "202" + req = MatchRequest(db="APPL_DB", table="PORT_TABLE", field="lanes", value="202") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "PORT_TABLE:Ethernet200" in ret['keys'] def test_for_no_match(self): - req = MatchRequest() - req.db = "ASIC_DB" - req.table = "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH" - req.field = "SAI_SWITCH_ATTR_SRC_MAC_ADDRESS" - req.value = "DE:AD:EE:EE:EE" + req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_SWITCH", field="SAI_SWITCH_ATTR_SRC_MAC_ADDRESS", value="DE:AD:EE:EE:EE") ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_ENTRIES"] + assert ret["error"] == EXCEP_DICT["NO_ENTRIES"] assert len(ret["keys"]) == 0 def test_for_no_key_match(self): - req = MatchRequest() - req.db = "ASIC_DB" - req.table = "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH" - req.key_pattern = "oid:0x22*" + req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_SWITCH", key_pattern="oid:0x22*") ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["INV_PTTRN"] + assert ret["error"] == EXCEP_DICT["NO_MATCHES"] def test_field_value_no_match(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "FAN_INFO" - req.key_pattern = "*" - req.field = "led_status" - req.value = "yellow" + req = MatchRequest(db="STATE_DB", table="FAN_INFO", key_pattern="*", field="led_status", value="yellow") ret = self.match_engine.fetch(req) - assert ret["error"] == error_dict["NO_ENTRIES"] + assert ret["error"] == EXCEP_DICT["NO_ENTRIES"] assert len(ret["keys"]) == 0 def test_return_keys(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "REBOOT_CAUSE" - req.return_fields = ["cause"] + req = MatchRequest(db="STATE_DB", table="REBOOT_CAUSE", return_fields=["cause"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 2 @@ -203,36 +141,21 @@ def test_return_keys(self): assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] def test_return_fields_with_key_filtering(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "REBOOT_CAUSE" - req.key_pattern = "2020_10_09_02*" - req.return_fields = ["cause"] + req = MatchRequest(db="STATE_DB", table="REBOOT_CAUSE", key_pattern="2020_10_09_02*", return_fields=["cause"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] def test_return_fields_with_field_value_filtering(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "CHASSIS_MODULE_TABLE" - req.field = "oper_status" - req.value = "Offline" - req.return_fields = ["slot"] + req = MatchRequest(db="STATE_DB", table="CHASSIS_MODULE_TABLE", field="oper_status", value="Offline", return_fields = ["slot"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "18" == ret["return_values"]["CHASSIS_MODULE_TABLE|FABRIC-CARD1"]["slot"] def test_return_fields_with_all_filtering(self): - req = MatchRequest() - req.db = "STATE_DB" - req.table = "VXLAN_TUNNEL_TABLE" - req.key_pattern = "EVPN_25.25.25.2*" - req.field = "operstatus" - req.value = "down" - req.return_fields = ["src_ip"] + req = MatchRequest(db="STATE_DB", table="VXLAN_TUNNEL_TABLE", key_pattern="EVPN_25.25.25.2*", field="operstatus", value="down", return_fields = ["src_ip"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 3 @@ -241,11 +164,7 @@ def test_return_fields_with_all_filtering(self): assert "1.1.1.1" == ret["return_values"]["VXLAN_TUNNEL_TABLE|EVPN_25.25.25.27"]["src_ip"] def test_just_keys_false(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "SFLOW" - req.key_pattern = "global" - req.just_keys = False + req = MatchRequest(db="CONFIG_DB", table="SFLOW", key_pattern="global", just_keys=False) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 @@ -256,36 +175,24 @@ def test_just_keys_false(self): assert not ddiff, ddiff def test_file_source(self): - req = MatchRequest() - req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") - req.table = "COPP_TRAP" - req.field = "trap_ids" - req.value = "arp_req" + file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req = MatchRequest(file=file, table="COPP_TRAP", field="trap_ids", value="arp_req") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "COPP_TRAP|arp" in ret["keys"] def test_file_source_with_key_ptrn(self): - req = MatchRequest() - req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") - req.table = "COPP_GROUP" - req.key_pattern = "queue4*" - req.field = "red_action" - req.value = "drop" + file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop") ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "COPP_GROUP|queue4_group2" in ret["keys"] def test_file_source_with_not_only_return_keys(self): - req = MatchRequest() - req.file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") - req.table = "COPP_GROUP" - req.key_pattern = "queue4*" - req.field = "red_action" - req.value = "drop" - req.just_keys = False + file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop", just_keys=False) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 @@ -295,14 +202,7 @@ def test_file_source_with_not_only_return_keys(self): assert not ddiff, ddiff def test_match_entire_list(self): - req = MatchRequest() - req.db = "CONFIG_DB" - req.table = "PORT" - req.key_pattern = "*" - req.field = "lanes" - req.value = "61,62,63,64" - req.match_entire_list = True - req.just_keys = True + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="lanes", value = "61,62,63,64", match_entire_list=True, just_keys=True) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 From 6da04b4a2aca232542fd8503bf726e855757a2ee Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 9 Jun 2021 08:04:57 +0000 Subject: [PATCH 10/17] Comments addressed Signed-off-by: Vivek Reddy Karri --- dump/helper.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dump/helper.py b/dump/helper.py index 352439f9a2..806f2d0e26 100644 --- a/dump/helper.py +++ b/dump/helper.py @@ -26,10 +26,10 @@ def handle_multiple_keys_matched_error(err_str, key_to_go_with="", excep=False): print("ERROR (AMBIGUITY): {} \n Proceeding with the key {}".format(err_str, key_to_go_with), file = sys.stderr) -def sort_lists(ret): +def sort_lists(ret_template): """ Used to sort the nested list returned by the template dict. """ - for db in ret.keys(): - for key in ret[db].keys(): - if isinstance(ret[db][key], list): - ret[db][key].sort() - return ret + for db in ret_template.keys(): + for key in ret_template[db].keys(): + if isinstance(ret_template[db][key], list): + ret_template[db][key].sort() + return ret_template From 1a08fe8db5cce196eb0b44e506145607c56cff63 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 10 Jun 2021 04:32:47 +0000 Subject: [PATCH 11/17] pylint issues dealt with Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 8 +++----- tests/dump_tests/match_engine_test.py | 23 +++++++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index a5994a41c7..aeefbe061b 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -1,6 +1,6 @@ -import re, json, os, fnmatch -from dump.helper import verbose_print +import json, fnmatch from abc import ABC, abstractmethod +from dump.helper import verbose_print from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig EXCEP_DICT = { @@ -286,6 +286,4 @@ def fetch(self, req): if not filtered_keys: return self.__display_error(EXCEP_DICT["NO_ENTRIES"]) return self.__fill_template(src, req, filtered_keys, template) - - - + diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index 4636e26bb2..ac16c25f44 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -1,11 +1,10 @@ -import json, os, sys -import jsonpatch +import os, sys import unittest import pytest from dump.match_infra import MatchEngine, EXCEP_DICT, MatchRequest from deepdiff import DeepDiff -test_path = os.path.join(os.path.abspath(__file__),"../") +test_path = os.path.join(os.path.abspath(__file__), "../") sys.path.append(test_path) @@ -58,7 +57,7 @@ def test_no_value(self): self.assertRaisesWithMessage(EXCEP_DICT["NO_VALUE"], MatchRequest, db="APPL_DB", table="COPP_TABLE", key_pattern="*", field="trap_ids", value="") def test_no_table(self): - self.assertRaisesWithMessage(EXCEP_DICT["NO_TABLE"], MatchRequest, db="APPL_DB", table="", key_pattern="*", field="trap_ids", value = "bgpv6") + self.assertRaisesWithMessage(EXCEP_DICT["NO_TABLE"], MatchRequest, db="APPL_DB", table="", key_pattern="*", field="trap_ids", value="bgpv6") def test_just_keys_return_fields_compat(self): self.assertRaisesWithMessage(EXCEP_DICT["JUST_KEYS_COMPAT"], MatchRequest, db="APPL_DB", return_fields=["trap_group"], table="COPP_TABLE", @@ -148,14 +147,14 @@ def test_return_fields_with_key_filtering(self): assert "reboot" == ret["return_values"]["REBOOT_CAUSE|2020_10_09_02_33_06"]["cause"] def test_return_fields_with_field_value_filtering(self): - req = MatchRequest(db="STATE_DB", table="CHASSIS_MODULE_TABLE", field="oper_status", value="Offline", return_fields = ["slot"]) + req = MatchRequest(db="STATE_DB", table="CHASSIS_MODULE_TABLE", field="oper_status", value="Offline", return_fields=["slot"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "18" == ret["return_values"]["CHASSIS_MODULE_TABLE|FABRIC-CARD1"]["slot"] def test_return_fields_with_all_filtering(self): - req = MatchRequest(db="STATE_DB", table="VXLAN_TUNNEL_TABLE", key_pattern="EVPN_25.25.25.2*", field="operstatus", value="down", return_fields = ["src_ip"]) + req = MatchRequest(db="STATE_DB", table="VXLAN_TUNNEL_TABLE", key_pattern="EVPN_25.25.25.2*", field="operstatus", value="down", return_fields=["src_ip"]) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 3 @@ -170,12 +169,12 @@ def test_just_keys_false(self): assert len(ret["keys"]) == 1 recv_dict = ret["keys"][0] assert isinstance(recv_dict, dict) - exp_dict = {"SFLOW|global": {"admin_state": "up", "polling_interval": "0"}} + exp_dict = {"SFLOW|global": {"admin_state": "up", "polling_interval": "0"}} ddiff = DeepDiff(exp_dict, recv_dict) assert not ddiff, ddiff def test_file_source(self): - file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") req = MatchRequest(file=file, table="COPP_TRAP", field="trap_ids", value="arp_req") ret = self.match_engine.fetch(req) assert ret["error"] == "" @@ -183,7 +182,7 @@ def test_file_source(self): assert "COPP_TRAP|arp" in ret["keys"] def test_file_source_with_key_ptrn(self): - file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop") ret = self.match_engine.fetch(req) assert ret["error"] == "" @@ -191,18 +190,18 @@ def test_file_source_with_key_ptrn(self): assert "COPP_GROUP|queue4_group2" in ret["keys"] def test_file_source_with_not_only_return_keys(self): - file = os.path.join(os.path.dirname(__file__),"files/copp_cfg.json") + file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop", just_keys=False) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 recv_dict = ret["keys"][0] - exp_dict = {"COPP_GROUP|queue4_group2": {"trap_action": "copy", "trap_priority": "4", "queue": "4", "meter_type": "packets", "mode": "sr_tcm", "cir": "600", "cbs": "600", "red_action": "drop"}} + exp_dict = {"COPP_GROUP|queue4_group2": {"trap_action": "copy", "trap_priority": "4", "queue": "4", "meter_type": "packets", "mode": "sr_tcm", "cir": "600", "cbs": "600", "red_action": "drop"}} ddiff = DeepDiff(exp_dict, recv_dict) assert not ddiff, ddiff def test_match_entire_list(self): - req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="lanes", value = "61,62,63,64", match_entire_list=True, just_keys=True) + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="lanes", value="61,62,63,64", match_entire_list=True, just_keys=True) ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 From 385964802e1bd395f2dde1d663e8058966e47f92 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 10 Jun 2021 15:23:23 +0000 Subject: [PATCH 12/17] Minor Fixes Signed-off-by: Vivek Reddy Karri --- tests/dump_tests/match_engine_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index ac16c25f44..b411173e76 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -205,7 +205,4 @@ def test_match_entire_list(self): ret = self.match_engine.fetch(req) assert ret["error"] == "" assert len(ret["keys"]) == 1 - assert "PORT|Ethernet60" in ret["keys"] - - - \ No newline at end of file + assert "PORT|Ethernet60" in ret["keys"] From 2c73a59a9ff8516256f9ab88ed4e1e33b710acdf Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Thu, 10 Jun 2021 18:29:30 +0000 Subject: [PATCH 13/17] Moved the mock files dir Signed-off-by: Vivek Reddy Karri --- tests/{dump_tests/files => dump_input}/copp_cfg.json | 0 tests/dump_tests/match_engine_test.py | 9 +++++---- 2 files changed, 5 insertions(+), 4 deletions(-) rename tests/{dump_tests/files => dump_input}/copp_cfg.json (100%) diff --git a/tests/dump_tests/files/copp_cfg.json b/tests/dump_input/copp_cfg.json similarity index 100% rename from tests/dump_tests/files/copp_cfg.json rename to tests/dump_input/copp_cfg.json diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index b411173e76..0c8f2c2833 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -4,7 +4,8 @@ from dump.match_infra import MatchEngine, EXCEP_DICT, MatchRequest from deepdiff import DeepDiff -test_path = os.path.join(os.path.abspath(__file__), "../") +test_path = os.path.join(os.path.dirname(__file__), "../") +dump_test_input = os.path.join(test_path, "dump_input") sys.path.append(test_path) @@ -174,7 +175,7 @@ def test_just_keys_false(self): assert not ddiff, ddiff def test_file_source(self): - file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") + file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_TRAP", field="trap_ids", value="arp_req") ret = self.match_engine.fetch(req) assert ret["error"] == "" @@ -182,7 +183,7 @@ def test_file_source(self): assert "COPP_TRAP|arp" in ret["keys"] def test_file_source_with_key_ptrn(self): - file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") + file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop") ret = self.match_engine.fetch(req) assert ret["error"] == "" @@ -190,7 +191,7 @@ def test_file_source_with_key_ptrn(self): assert "COPP_GROUP|queue4_group2" in ret["keys"] def test_file_source_with_not_only_return_keys(self): - file = os.path.join(os.path.dirname(__file__), "files/copp_cfg.json") + file = os.path.join(dump_test_input, "copp_cfg.json") req = MatchRequest(file=file, table="COPP_GROUP", key_pattern="queue4*", field="red_action", value="drop", just_keys=False) ret = self.match_engine.fetch(req) assert ret["error"] == "" From 30528b440e7c4315b712c6fd8b9f27eb468f925c Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Tue, 22 Jun 2021 23:12:28 +0000 Subject: [PATCH 14/17] Namespace changes updated Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 28 ++++++++++++------ tests/dump_tests/match_engine_test.py | 41 ++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index aeefbe061b..f3cf8826ee 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from dump.helper import verbose_print from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig +from sonic_py_common import multi_asic EXCEP_DICT = { "INV_REQ": "Argument should be of type MatchRequest", @@ -16,7 +17,8 @@ "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty", "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type", "NO_ENTRIES": "No Keys found after applying the filtering criteria", - "FILE_R_EXEP": "Exception Caught While Reading the json cfg file provided" + "FILE_R_EXEP": "Exception Caught While Reading the json cfg file provided", + "INV_NS": "Namespace is invalid" } class MatchRequest: @@ -85,6 +87,9 @@ def __static_checks(self): if self.field and not self.value: return EXCEP_DICT["NO_VALUE"] + if self.ns not in multi_asic.get_namespace_list(): + return EXCEP_DICT["INV_NS"] + verbose_print("MatchRequest Checks Passed") return "" @@ -141,7 +146,7 @@ def hget(self, db, key, field): return "" @abstractmethod - def sep(self, db): + def get_separator(self, db): return "" class RedisSource(SourceAdapter): @@ -152,18 +157,23 @@ def __init__(self): def connect(self, db, ns): try: - self.conn = SonicV2Connector(namespace=ns, host="127.0.0.1") + if not SonicDBConfig.isInit(): + if multi_asic.is_multi_asic(): + SonicDBConfig.load_sonic_global_db_config() + else: + SonicDBConfig.load_sonic_db_config() + self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True) #host="127.0.0.1" self.conn.connect(db) except Exception as e: verbose_print("RedisSource: Connection Failed\n" + str(e)) return False return True - def sep(self, db): + def get_separator(self, db): return self.conn.get_db_separator(db) def getKeys(self, db, table, key_pattern): - return self.conn.keys(db, table + self.sep(db) + key_pattern) + return self.conn.keys(db, table + self.get_separator(db) + key_pattern) def get(self, db, key): return self.conn.get_all(db, key) @@ -186,7 +196,7 @@ def connect(self, db, ns): return False return True - def sep(self, db): + def get_separator(self, db): return SonicDBConfig.getSeparator("CONFIG_DB") def getKeys(self, db, table, key_pattern): @@ -195,15 +205,15 @@ def getKeys(self, db, table, key_pattern): # https://docs.python.org/3.7/library/fnmatch.html kp = key_pattern.replace("[^", "[!") kys = fnmatch.filter(self.json_data[table].keys(), kp) - return [table+self.sep(db)+ky for ky in kys] + return [table + self.get_separator(db) + ky for ky in kys] def get(self, db, key): - sep = self.sep(db) + sep = self.get_separator(db) table, key = key.split(sep, 1) return self.json_data.get(table, {}).get(key, {}) def hget(self, db, key, field): - sep = self.sep(db) + sep = self.get_separator(db) table, key = key.split(sep, 1) return self.json_data.get(table, "").get(key, "").get(field, "") diff --git a/tests/dump_tests/match_engine_test.py b/tests/dump_tests/match_engine_test.py index 0c8f2c2833..a4d4330b9b 100644 --- a/tests/dump_tests/match_engine_test.py +++ b/tests/dump_tests/match_engine_test.py @@ -3,6 +3,7 @@ import pytest from dump.match_infra import MatchEngine, EXCEP_DICT, MatchRequest from deepdiff import DeepDiff +from importlib import reload test_path = os.path.join(os.path.dirname(__file__), "../") dump_test_input = os.path.join(test_path, "dump_input") @@ -48,7 +49,11 @@ def test_no_file(self): def test_invalid_db(self): self.assertRaisesWithMessage(EXCEP_DICT["INV_DB"], MatchRequest, db="CONFIGURATION_DB") - + + def test_invalid_namespace(self): + self.assertRaisesWithMessage(EXCEP_DICT["INV_NS"], MatchRequest, db="APPL_DB", table="PORT_TABLE", + field="lanes", value="202", ns="asic4") + def test_bad_key_pattern(self): req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="") ret = self.match_engine.fetch(req) @@ -207,3 +212,37 @@ def test_match_entire_list(self): assert ret["error"] == "" assert len(ret["keys"]) == 1 assert "PORT|Ethernet60" in ret["keys"] + + +class TestNonDefaultNameSpace(unittest.TestCase): + + @classmethod + def setup_class(cls): + print("SETUP") + os.environ["UTILITIES_UNIT_TESTING"] = "2" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "multi_asic" + from ..mock_tables import mock_multi_asic + reload(mock_multi_asic) + from ..mock_tables import dbconnector + dbconnector.load_namespace_config() + + def teardown_class(cls): + print("TEARDOWN") + os.environ["UTILITIES_UNIT_TESTING"] = "0" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "" + + def test_namespace_asic0(self): + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="*", field="asic_port_name", value="Eth0-ASIC0", ns="asic0") + match_engine = MatchEngine() + ret = match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "PORT|Ethernet0" in ret["keys"] + + def test_namespace_asic1(self): + req = MatchRequest(db="CONFIG_DB", table="PORT", key_pattern="Ethernet-BP256", ns="asic1") + match_engine = MatchEngine() + ret = match_engine.fetch(req) + assert ret["error"] == "" + assert len(ret["keys"]) == 1 + assert "PORT|Ethernet-BP256" in ret["keys"] From 7eded22d85a26f624bb7427de6d690e14aea5de9 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 23 Jun 2021 00:38:53 +0000 Subject: [PATCH 15/17] Multi-Asic Changes Made Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index f3cf8826ee..81cde1e084 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -2,7 +2,8 @@ from abc import ABC, abstractmethod from dump.helper import verbose_print from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig -from sonic_py_common import multi_asic +from utilities_common.multi_asic import multi_asic_ns_choices, multi_asic +from utilities_common.constants import DEFAULT_NAMESPACE EXCEP_DICT = { "INV_REQ": "Argument should be of type MatchRequest", @@ -87,8 +88,8 @@ def __static_checks(self): if self.field and not self.value: return EXCEP_DICT["NO_VALUE"] - if self.ns not in multi_asic.get_namespace_list(): - return EXCEP_DICT["INV_NS"] + if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic_ns_choices(): + return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic_ns_choices()) verbose_print("MatchRequest Checks Passed") @@ -115,11 +116,11 @@ def __str__(self): if len(self.return_fields) > 0: str += "return_fields: " + ",".join(self.return_fields) + " " if self.ns: - str += "namespace: ," + self.ns + str += "namespace: , " + self.ns if self.match_entire_list: - str += "match_list: True ," + str += "match_list: True , " else: - str += "match_list: False ," + str += "match_list: False , " return str class SourceAdapter(ABC): From 0a4765d6a9dd4c0586f5bbd5d278f865dbcc369e Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Wed, 23 Jun 2021 00:41:58 +0000 Subject: [PATCH 16/17] Minor Changes Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index 81cde1e084..216917342f 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -163,7 +163,7 @@ def connect(self, db, ns): SonicDBConfig.load_sonic_global_db_config() else: SonicDBConfig.load_sonic_db_config() - self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True) #host="127.0.0.1" + self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True) self.conn.connect(db) except Exception as e: verbose_print("RedisSource: Connection Failed\n" + str(e)) From 0cdc8651dee983b919a0e1ac4c1cceeb4a718a7b Mon Sep 17 00:00:00 2001 From: Vivek Reddy Karri Date: Fri, 25 Jun 2021 01:09:16 +0000 Subject: [PATCH 17/17] moved from utilities common to sonic_py_common Signed-off-by: Vivek Reddy Karri --- dump/match_infra.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dump/match_infra.py b/dump/match_infra.py index 216917342f..fe0d8126a9 100644 --- a/dump/match_infra.py +++ b/dump/match_infra.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from dump.helper import verbose_print from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig -from utilities_common.multi_asic import multi_asic_ns_choices, multi_asic +from sonic_py_common import multi_asic from utilities_common.constants import DEFAULT_NAMESPACE EXCEP_DICT = { @@ -88,8 +88,8 @@ def __static_checks(self): if self.field and not self.value: return EXCEP_DICT["NO_VALUE"] - if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic_ns_choices(): - return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic_ns_choices()) + if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic.get_namespace_list(): + return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic.get_namespace_list()) verbose_print("MatchRequest Checks Passed")