From f341028afafa85f631e983c3281ac9e99c513d33 Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 14 Jun 2022 08:48:59 +0000 Subject: [PATCH 1/6] Update db_migrator to support PORT_QOS_MAP|global Signed-off-by: bingwang --- scripts/db_migrator.py | 16 +++++++++++++ .../qos_map_table_global_expected.json | 23 +++++++++++++++++++ .../config_db/qos_map_table_global_input.json | 21 +++++++++++++++++ tests/db_migrator_test.py | 23 +++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 tests/db_migrator_input/config_db/qos_map_table_global_expected.json create mode 100644 tests/db_migrator_input/config_db/qos_map_table_global_input.json diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 38ec5e6531..ddf1e8768b 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -422,6 +422,20 @@ def migrate_pfcwd_sw_enable_table(self): v['pfcwd_sw_enable'] = v['pfc_enable'] self.configDB.set_entry('PORT_QOS_MAP', k, v) + def migrate_port_qos_map_global(self): + """ + Generate dscp_to_tc_map for switch. + """ + dscp_to_tc_map_table_names = self.configDB.get_keys('DSCP_TO_TC_MAP') + if len(dscp_to_tc_map_table_names) == 0: + return True + + qos_maps = self.configDB.get_table('PORT_QOS_MAP') + if 'global' not in qos_maps.keys(): + # We are unlikely to have more than 1 DSCP_TO_TC_MAP in previous versions + self.configDB.set_entry('PORT_QOS_MAP', 'global', dscp_to_tc_map_table_names[0]) + self.log_info("Created entry for global DSCP_TO_TC_MAP {}".format(dscp_to_tc_map_table_names[0])) + def version_unknown(self): """ version_unknown tracks all SONiC versions that doesn't have a version @@ -614,6 +628,8 @@ def common_migration_ops(self): # Migrate pfcwd_sw_enable table self.migrate_pfcwd_sw_enable_table() + # Migrate entry for switch level DSCP_TO_TC_MAP + self.migrate_port_qos_map_global() def migrate(self): version = self.get_version() diff --git a/tests/db_migrator_input/config_db/qos_map_table_global_expected.json b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json new file mode 100644 index 0000000000..ad5057a79f --- /dev/null +++ b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json @@ -0,0 +1,23 @@ +{ + "VERSIONS|DATABASE": { + "VERSION": "version_2_0_0" + }, + "DSCP_TO_TC_MAP": { + "AZURE": { + "0": "0", + "1": "1" + } + }, + "PORT_QOS_MAP": { + "global": { + "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]" + }, + "Ethernet0": { + "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]", + "pfc_enable": "3,4", + "pfc_to_queue_map": "[MAP_PFC_PRIORITY_TO_QUEUE|AZURE]", + "tc_to_pg_map": "[TC_TO_PRIORITY_GROUP_MAP|AZURE]", + "tc_to_queue_map": "[TC_TO_QUEUE_MAP|AZURE]" + } + } +} diff --git a/tests/db_migrator_input/config_db/qos_map_table_global_input.json b/tests/db_migrator_input/config_db/qos_map_table_global_input.json new file mode 100644 index 0000000000..003bd20413 --- /dev/null +++ b/tests/db_migrator_input/config_db/qos_map_table_global_input.json @@ -0,0 +1,21 @@ +{ + "VERSIONS|DATABASE": { + "VERSION": "version_2_0_0" + }, + "DSCP_TO_TC_MAP": { + "AZURE": { + "0": "0", + "1": "1" + } + }, + "PORT_QOS_MAP": { + "Ethernet0": { + "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]", + "pfc_enable": "3,4", + "pfc_to_queue_map": "[MAP_PFC_PRIORITY_TO_QUEUE|AZURE]", + "tc_to_pg_map": "[TC_TO_PRIORITY_GROUP_MAP|AZURE]", + "tc_to_queue_map": "[TC_TO_QUEUE_MAP|AZURE]" + } + } +} + diff --git a/tests/db_migrator_test.py b/tests/db_migrator_test.py index d8a070734c..40984412de 100644 --- a/tests/db_migrator_test.py +++ b/tests/db_migrator_test.py @@ -274,3 +274,26 @@ def test_pfc_enable_migrator(self): diff = DeepDiff(resulting_table, expected_table, ignore_order=True) assert not diff +class TestGlobalDscpToTcMapMigrator(object): + @classmethod + def setup_class(cls): + os.environ['UTILITIES_UNIT_TESTING'] = "2" + + @classmethod + def teardown_class(cls): + os.environ['UTILITIES_UNIT_TESTING'] = "0" + dbconnector.dedicated_dbs['CONFIG_DB'] = None + + def test_global_dscp_to_tc_map_migrator(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'qos_map_table_global_input') + import db_migrator + dbmgtr = db_migrator.DBMigrator(None) + dbmgtr.migrate() + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'qos_map_table_global_expected') + expected_db = Db() + + resulting_table = dbmgtr.configDB.get_table('PORT_QOS_MAP') + expected_table = expected_db.cfgdb.get_table('PORT_QOS_MAP') + + diff = DeepDiff(resulting_table, expected_table, ignore_order=True) + assert not diff \ No newline at end of file From 23b4851d165c70c49d8f5a87f8a590a2c1a57085 Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 14 Jun 2022 08:52:15 +0000 Subject: [PATCH 2/6] Append empty line Signed-off-by: bingwang --- tests/db_migrator_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/db_migrator_test.py b/tests/db_migrator_test.py index 40984412de..264b468976 100644 --- a/tests/db_migrator_test.py +++ b/tests/db_migrator_test.py @@ -296,4 +296,5 @@ def test_global_dscp_to_tc_map_migrator(self): expected_table = expected_db.cfgdb.get_table('PORT_QOS_MAP') diff = DeepDiff(resulting_table, expected_table, ignore_order=True) - assert not diff \ No newline at end of file + assert not diff + \ No newline at end of file From c544f57cb3603d2c03a4861eb96984d223901649 Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 5 Jul 2022 02:25:37 +0000 Subject: [PATCH 3/6] Fix vstest --- scripts/db_migrator.py | 7 +++++-- .../qos_map_table_global_expected.json | 21 +++++-------------- .../config_db/qos_map_table_global_input.json | 17 +++------------ tests/db_migrator_test.py | 6 +++++- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index ddf1e8768b..1cd6b913cc 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -426,6 +426,9 @@ def migrate_port_qos_map_global(self): """ Generate dscp_to_tc_map for switch. """ + asics_require_global_dscp_to_tc_map = ["broadcom"] + if self.asic_type not in asics_require_global_dscp_to_tc_map: + return True dscp_to_tc_map_table_names = self.configDB.get_keys('DSCP_TO_TC_MAP') if len(dscp_to_tc_map_table_names) == 0: return True @@ -433,8 +436,8 @@ def migrate_port_qos_map_global(self): qos_maps = self.configDB.get_table('PORT_QOS_MAP') if 'global' not in qos_maps.keys(): # We are unlikely to have more than 1 DSCP_TO_TC_MAP in previous versions - self.configDB.set_entry('PORT_QOS_MAP', 'global', dscp_to_tc_map_table_names[0]) - self.log_info("Created entry for global DSCP_TO_TC_MAP {}".format(dscp_to_tc_map_table_names[0])) + self.configDB.set_entry('PORT_QOS_MAP', 'global', {"dscp_to_tc_map": "[DSCP_TO_TC_MAP|{}]".format(dscp_to_tc_map_table_names[0])}) + log.log_info("Created entry for global DSCP_TO_TC_MAP {}".format(dscp_to_tc_map_table_names[0])) def version_unknown(self): """ diff --git a/tests/db_migrator_input/config_db/qos_map_table_global_expected.json b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json index ad5057a79f..ece334ae8c 100644 --- a/tests/db_migrator_input/config_db/qos_map_table_global_expected.json +++ b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json @@ -2,22 +2,11 @@ "VERSIONS|DATABASE": { "VERSION": "version_2_0_0" }, - "DSCP_TO_TC_MAP": { - "AZURE": { - "0": "0", - "1": "1" - } + "DSCP_TO_TC_MAP|AZURE": { + "0": "0", + "1": "1" }, - "PORT_QOS_MAP": { - "global": { - "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]" - }, - "Ethernet0": { - "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]", - "pfc_enable": "3,4", - "pfc_to_queue_map": "[MAP_PFC_PRIORITY_TO_QUEUE|AZURE]", - "tc_to_pg_map": "[TC_TO_PRIORITY_GROUP_MAP|AZURE]", - "tc_to_queue_map": "[TC_TO_QUEUE_MAP|AZURE]" - } + "PORT_QOS_MAP|global": { + "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]" } } diff --git a/tests/db_migrator_input/config_db/qos_map_table_global_input.json b/tests/db_migrator_input/config_db/qos_map_table_global_input.json index 003bd20413..dd4b78bb0e 100644 --- a/tests/db_migrator_input/config_db/qos_map_table_global_input.json +++ b/tests/db_migrator_input/config_db/qos_map_table_global_input.json @@ -2,20 +2,9 @@ "VERSIONS|DATABASE": { "VERSION": "version_2_0_0" }, - "DSCP_TO_TC_MAP": { - "AZURE": { - "0": "0", - "1": "1" - } - }, - "PORT_QOS_MAP": { - "Ethernet0": { - "dscp_to_tc_map": "[DSCP_TO_TC_MAP|AZURE]", - "pfc_enable": "3,4", - "pfc_to_queue_map": "[MAP_PFC_PRIORITY_TO_QUEUE|AZURE]", - "tc_to_pg_map": "[TC_TO_PRIORITY_GROUP_MAP|AZURE]", - "tc_to_queue_map": "[TC_TO_QUEUE_MAP|AZURE]" - } + "DSCP_TO_TC_MAP|AZURE": { + "0": "0", + "1": "1" } } diff --git a/tests/db_migrator_test.py b/tests/db_migrator_test.py index 264b468976..8e164d33ea 100644 --- a/tests/db_migrator_test.py +++ b/tests/db_migrator_test.py @@ -25,6 +25,8 @@ def get_sonic_version_info_mlnx(): return {'asic_type': 'mellanox'} +def get_sonic_version_info_brcm(): + return {'asic_type': 'broadcom'} class TestMellanoxBufferMigrator(object): @classmethod @@ -288,6 +290,8 @@ def test_global_dscp_to_tc_map_migrator(self): dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'qos_map_table_global_input') import db_migrator dbmgtr = db_migrator.DBMigrator(None) + dbmgtr.asic_type = "broadcom" + dbmgtr.hwsku = "vs" dbmgtr.migrate() dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'qos_map_table_global_expected') expected_db = Db() @@ -297,4 +301,4 @@ def test_global_dscp_to_tc_map_migrator(self): diff = DeepDiff(resulting_table, expected_table, ignore_order=True) assert not diff - \ No newline at end of file + From 2de6adb9924caa2da1eb3bdd8f137a162baf9773 Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 5 Jul 2022 02:31:06 +0000 Subject: [PATCH 4/6] Rm unused function --- tests/db_migrator_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/db_migrator_test.py b/tests/db_migrator_test.py index 8e164d33ea..412f7f30a3 100644 --- a/tests/db_migrator_test.py +++ b/tests/db_migrator_test.py @@ -25,8 +25,6 @@ def get_sonic_version_info_mlnx(): return {'asic_type': 'mellanox'} -def get_sonic_version_info_brcm(): - return {'asic_type': 'broadcom'} class TestMellanoxBufferMigrator(object): @classmethod From 34740d5fa0a6b64d3b2c41b943f804434e4d4fe6 Mon Sep 17 00:00:00 2001 From: bingwang Date: Fri, 15 Jul 2022 07:49:32 +0000 Subject: [PATCH 5/6] Address comments --- scripts/db_migrator.py | 4 ++-- tests/db_migrator_test.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 1cd6b913cc..07809f909a 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -428,10 +428,10 @@ def migrate_port_qos_map_global(self): """ asics_require_global_dscp_to_tc_map = ["broadcom"] if self.asic_type not in asics_require_global_dscp_to_tc_map: - return True + return dscp_to_tc_map_table_names = self.configDB.get_keys('DSCP_TO_TC_MAP') if len(dscp_to_tc_map_table_names) == 0: - return True + return qos_maps = self.configDB.get_table('PORT_QOS_MAP') if 'global' not in qos_maps.keys(): diff --git a/tests/db_migrator_test.py b/tests/db_migrator_test.py index 412f7f30a3..dc02432319 100644 --- a/tests/db_migrator_test.py +++ b/tests/db_migrator_test.py @@ -300,3 +300,12 @@ def test_global_dscp_to_tc_map_migrator(self): diff = DeepDiff(resulting_table, expected_table, ignore_order=True) assert not diff + # Check port_qos_map|global is not generated on mellanox asic + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'qos_map_table_global_input') + dbmgtr_mlnx = db_migrator.DBMigrator(None) + dbmgtr_mlnx.asic_type = "mellanox" + dbmgtr_mlnx.hwsku = "vs" + dbmgtr_mlnx.migrate() + resulting_table = dbmgtr_mlnx.configDB.get_table('PORT_QOS_MAP') + assert resulting_table == {} + From 1e39d5762f41c251847264484fe0f1ee3b1c124d Mon Sep 17 00:00:00 2001 From: bingwang Date: Thu, 21 Jul 2022 09:05:40 +0000 Subject: [PATCH 6/6] Migrate on version --- scripts/db_migrator.py | 14 ++++++++++---- .../config_db/qos_map_table_global_expected.json | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 07809f909a..9389f3749e 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -45,7 +45,7 @@ def __init__(self, namespace, socket=None): none-zero values. build: sequentially increase within a minor version domain. """ - self.CURRENT_VERSION = 'version_2_0_0' + self.CURRENT_VERSION = 'version_2_0_1' self.TABLE_NAME = 'VERSIONS' self.TABLE_KEY = 'DATABASE' @@ -575,9 +575,17 @@ def version_1_0_6(self): def version_2_0_0(self): """ - Current latest version. Nothing to do here. + Version 2_0_0. """ log.log_info('Handling version_2_0_0') + self.migrate_port_qos_map_global() + return 'version_2_0_1' + + def version_2_0_1(self): + """ + Current latest version. Nothing to do here. + """ + log.log_info('Handling version_2_0_1') return None def get_version(self): @@ -631,8 +639,6 @@ def common_migration_ops(self): # Migrate pfcwd_sw_enable table self.migrate_pfcwd_sw_enable_table() - # Migrate entry for switch level DSCP_TO_TC_MAP - self.migrate_port_qos_map_global() def migrate(self): version = self.get_version() diff --git a/tests/db_migrator_input/config_db/qos_map_table_global_expected.json b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json index ece334ae8c..346cad34fa 100644 --- a/tests/db_migrator_input/config_db/qos_map_table_global_expected.json +++ b/tests/db_migrator_input/config_db/qos_map_table_global_expected.json @@ -1,6 +1,6 @@ { "VERSIONS|DATABASE": { - "VERSION": "version_2_0_0" + "VERSION": "version_2_0_1" }, "DSCP_TO_TC_MAP|AZURE": { "0": "0",