[201911][db_migrator] fix old 1911 feature config migration to a new schema

scripts/db_migrator.py

@@ -9,18 +9,20 @@
 from sonic_py_common import device_info, logger
 from swsssdk import ConfigDBConnector, SonicDBConfig, SonicV2Connector
 
+INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
+
 # mock the redis for unit test purposes #
 try:
     if os.environ["UTILITIES_UNIT_TESTING"] == "2":
         modules_path = os.path.join(os.path.dirname(__file__), "..")
-        tests_path = os.path.join(modules_path, "tests")
+        tests_path = os.path.join(modules_path, "sonic-utilities-tests")
         mocked_db_path = os.path.join(tests_path, "db_migrator_input")
         sys.path.insert(0, modules_path)
         sys.path.insert(0, tests_path)
+        INIT_CFG_FILE = os.path.join(mocked_db_path, "init_cfg.json")
 except KeyError:
     pass
 
-INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
 SYSLOG_IDENTIFIER = 'db_migrator'
 
 # Global logger instance
@@ -154,6 +156,22 @@ def migrate_intf_table(self):
             self.appDB.set(self.appDB.APPL_DB, table, 'NULL', 'NULL')
             if_db.append(if_name)
 
+    def migrate_feature_table(self):
+        '''
+        Combine CONTAINER_FEATURE and FEATURE tables into FEATURE table.
+        '''
+        feature_table = self.configDB.get_table('FEATURE')
+        for feature, config in feature_table.items():
+            if 'status' in config:
+                state = config.pop('status', 'disabled')
+                config['state'] = state
+                self.configDB.set_entry('FEATURE', feature, config)
+
+        container_feature_table = self.configDB.get_table('CONTAINER_FEATURE')
+        for feature, config in container_feature_table.items():
+            self.configDB.mod_entry('FEATURE', feature, config)
+            self.configDB.set_entry('CONTAINER_FEATURE', feature, None)
+
     def version_unknown(self):
         """
         version_unknown tracks all SONiC versions that doesn't have a version
@@ -207,6 +225,8 @@ def version_1_0_3(self):
         """
         log.log_info('Handling version_1_0_3')
 
+        self.migrate_feature_table()
+
         # Check ASIC type, if Mellanox platform then need DB migration
         if self.asic_type == "mellanox":
             if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_3', 'version_1_0_4') and self.mellanox_buffer_migrator.mlnx_migrate_buffer_profile('version_1_0_3', 'version_1_0_4'):
@@ -246,15 +266,13 @@ def get_version(self):
 
         return 'version_unknown'
 
-
     def set_version(self, version=None):
         if not version:
             version = self.CURRENT_VERSION
         log.log_info('Setting version to ' + version)
         entry = { self.TABLE_FIELD : version }
         self.configDB.set_entry(self.TABLE_NAME, self.TABLE_KEY, entry)
 
-
     def common_migration_ops(self):
         try:
             with open(INIT_CFG_FILE) as f:
@@ -263,15 +281,17 @@ def common_migration_ops(self):
             raise Exception(str(e))
 
         for init_cfg_table, table_val in init_db.items():
-            data = self.configDB.get_table(init_cfg_table)
-            if data:
-                # Ignore overriding the values that pre-exist in configDB
-                continue
             log.log_info("Migrating table {} from INIT_CFG to config_db".format(init_cfg_table))
-            # Update all tables that do not exist in configDB but are present in INIT_CFG
-            for init_table_key, init_table_val in table_val.items():
-                self.configDB.set_entry(init_cfg_table, init_table_key, init_table_val)
-
+            for key in table_val:
+                curr_cfg = self.configDB.get_entry(init_cfg_table, key)
+                init_cfg = table_val[key]
+
+                # Override init config with current config.
+                # This will leave new fields from init_config
+                # in new_config, but not override existing configuration.
+                new_cfg = init_cfg.copy()
+                new_cfg.update(curr_cfg)
+                self.configDB.set_entry(init_cfg_table, key, new_cfg)
 
     def migrate(self):
         version = self.get_version()

setup.py

@@ -58,6 +58,7 @@
         'show': ['aliases.ini'],
         'sonic-utilities-tests': ['acl_input/*',
                                   'db_migrator_input/config_db/*.json',
+                                  'db_migrator_input/init_cfg.json',
                                   'mock_tables/*.py',
                                   'mock_tables/*.json',
                                   'mock_tables/asic0/*',

sonic-utilities-tests/db_migrator_input/config_db/feature-expected.json

@@ -0,0 +1,26 @@
+{
+    "FEATURE|swss": {
+        "auto_restart": "disabled",
+        "has_global_scope": "False",
+        "has_per_asic_scope": "True",
+        "has_timer": "False",
+        "high_mem_alert": "disabled",
+        "state": "enabled"
+    },
+    "FEATURE|syncd": {
+        "auto_restart": "enabled",
+        "has_global_scope": "False",
+        "has_per_asic_scope": "True",
+        "has_timer": "False",
+        "high_mem_alert": "disabled",
+        "state": "enabled"
+    },
+    "FEATURE|telemetry": {
+        "auto_restart": "enabled",
+        "has_global_scope": "False",
+        "has_per_asic_scope": "True",
+        "has_timer": "False",
+        "high_mem_alert": "disabled",
+        "state": "enabled"
+    }
+}

sonic-utilities-tests/db_migrator_input/config_db/feature-input.json

@@ -0,0 +1,16 @@
+{
+    "CONTAINER_FEATURE|swss": {
+        "auto_restart": "disabled",
+        "high_mem_alert": "disabled"
+    },
+    "CONTAINER_FEATURE|telemetry": {
+        "auto_restart": "enabled",
+        "high_mem_alert": "disabled"
+    },
+    "FEATURE|telemetry": {
+        "status": "enabled"
+    },
+    "FEATURE|syncd": {
+        "state": "enabled"
+    }
+}

sonic-utilities-tests/db_migrator_input/init_cfg.json

@@ -0,0 +1,28 @@
+{
+    "FEATURE": {
+        "swss": {
+            "auto_restart": "enabled",
+            "has_global_scope": "False",
+            "has_per_asic_scope": "True",
+            "has_timer": "False",
+            "high_mem_alert": "disabled",
+            "state": "enabled"
+        },
+        "syncd": {
+            "auto_restart": "enabled",
+            "has_global_scope": "False",
+            "has_per_asic_scope": "True",
+            "has_timer": "False",
+            "high_mem_alert": "disabled",
+            "state": "enabled"
+        },
+        "telemetry": {
+            "auto_restart": "disabled",
+            "has_global_scope": "False",
+            "has_per_asic_scope": "True",
+            "has_timer": "False",
+            "high_mem_alert": "disabled",
+            "state": "disabled"
+        }
+    }
+}

sonic-utilities-tests/db_migrator_test.py

@@ -3,6 +3,8 @@
 import pytest
 import sys
 
+from deepdiff import DeepDiff
+
 from sonic_py_common import device_info
 
 import mock_tables.dbconnector
@@ -107,3 +109,37 @@ def test_mellanox_buffer_migrator_for_cold_reboot(self, sku_version, topo):
                 assert dbmgtr.mellanox_buffer_migrator.is_buffer_config_default
 
         self.clear_dedicated_mock_dbs()
+
+
+class TestInitConfigMigrator(object):
+    @classmethod
+    def setup_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "2"
+
+    @classmethod
+    def teardown_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "0"
+        mock_tables.dbconnector.dedicated_dbs['CONFIG_DB'] = None
+
+    def mock_dedicated_config_db(self, filename):
+        jsonfile = os.path.join(mock_db_path, 'config_db', filename)
+        mock_tables.dbconnector.dedicated_dbs['CONFIG_DB'] = jsonfile
+        db = Db()
+        return db
+
+    def test_init_config_feature_migration(self):
+        self.mock_dedicated_config_db('feature-input')
+
+        import db_migrator
+        dbmgtr = db_migrator.DBMigrator(None)
+        dbmgtr.migrate()
+        expected_db = self.mock_dedicated_config_db('feature-expected')
+
+        resulting_table = dbmgtr.configDB.get_table('FEATURE')
+        expected_table = expected_db.cfgdb.get_table('FEATURE')
+
+        diff = DeepDiff(resulting_table, expected_table, ignore_order=True)
+        assert not diff
+
+        assert not expected_db.cfgdb.get_table('CONTAINER_FEATURE')
+