Merge branch 'sonic-net:master' into asic-sdk-health-event

Author: stephenxs

generic_config_updater/gcu_field_operation_validators.conf.json

@@ -17,10 +17,11 @@
     "helper_data": {
         "rdma_config_update_validator": {
             "mellanox_asics": {
-                "spc1": [ "ACS-MSN2700", "ACS-MSN2740", "ACS-MSN2100", "ACS-MSN2410", "ACS-MSN2010", "Mellanox-SN2700", "Mellanox-SN2700-D48C8",
+                "spc1": [ "ACS-MSN2700", "ACS-MSN2740", "ACS-MSN2100", "ACS-MSN2410", "ACS-MSN2010", "Mellanox-SN2700", "Mellanox-SN2700-C28D8", "Mellanox-SN2700-D40C8S8", "Mellanox-SN2700-D44C10", "Mellanox-SN2700-D48C8",
                           "ACS-MSN2700-A1", "Mellanox-SN2700-A1", "Mellanox-SN2700-A1-C28D8", "Mellanox-SN2700-A1-D40C8S8", "Mellanox-SN2700-A1-D44C10", "Mellanox-SN2700-A1-D48C8" ],
-                "spc2": [ "ACS-MSN3800", "Mellanox-SN3800-D112C8" ],
-                "spc3": [ "ACS-MSN4700", "ACS-MSN4600", "ACS-MSN4600C", "ACS-MSN4410", "Mellanox-SN4600C-D112C8", "Mellanox-SN4600C-C64", "Mellanox-SN4700-O8C48" ],
+                "spc2": [ "ACS-MSN3800", "Mellanox-SN3800-D112C8", "ACS-MSN3420", "ACS-MSN3700C", "ACS-MSN3700", "Mellanox-SN3800-C64", "Mellanox-SN3800-D100C12S2", "Mellanox-SN3800-D24C52", "Mellanox-SN3800-D28C49S1", "Mellanox-SN3800-D28C50" ],
+                "spc3": [ "ACS-MSN4700", "ACS-MSN4600", "ACS-MSN4600C", "ACS-MSN4410", "Mellanox-SN4600C-D112C8", "Mellanox-SN4600C-C64", "Mellanox-SN4700-O8C48", "Mellanox-SN4600C-D100C12S2", "Mellanox-SN4600C-D48C40",
+                          "Mellanox-SN4700-A96C8V8", "Mellanox-SN4700-C128", "Mellanox-SN4700-O28", "Mellanox-SN4700-O8V48", "Mellanox-SN4700-V48C32"],
                 "spc4": [ "ACS-SN5600"]
             },
             "broadcom_asics": {

scripts/db_migrator.py

@@ -618,6 +618,30 @@ def migrate_telemetry(self):
         if not certs:
             self.configDB.set_entry("TELEMETRY", "certs", telemetry_data.get("certs"))
 
+    def migrate_gnmi(self):
+        # If there's GNMI table in CONFIG_DB, no need to migrate
+        gnmi = self.configDB.get_entry('GNMI', 'gnmi')
+        certs = self.configDB.get_entry('GNMI', 'certs')
+        if gnmi and certs:
+            return
+        if self.config_src_data:
+            if 'GNMI' in self.config_src_data:
+                # If there's GNMI in minigraph or golden config, copy configuration from config_src_data
+                gnmi_data = self.config_src_data['GNMI']
+                log.log_notice('Migrate GNMI configuration')
+                if 'gnmi' in gnmi_data:
+                    self.configDB.set_entry("GNMI", "gnmi", gnmi_data.get('gnmi'))
+                if 'certs' in gnmi_data:
+                    self.configDB.set_entry("GNMI", "certs", gnmi_data.get('certs'))
+        else:
+            # If there's no minigraph or golden config, copy configuration from CONFIG_DB TELEMETRY table
+            gnmi = self.configDB.get_entry('TELEMETRY', 'gnmi')
+            if gnmi:
+                self.configDB.set_entry("GNMI", "gnmi", gnmi)
+            certs = self.configDB.get_entry('TELEMETRY', 'certs')
+            if certs:
+                self.configDB.set_entry("GNMI", "certs", certs)
+
     def migrate_console_switch(self):
         # CONSOLE_SWITCH - add missing key
         if not self.config_src_data or 'CONSOLE_SWITCH' not in self.config_src_data:
@@ -1137,9 +1161,20 @@ def version_202311_01(self):
     def version_202311_02(self):
         """
         Version 202311_02.
-        This is current last erversion for 202311 branch
         """
         log.log_info('Handling version_202311_02')
+        # Update GNMI table
+        self.migrate_gnmi()
+
+        self.set_version('version_202311_03')
+        return 'version_202311_03'
+
+    def version_202311_03(self):
+        """
+        Version 202311_03.
+        This is current last erversion for 202311 branch
+        """
+        log.log_info('Handling version_202311_03')
         self.set_version('version_202405_01')
         return 'version_202405_01'
 

scripts/route_check.py

@@ -548,6 +548,10 @@ def check_frr_pending_routes():
                 if entry['vrfName'] != 'default':
                     continue
 
+                # skip if this bgp source prefix is not selected as best
+                if not entry.get('selected', False):
+                    continue
+
                 if not entry.get('offloaded', False):
                     missed_rt.append(entry)
 

tests/db_migrator_input/config_db/dns-nameserver-expected.json

@@ -6,6 +6,6 @@
         "state": "enabled"
     },
     "VERSIONS|DATABASE": {
-        "VERSION": "version_4_0_4"
+        "VERSION": "version_202405_01"
     }
 }

tests/db_migrator_input/config_db/gnmi-configdb-expected.json

@@ -0,0 +1,15 @@
+{
+    "GNMI|gnmi": {
+        "client_auth": "true", 
+        "log_level": "2", 
+        "port": "50051"
+    }, 
+    "GNMI|certs": {
+        "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key", 
+        "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", 
+        "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
+    },
+    "VERSIONS|DATABASE": {
+        "VERSION": "version_202405_01"
+    }
+}

tests/db_migrator_input/config_db/gnmi-input.json

@@ -0,0 +1,15 @@
+{
+    "TELEMETRY|gnmi": {
+        "client_auth": "true", 
+        "log_level": "2", 
+        "port": "50051"
+    }, 
+    "TELEMETRY|certs": {
+        "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key", 
+        "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", 
+        "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
+    },
+    "VERSIONS|DATABASE": {
+        "VERSION": "version_202311_01"
+    }
+}

tests/db_migrator_input/config_db/gnmi-minigraph-expected.json

@@ -0,0 +1,15 @@
+{
+    "GNMI|gnmi": {
+        "client_auth": "true", 
+        "log_level": "2", 
+        "port": "50052"
+    }, 
+    "GNMI|certs": {
+        "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key", 
+        "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", 
+        "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
+    },
+    "VERSIONS|DATABASE": {
+        "VERSION": "version_202405_01"
+    }
+}

tests/db_migrator_test.py

@@ -379,7 +379,7 @@ def test_dns_nameserver_migrator(self):
         dbmgtr.migrate()
         dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'dns-nameserver-expected')
         expected_db = Db()
-        advance_version_for_expected_database(dbmgtr.configDB, expected_db.cfgdb, 'version_4_0_4')
+        advance_version_for_expected_database(dbmgtr.configDB, expected_db.cfgdb, 'version_202405_01')
         resulting_keys = dbmgtr.configDB.keys(dbmgtr.configDB.CONFIG_DB, 'DNS_NAMESERVER*')
         expected_keys = expected_db.cfgdb.keys(expected_db.cfgdb.CONFIG_DB, 'DNS_NAMESERVER*')
 
@@ -767,6 +767,7 @@ def test_fast_reboot_upgrade_to_4_0_3(self):
         assert not self.check_config_db(dbmgtr.configDB, expected_db.cfgdb)
         assert dbmgtr.CURRENT_VERSION == expected_db.cfgdb.get_entry('VERSIONS', 'DATABASE')['VERSION'], '{} {}'.format(dbmgtr.CURRENT_VERSION, dbmgtr.get_version())
 
+
 class TestSflowSampleDirectionMigrator(object):
     @classmethod
     def setup_class(cls):
@@ -885,3 +886,59 @@ def test_init(self, mock_args):
         mock_args.return_value=argparse.Namespace(namespace=None, operation='get_version', socket=None)
         import db_migrator
         db_migrator.main()
+
+
+class TestGNMIMigrator(object):
+    @classmethod
+    def setup_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "2"
+
+    @classmethod
+    def teardown_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "0"
+        dbconnector.dedicated_dbs['CONFIG_DB'] = None
+
+    def test_dns_nameserver_migrator_minigraph(self):
+        dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'gnmi-input')
+        import db_migrator
+        dbmgtr = db_migrator.DBMigrator(None)
+        # Set config_src_data
+        dbmgtr.config_src_data = {
+            'GNMI': {
+                'gnmi': {
+                    "client_auth": "true", 
+                    "log_level": "2", 
+                    "port": "50052"
+                }, 
+                'certs': {
+                    "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key", 
+                    "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", 
+                    "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
+                }
+            }
+        }
+        dbmgtr.migrate()
+        dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'gnmi-minigraph-expected')
+        expected_db = Db()
+        advance_version_for_expected_database(dbmgtr.configDB, expected_db.cfgdb, 'version_202405_01')
+        resulting_table = dbmgtr.configDB.get_table("GNMI")
+        expected_table = expected_db.cfgdb.get_table("GNMI")
+
+        diff = DeepDiff(resulting_table, expected_table, ignore_order=True)
+        assert not diff
+
+    def test_dns_nameserver_migrator_configdb(self):
+        dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'gnmi-input')
+        import db_migrator
+        dbmgtr = db_migrator.DBMigrator(None)
+        # Set config_src_data
+        dbmgtr.config_src_data = {}
+        dbmgtr.migrate()
+        dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'gnmi-configdb-expected')
+        expected_db = Db()
+        advance_version_for_expected_database(dbmgtr.configDB, expected_db.cfgdb, 'version_202405_01')
+        resulting_table = dbmgtr.configDB.get_table("GNMI")
+        expected_table = expected_db.cfgdb.get_table("GNMI")
+
+        diff = DeepDiff(resulting_table, expected_table, ignore_order=True)
+        assert not diff

tests/route_check_test_data.py

@@ -394,20 +394,23 @@
                     "prefix": "0.0.0.0/0",
                     "vrfName": "default",
                     "protocol": "bgp",
-                    "offloaded": "true",
+                    "selected": True,
+                    "offloaded": True,
                 },
             ],
             "10.10.196.12/31": [
                 {
                     "prefix": "10.10.196.12/31",
                     "vrfName": "default",
                     "protocol": "bgp",
-                    "offloaded": "true",
+                    "selected": True,
+                    "offloaded": True,
                 },
             ],
             "10.10.196.24/31": [
                 {
                     "protocol": "connected",
+                    "selected": True,
                 },
             ],
         },
@@ -442,30 +445,82 @@
                     "prefix": "0.0.0.0/0",
                     "vrfName": "default",
                     "protocol": "bgp",
-                    "offloaded": "true",
+                    "selected": True,
+                    "offloaded": True,
                 },
             ],
             "10.10.196.12/31": [
                 {
                     "prefix": "10.10.196.12/31",
                     "vrfName": "default",
                     "protocol": "bgp",
+                    "selected": True,
                 },
             ],
             "10.10.196.24/31": [
                 {
                     "protocol": "connected",
+                    "selected": True,
                 },
             ],
         },
         RESULT: {
             "missed_FRR_routes": [
-                {"prefix": "10.10.196.12/31", "vrfName": "default", "protocol": "bgp"}
+                {"prefix": "10.10.196.12/31", "vrfName": "default", "protocol": "bgp", "selected": True}
             ],
         },
         RET: -1,
     },
-    "10": {
+    "12": {
+        DESCR: "skip bgp routes offloaded check, if not selected as best",
+        ARGS: "route_check -m INFO -i 1000",
+        PRE: {
+            APPL_DB: {
+                ROUTE_TABLE: {
+                    "0.0.0.0/0" : { "ifname": "portchannel0" },
+                    "10.10.196.12/31" : { "ifname": "portchannel0" },
+                },
+                INTF_TABLE: {
+                    "PortChannel1013:10.10.196.24/31": {},
+                    "PortChannel1023:2603:10b0:503:df4::5d/126": {},
+                    "PortChannel1024": {}
+                }
+            },
+            ASIC_DB: {
+                RT_ENTRY_TABLE: {
+                    RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + RT_ENTRY_KEY_SUFFIX: {},
+                    RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + RT_ENTRY_KEY_SUFFIX: {},
+                    RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + RT_ENTRY_KEY_SUFFIX: {},
+                    RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + RT_ENTRY_KEY_SUFFIX: {}
+                }
+            },
+        },
+        FRR_ROUTES: {
+            "0.0.0.0/0": [
+                {
+                    "prefix": "0.0.0.0/0",
+                    "vrfName": "default",
+                    "protocol": "bgp",
+                    "selected": True,
+                    "offloaded": True,
+                },
+            ],
+            "10.10.196.12/31": [
+                {
+                    "prefix": "10.10.196.12/31",
+                    "vrfName": "default",
+                    "protocol": "bgp",
+                },
+            ],
+            "10.10.196.24/31": [
+                {
+                    "protocol": "connected",
+                    "selected": True,
+                },
+            ],
+        },
+    },
+    "13": {
         DESCR: "basic good one with IPv6 address",
         ARGS: "route_check -m INFO -i 1000",
         PRE: {
@@ -483,7 +538,7 @@
             }
         }
     },
-    "11": {
+    "14": {
         DESCR: "dualtor ignore vlan neighbor route miss case",
         ARGS: "route_check -i 15",
         RET: -1,