Merge pull request #5 from gechiang/202205

[Manual Sync] From sonic-net/sonic-utilities 202205 to Azure/sonic-utitities.msft repo

Author: gechiang

config/vlan.py

@@ -112,11 +112,24 @@ def del_vlan(db, vid, no_restart_dhcp_relay):
         # We need to restart dhcp_relay service after dhcpv6_relay config change
         if is_dhcp_relay_running():
             dhcp_relay_util.handle_restart_dhcp_relay_service()
+    
+    vlans = db.cfgdb.get_keys('VLAN')
+    if not vlans:
+        docker_exec_cmd = "docker exec -i swss {}"
+        _, rc = clicommon.run_command(docker_exec_cmd.format("supervisorctl status ndppd"), ignore_error=True, return_cmd=True)
+        if rc == 0:
+            click.echo("No VLANs remaining, stopping ndppd service")
+            clicommon.run_command(docker_exec_cmd.format("supervisorctl stop ndppd"), ignore_error=True, return_cmd=True)
+            clicommon.run_command(docker_exec_cmd.format("rm -f /etc/supervisor/conf.d/ndppd.conf"), ignore_error=True, return_cmd=True)
+            clicommon.run_command(docker_exec_cmd.format("supervisorctl update"), return_cmd=True)
 
 
 def restart_ndppd():
     verify_swss_running_cmd = "docker container inspect -f '{{.State.Status}}' swss"
     docker_exec_cmd = "docker exec -i swss {}"
+    ndppd_status_cmd= "supervisorctl status ndppd"
+    ndppd_conf_copy_cmd = "cp /usr/share/sonic/templates/ndppd.conf /etc/supervisor/conf.d/"
+    supervisor_update_cmd = "supervisorctl update"
     ndppd_config_gen_cmd = "sonic-cfggen -d -t /usr/share/sonic/templates/ndppd.conf.j2,/etc/ndppd.conf"
     ndppd_restart_cmd = "supervisorctl restart ndppd"
 
@@ -126,9 +139,16 @@ def restart_ndppd():
         click.echo(click.style('SWSS container is not running, changes will take effect the next time the SWSS container starts', fg='red'),)
         return
 
-    clicommon.run_command(docker_exec_cmd.format(ndppd_config_gen_cmd), display_cmd=True)
+    _, rc = clicommon.run_command(docker_exec_cmd.format(ndppd_status_cmd), ignore_error=True, return_cmd=True)
+
+    if rc != 0:
+        clicommon.run_command(docker_exec_cmd.format(ndppd_conf_copy_cmd))
+        clicommon.run_command(docker_exec_cmd.format(supervisor_update_cmd), return_cmd=True)
+
+    click.echo("Starting ndppd service")
+    clicommon.run_command(docker_exec_cmd.format(ndppd_config_gen_cmd))
     sleep(3)
-    clicommon.run_command(docker_exec_cmd.format(ndppd_restart_cmd), display_cmd=True)
+    clicommon.run_command(docker_exec_cmd.format(ndppd_restart_cmd), return_cmd=True)
 
 
 @vlan.command('proxy_arp')

generic_config_updater/field_operation_validators.py

@@ -3,24 +3,8 @@
 
 def rdma_config_update_validator():
     version_info = device_info.get_sonic_version_info()
-    build_version = version_info.get('build_version')
     asic_type = version_info.get('asic_type')
 
     if (asic_type != 'mellanox' and asic_type != 'broadcom' and asic_type != 'cisco-8000'):
         return False
-
-    version_substrings = build_version.split('.')
-    branch_version = None
-
-    for substring in version_substrings:
-        if substring.isdigit() and re.match(r'^\d{8}$', substring):
-            branch_version = substring
-            break
-
-    if branch_version is None:
-        return False
-
-    if asic_type == 'cisco-8000':
-        return branch_version >= "20201200"
-    else:
-        return branch_version >= "20181100"
+    return True

scripts/fast-reboot

@@ -17,6 +17,7 @@ STRICT=no
 REBOOT_METHOD="/sbin/kexec -e"
 ASSISTANT_IP_LIST=""
 ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
+LAG_KEEPALIVE_SCRIPT="/usr/local/bin/lag_keepalive.py"
 WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
 DEVPATH="/usr/share/sonic/device"
 PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)
@@ -645,6 +646,13 @@ fi
 # disable trap-handlers which were set before
 trap '' EXIT HUP INT QUIT TERM KILL ABRT ALRM
 
+# start sending LACPDUs to keep the LAGs refreshed
+# this is a non-blocking call, and the process will die in 300s
+debug "Starting lag_keepalive to send LACPDUs ..."
+timeout 300 python ${LAG_KEEPALIVE_SCRIPT} &
+# give the lag_keepalive script a chance to get ready (30s) and collect one lacpdu before going down (30s)
+sleep 60
+
 if [ -x ${LOG_SSD_HEALTH} ]; then
     debug "Collecting logs to check ssd health before ${REBOOT_TYPE}..."
     ${LOG_SSD_HEALTH}

scripts/lag_keepalive.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from scapy.config import conf
+conf.ipv6_enabled = False
+from scapy.all import sendp, sniff
+from swsscommon.swsscommon import ConfigDBConnector
+import time, threading, traceback
+import syslog
+
+SYSLOG_ID = 'lag_keepalive'
+
+
+def log_info(msg):
+    syslog.openlog(SYSLOG_ID)
+    syslog.syslog(syslog.LOG_INFO, msg)
+    syslog.closelog()
+
+
+def log_error(msg):
+    syslog.openlog(SYSLOG_ID)
+    syslog.syslog(syslog.LOG_ERR, msg)
+    syslog.closelog()
+
+
+def sniff_lacpdu(device_mac, lag_member, lag_member_to_packet):
+    sniffed_packet = sniff(iface=lag_member,
+        filter="ether proto 0x8809 and ether src {}".format(device_mac),
+        count=1, timeout=30)
+    lag_member_to_packet[lag_member] = sniffed_packet
+
+
+def get_lacpdu_per_lag_member():
+    appDB = ConfigDBConnector()
+    appDB.db_connect('APPL_DB')
+    appDB_lag_info = appDB.get_keys('LAG_MEMBER_TABLE')
+    configDB = ConfigDBConnector()
+    configDB.db_connect('CONFIG_DB')
+    device_mac = configDB.get(configDB.CONFIG_DB,  "DEVICE_METADATA|localhost", "mac")
+    hwsku = configDB.get(configDB.CONFIG_DB,  "DEVICE_METADATA|localhost", "hwsku")
+    active_lag_members = list()
+    lag_member_to_packet = dict()
+    sniffer_threads = list()
+    for lag_entry in appDB_lag_info:
+        lag_name = str(lag_entry[0])
+        oper_status = appDB.get(appDB.APPL_DB,"LAG_TABLE:{}".format(lag_name), "oper_status")
+        if oper_status == "up":
+            # only apply the workaround for active lags
+            lag_member = str(lag_entry[1])
+            active_lag_members.append(lag_member)
+            # use threading to capture lacpdus from several lag members simultaneously
+            sniffer_thread = threading.Thread(target=sniff_lacpdu,
+                args=(device_mac, lag_member, lag_member_to_packet))
+            sniffer_thread.start()
+            sniffer_threads.append(sniffer_thread)
+
+    # sniff for lacpdu should finish in <= 30s. sniff timeout is also set to 30s
+    for sniffer in sniffer_threads:
+        sniffer.join(timeout=30)
+    
+    return active_lag_members, lag_member_to_packet
+
+
+def lag_keepalive(lag_member_to_packet):
+    while True:
+        for lag_member, packet in lag_member_to_packet.items():
+            try:
+                sendp(packet, iface=lag_member, verbose=False)
+            except Exception:
+                # log failure and continue to send lacpdu
+                traceback_msg = traceback.format_exc()
+                log_error("Failed to send LACPDU packet from interface {} with error: {}".format(
+                    lag_member, traceback_msg))
+                continue
+        log_info("sent LACPDU packets via {}".format(lag_member_to_packet.keys()))
+        time.sleep(1)
+
+
+def main():
+    while True:
+        try:
+            active_lag_members, lag_member_to_packet = get_lacpdu_per_lag_member()
+            if len(active_lag_members) != len(lag_member_to_packet.keys()):
+                log_error("Failed to capture LACPDU packets for some lag members. " +\
+                "Active lag members: {}. LACPDUs captured for: {}".format(
+                    active_lag_members, lag_member_to_packet.keys()))
+
+            log_info("ready to send LACPDU packets via {}".format(lag_member_to_packet.keys()))
+        except Exception:
+            traceback_msg = traceback.format_exc()
+            log_error("Failed to get LAG members and LACPDUs with error: {}".format(
+                traceback_msg))
+            # keep attempting until sniffed packets are ready
+            continue
+        # if no exceptions are thrown, break from loop as LACPDUs are ready to be sent
+        break
+
+    if lag_member_to_packet:
+        # start an infinite loop to keep sending lacpdus from lag member ports
+        lag_keepalive(lag_member_to_packet)
+
+if __name__ == "__main__":
+    main()

scripts/route_check.py

@@ -434,7 +434,7 @@ def filter_out_vnet_routes(routes):
     vnet_routes = []
 
     for vnet_route_db_key in vnet_routes_db_keys:
-        vnet_route_attrs = vnet_route_db_key.split(':')
+        vnet_route_attrs = vnet_route_db_key.split(':', 1)
         vnet_name = vnet_route_attrs[0]
         vnet_route = vnet_route_attrs[1]
         vnet_routes.append(vnet_route)

setup.py

@@ -113,6 +113,7 @@
         'scripts/intfutil',
         'scripts/intfstat',
         'scripts/ipintutil',
+        'scripts/lag_keepalive.py',
         'scripts/lldpshow',
         'scripts/log_ssd_health',
         'scripts/mellanox_buffer_migrator.py',

sfputil/main.py

@@ -1201,6 +1201,18 @@ def reset(port_name):
 
         i += 1
 
+def update_firmware_info_to_state_db(port_name):
+    physical_port = logical_port_to_physical_port_index(port_name)
+
+    namespaces = multi_asic.get_front_end_namespaces()
+    for namespace in namespaces:
+        state_db = SonicV2Connector(use_unix_socket_path=False, namespace=namespace)
+        if state_db is not None:
+            state_db.connect(state_db.STATE_DB)
+            active_firmware, inactive_firmware = platform_chassis.get_sfp(physical_port).get_transceiver_info_firmware_versions()
+            state_db.set(state_db.STATE_DB, 'TRANSCEIVER_INFO|{}'.format(port_name), "active_firmware", active_firmware)
+            state_db.set(state_db.STATE_DB, 'TRANSCEIVER_INFO|{}'.format(port_name), "inactive_firmware", inactive_firmware)
+
 # 'firmware' subgroup
 @cli.group()
 def firmware():
@@ -1271,7 +1283,7 @@ def is_fw_switch_done(port_name):
 
         if fw_info['status'] == True:
             (ImageA, ImageARunning, ImageACommitted, ImageAInvalid,
-             ImageB, ImageBRunning, ImageBCommitted, ImageBInvalid) = fw_info['result']
+             ImageB, ImageBRunning, ImageBCommitted, ImageBInvalid, _, _) = fw_info['result']
 
             if (ImageARunning == 1) and (ImageAInvalid == 1):       # ImageA is running, but also invalid.
                 click.echo("FW info error : ImageA shows running, but also shows invalid!")
@@ -1382,6 +1394,7 @@ def download_firmware(port_name, filepath):
     sfp.set_optoe_write_max(1)
 
     status = api.cdb_firmware_download_complete()
+    update_firmware_info_to_state_db(port_name)
     click.echo('CDB: firmware download complete')
     return status
 
@@ -1409,6 +1422,7 @@ def run(port_name, mode):
         click.echo('Failed to run firmware in mode={}! CDB status: {}'.format(mode, status))
         sys.exit(EXIT_FAIL)
 
+    update_firmware_info_to_state_db(port_name)
     click.echo("Firmware run in mode={} success".format(mode))
 
 # 'commit' subcommand
@@ -1430,6 +1444,7 @@ def commit(port_name):
         click.echo('Failed to commit firmware! CDB status: {}'.format(status))
         sys.exit(EXIT_FAIL)
 
+    update_firmware_info_to_state_db(port_name)
     click.echo("Firmware commit successful")
 
 # 'upgrade' subcommand

sonic_installer/bootloader/uboot.py

@@ -89,7 +89,8 @@ def set_fips(self, image, enable):
         cmdline = out.strip()
         cmdline = re.sub('^linuxargs=', '', cmdline)
         cmdline = re.sub(r' sonic_fips=[^\s]', '', cmdline) + " sonic_fips=" + fips
-        run_command('/usr/bin/fw_setenv linuxargs ' +  cmdline)
+        cmdline = '"' + cmdline + '"'
+        run_command('/usr/bin/fw_setenv linuxargs ' +  cmdline )
         click.echo('Done')
 
     def get_fips(self, image):

tests/sfputil_test.py

@@ -710,12 +710,12 @@ def test_run_firmwre(self, mock_chassis):
     @patch('sfputil.main.logical_port_to_physical_port_index', MagicMock(return_value=1))
     @pytest.mark.parametrize("mock_response, expected", [
         ({'status': False, 'result': None}                                 , -1),
-        ({'status': True,  'result': ("1.0.1", 1, 1, 0, "1.0.2", 0, 0, 0)} , -1),
-        ({'status': True,  'result': ("1.0.1", 0, 0, 0, "1.0.2", 1, 1, 0)} , -1),
-        ({'status': True,  'result': ("1.0.1", 1, 0, 0, "1.0.2", 0, 1, 0)} ,  1),
-        ({'status': True,  'result': ("1.0.1", 0, 1, 0, "1.0.2", 1, 0, 0)} ,  1),
-        ({'status': True,  'result': ("1.0.1", 1, 0, 1, "1.0.2", 0, 1, 0)} , -1),
-        ({'status': True,  'result': ("1.0.1", 0, 1, 0, "1.0.2", 1, 0, 1)} , -1),
+        ({'status': True,  'result': ("1.0.1", 1, 1, 0, "1.0.2", 0, 0, 0, "1.0.1", "1.0.2")} , -1),
+        ({'status': True,  'result': ("1.0.1", 0, 0, 0, "1.0.2", 1, 1, 0, "1.0.2", "1.0.1")} , -1),
+        ({'status': True,  'result': ("1.0.1", 1, 0, 0, "1.0.2", 0, 1, 0, "1.0.1", "1.0.2")} ,  1),
+        ({'status': True,  'result': ("1.0.1", 0, 1, 0, "1.0.2", 1, 0, 0, "1.0.2", "1.0.1")} ,  1),
+        ({'status': True,  'result': ("1.0.1", 1, 0, 1, "1.0.2", 0, 1, 0, "1.0.1", "1.0.2")} , -1),
+        ({'status': True,  'result': ("1.0.1", 0, 1, 0, "1.0.2", 1, 0, 1, "1.0.2", "1.0.1")} , -1),
 
         # "is_fw_switch_done" function will waiting until timeout under below condition, so that this test will spend around 1min.
         ({'status': False, 'result': 0}                                    , -1),
@@ -790,3 +790,32 @@ def test_firmware_download_RJ45(self):
         result = runner.invoke(sfputil.cli.commands['firmware'].commands['download'], ["Ethernet0", "a.b"])
         assert result.output == 'This functionality is not applicable for RJ45 port Ethernet0.\n'
         assert result.exit_code == EXIT_FAIL
+
+    @patch('sfputil.main.is_sfp_present', MagicMock(return_value=True))
+    @patch('sfputil.main.is_port_type_rj45', MagicMock(return_value=False))
+    @patch('sfputil.main.run_firmware', MagicMock(return_value=1))
+    @patch('sfputil.main.update_firmware_info_to_state_db', MagicMock())
+    def test_firmware_run_cli(self):
+        runner = CliRunner()
+        result = runner.invoke(sfputil.cli.commands['firmware'].commands['run'], ["Ethernet0"])
+        assert result.exit_code == 0
+
+    @patch('sfputil.main.is_sfp_present', MagicMock(return_value=True))
+    @patch('sfputil.main.is_port_type_rj45', MagicMock(return_value=False))
+    @patch('sfputil.main.commit_firmware', MagicMock(return_value=1))
+    @patch('sfputil.main.update_firmware_info_to_state_db', MagicMock())
+    def test_firmware_commit_cli(self):
+        runner = CliRunner()
+        result = runner.invoke(sfputil.cli.commands['firmware'].commands['commit'], ["Ethernet0"])
+        assert result.exit_code == 0
+
+    @patch('sfputil.main.logical_port_to_physical_port_index', MagicMock(return_value=1))
+    @patch('sonic_py_common.multi_asic.get_front_end_namespaces', MagicMock(return_value=['']))
+    @patch('sfputil.main.SonicV2Connector', MagicMock())
+    @patch('sfputil.main.platform_chassis')
+    def test_update_firmware_info_to_state_db(self, mock_chassis):
+        mock_sfp = MagicMock()
+        mock_chassis.get_sfp = MagicMock(return_value=mock_sfp)
+        mock_sfp.get_transceiver_info_firmware_versions.return_value = ['a.b.c', 'd.e.f']
+
+        sfputil.update_firmware_info_to_state_db("Ethernet0")

tests/vlan_test.py

@@ -403,6 +403,27 @@ def test_config_vlan_del_vlan(self, mock_restart_dhcp_relay_service):
         assert result.exit_code == 0
         assert result.output == show_vlan_brief_empty_output
 
+    def test_config_vlan_del_last_vlan(self):
+        runner = CliRunner()
+        db = Db()
+        db.cfgdb.delete_table("VLAN_MEMBER")
+        db.cfgdb.delete_table("VLAN_INTERFACE")
+        db.cfgdb.set_entry("VLAN", "Vlan2000", None)
+        db.cfgdb.set_entry("VLAN", "Vlan3000", None)
+        db.cfgdb.set_entry("VLAN", "Vlan4000", None)
+
+        with mock.patch("utilities_common.cli.run_command", mock.Mock(return_value=("", 0))) as mock_run_command:
+            result = runner.invoke(config.config.commands["vlan"].commands["del"], ["1000"], obj=db)
+            print(result.exit_code)
+            print(result.output)
+            mock_run_command.assert_has_calls([
+                mock.call("docker exec -i swss supervisorctl status ndppd", ignore_error=True, return_cmd=True),
+                mock.call("docker exec -i swss supervisorctl stop ndppd", ignore_error=True, return_cmd=True),
+                mock.call("docker exec -i swss rm -f /etc/supervisor/conf.d/ndppd.conf", ignore_error=True, return_cmd=True),
+                mock.call("docker exec -i swss supervisorctl update", return_cmd=True)
+            ])
+            assert result.exit_code == 0
+
     def test_config_vlan_del_nonexist_vlan_member(self):
         runner = CliRunner()
 
@@ -533,19 +554,29 @@ def test_config_vlan_proxy_arp_with_nonexist_vlan_intf(self):
             assert result.exit_code != 0
             assert "Interface Vlan1001 does not exist" in result.output
 
-    def test_config_vlan_proxy_arp_enable(self, mock_restart_dhcp_relay_service):
-        runner = CliRunner()
-        db = Db()
+    def test_config_vlan_proxy_arp_enable(self):
+        mock_cli_returns = [("running", 0),("", 1)] + [("", 0)] * 4
+        with mock.patch("utilities_common.cli.run_command", mock.Mock(side_effect=mock_cli_returns)) as mock_run_command:
+            runner = CliRunner()
+            db = Db()
 
-        result = runner.invoke(config.config.commands["vlan"].commands["proxy_arp"], ["1000", "enabled"], obj=db)
+            result = runner.invoke(config.config.commands["vlan"].commands["proxy_arp"], ["1000", "enabled"], obj=db)
 
-        print(result.exit_code)
-        print(result.output)
+            print(result.exit_code)
+            print(result.output)
+
+            expected_calls = [mock.call("docker container inspect -f '{{.State.Status}}' swss", return_cmd=True),
+                              mock.call('docker exec -i swss supervisorctl status ndppd', ignore_error=True, return_cmd=True),
+                              mock.call('docker exec -i swss cp /usr/share/sonic/templates/ndppd.conf /etc/supervisor/conf.d/'),
+                              mock.call('docker exec -i swss supervisorctl update', return_cmd=True),
+                              mock.call('docker exec -i swss sonic-cfggen -d -t /usr/share/sonic/templates/ndppd.conf.j2,/etc/ndppd.conf'),
+                              mock.call('docker exec -i swss supervisorctl restart ndppd', return_cmd=True)]
+            mock_run_command.assert_has_calls(expected_calls)
 
-        assert result.exit_code == 0 
-        assert db.cfgdb.get_entry("VLAN_INTERFACE", "Vlan1000") == {"proxy_arp": "enabled"}
+            assert result.exit_code == 0 
+            assert db.cfgdb.get_entry("VLAN_INTERFACE", "Vlan1000") == {"proxy_arp": "enabled"}
 
-    def test_config_vlan_proxy_arp_disable(self, mock_restart_dhcp_relay_service):
+    def test_config_vlan_proxy_arp_disable(self):
         runner = CliRunner()
         db = Db()