Merge branch 'master' into intf_cmds

Author: judyjoseph

config/main.py

@@ -13,7 +13,8 @@
 import time
 
 from minigraph import parse_device_desc_xml
-from portconfig import get_child_ports, get_port_config_file_name
+
+from portconfig import get_child_ports
 from sonic_py_common import device_info, multi_asic
 from sonic_py_common.interface import front_panel_prefix, portchannel_prefix, vlan_prefix, loopback_prefix
 from swsssdk import ConfigDBConnector, SonicV2Connector, SonicDBConfig
@@ -69,31 +70,12 @@ def readJsonFile(fileName):
         raise Exception(str(e))
     return result
 
-def _get_breakout_cfg_file_name():
-    """
-    Get name of config file for Dynamic Port Breakout
-    """
-    try:
-        (platform, hwsku) = device_info.get_platform_and_hwsku()
-    except Exception as e:
-        click.secho("Failed to get platform and hwsku with error:{}".format(str(e)), fg='red')
-        raise click.Abort()
-
-    try:
-        breakout_cfg_file_name = get_port_config_file_name(hwsku, platform)
-    except Exception as e:
-        click.secho("Breakout config file not found with error:{}".format(str(e)), fg='red')
-        raise click.Abort()
-
-    return breakout_cfg_file_name
-
-
 def _get_breakout_options(ctx, args, incomplete):
     """ Provides dynamic mode option as per user argument i.e. interface name """
     all_mode_options = []
     interface_name = args[-1]
 
-    breakout_cfg_file = _get_breakout_cfg_file_name()
+    breakout_cfg_file = device_info.get_path_to_port_config_file()
 
     if not os.path.isfile(breakout_cfg_file) or not breakout_cfg_file.endswith('.json'):
         return []
@@ -1659,63 +1641,35 @@ def reload():
                 click.secho(
                     "Command 'qos reload' failed with invalid namespace '{}'".
                         format(ns),
-                    fg='yellow'
+                    fg="yellow"
                 )
                 raise click.Abort()
             asic_id_suffix = str(asic_id)
 
-        buffer_template_file = os.path.join(
-            hwsku_path,
-            asic_id_suffix,
-            'buffers.json.j2'
-        )
-        buffer_output_file = "/tmp/buffers{}.json".format(asic_id_suffix)
-        qos_output_file = "/tmp/qos{}.json".format(asic_id_suffix)
-
-        cmd_ns = "" if ns is DEFAULT_NAMESPACE else "-n {}".format(ns)
+        buffer_template_file = os.path.join(hwsku_path, asic_id_suffix, "buffers.json.j2")
         if os.path.isfile(buffer_template_file):
-            command = "{} {} -d -t {} > {}".format(
-                SONIC_CFGGEN_PATH,
-                cmd_ns,
-                buffer_template_file,
-                buffer_output_file
-            )
-            clicommon.run_command(command, display_cmd=True)
-            qos_template_file = os.path.join(
-                hwsku_path,
-                asic_id_suffix,
-                'qos.json.j2'
-            )
-            sonic_version_file = os.path.join(
-                '/etc/sonic/', 'sonic_version.yml'
-            )
+            qos_template_file = os.path.join(hwsku_path, asic_id_suffix, "qos.json.j2")
             if os.path.isfile(qos_template_file):
-                command = "{} {} -d -t {} -y {} > {}".format(
+                cmd_ns = "" if ns is DEFAULT_NAMESPACE else "-n {}".format(ns)
+                sonic_version_file = os.path.join('/', "etc", "sonic", "sonic_version.yml")
+                command = "{} {} -d -t {},config-db -t {},config-db -y {} --write-to-db".format(
                     SONIC_CFGGEN_PATH,
                     cmd_ns,
+                    buffer_template_file,
                     qos_template_file,
-                    sonic_version_file,
-                    qos_output_file
+                    sonic_version_file
                 )
-                clicommon.run_command(command, display_cmd=True)
                 # Apply the configurations only when both buffer and qos
-                # configuration files are presented
-                command = "{} {} -j {} --write-to-db".format(
-                    SONIC_CFGGEN_PATH, cmd_ns, buffer_output_file
-                )
-                clicommon.run_command(command, display_cmd=True)
-                command = "{} {} -j {} --write-to-db".format(
-                    SONIC_CFGGEN_PATH, cmd_ns, qos_output_file
-                )
+                # configuration files are present
                 clicommon.run_command(command, display_cmd=True)
             else:
-                click.secho('QoS definition template not found at {}'.format(
+                click.secho("QoS definition template not found at {}".format(
                     qos_template_file
-                ), fg='yellow')
+                ), fg="yellow")
         else:
-            click.secho('Buffer definition template not found at {}'.format(
+            click.secho("Buffer definition template not found at {}".format(
                 buffer_template_file
-            ), fg='yellow')
+            ), fg="yellow")
 
 #
 # 'warm_restart' group ('config warm_restart ...')
@@ -2270,7 +2224,7 @@ def speed(ctx, interface_name, interface_speed, verbose):
 @click.pass_context
 def breakout(ctx, interface_name, mode, verbose, force_remove_dependencies, load_predefined_config):
     """ Set interface breakout mode """
-    breakout_cfg_file = _get_breakout_cfg_file_name()
+    breakout_cfg_file = device_info.get_path_to_port_config_file()
 
     if not os.path.isfile(breakout_cfg_file) or not breakout_cfg_file.endswith('.json'):
         click.secho("[ERROR] Breakout feature is not available without platform.json file", fg='red')

fdbutil/__init__.py

@@ -1,15 +1,13 @@
-#!/usr/bin/env python
-
+import argparse
 import json
-import sys
 import os
-import argparse
+import sys
 import syslog
-import traceback
 import time
+import traceback
 
-from ipaddress import ip_address, ip_network, ip_interface
 from collections import defaultdict
+from ipaddress import ip_address, ip_network, ip_interface
 
 def get_vlan_cidr_map(filename):
     """
@@ -35,7 +33,9 @@ def get_vlan_cidr_map(filename):
                 continue
             vlan, cidr = tuple(vlan_key.split('|'))
             if vlan in config_db_entries["VLAN"]:
-                vlan_cidr[vlan] = ip_interface(cidr).network
+                if vlan not in vlan_cidr:
+                    vlan_cidr[vlan] = {4: ip_address("0.0.0.0".decode()), 6: ip_address("::".decode())}
+                vlan_cidr[vlan][ip_interface(cidr).version] = ip_interface(cidr).network
 
     return vlan_cidr
 
@@ -65,8 +65,9 @@ def get_arp_entries_map(arp_filename, config_db_filename):
                 continue
             table, vlan, ip = tuple(key.split(':'))
             if "NEIGH_TABLE" in table and vlan in vlan_cidr.keys() \
-                and ip_address(ip) in ip_network(vlan_cidr[vlan]) and "neigh" in config.keys():
-                arp_map[config["neigh"].replace(':', '-')] = ""
+                and ip_address(ip) in ip_network(vlan_cidr[vlan][ip_interface(ip).version]) \
+                and "neigh" in config.keys():
+                arp_map[config["neigh"].replace(':', '-').upper()] = ""
 
     return arp_map
 
@@ -94,7 +95,7 @@ def filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_fi
     def filter_fdb_entry(fdb_entry):
         for key, _ in fdb_entry.items():
             if 'FDB_TABLE' in key:
-                return key.split(':')[-1] in arp_map
+                return key.split(':')[-1].upper() in arp_map
 
         # malformed entry, default to False so it will be deleted
         return False
@@ -124,44 +125,33 @@ def file_exists_or_raise(filename):
     if not os.path.exists(filename):
         raise Exception("file '{0}' does not exist".format(filename))
 
-def main():
+def main(argv=sys.argv):
     parser = argparse.ArgumentParser()
     parser.add_argument('-f', '--fdb', type=str, default='/tmp/fdb.json', help='fdb file name')
     parser.add_argument('-a', '--arp', type=str, default='/tmp/arp.json', help='arp file name')
     parser.add_argument('-c', '--config_db', type=str, default='/tmp/config_db.json', help='config db file name')
     parser.add_argument('-b', '--backup_file', type=bool, default=True, help='Back up old fdb entries file')
-    args = parser.parse_args()
+    args = parser.parse_args(argv[1:])
 
     fdb_filename = args.fdb
     arp_filename = args.arp
     config_db_filename = args.config_db
     backup_file = args.backup_file
 
+    res = 0
     try:
+        syslog.openlog('filter_fdb_entries')
         file_exists_or_raise(fdb_filename)
         file_exists_or_raise(arp_filename)
         file_exists_or_raise(config_db_filename)
     except Exception as e:
         syslog.syslog(syslog.LOG_ERR, "Got an exception %s: Traceback: %s" % (str(e), traceback.format_exc()))
-    else:
-        filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_file)
-
-    return 0
-
-if __name__ == '__main__':
-    res = 0
-    try:
-        syslog.openlog('filter_fdb_entries')
-        res = main()
     except KeyboardInterrupt:
         syslog.syslog(syslog.LOG_NOTICE, "SIGINT received. Quitting")
         res = 1
-    except Exception as e:
-        syslog.syslog(syslog.LOG_ERR, "Got an exception %s: Traceback: %s" % (str(e), traceback.format_exc()))
-        res = 2
+    else:
+        filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_file)
     finally:
         syslog.closelog()
-    try:
-        sys.exit(res)
-    except SystemExit:
-        os._exit(res)
+
+    return res

scripts/filter_fdb_entries.py fdbutil/filter_fdb_entries.pyscripts/filter_fdb_entries.py renamed to fdbutil/filter_fdb_entries.py

@@ -1,4 +1,4 @@
 [pytest]
 filterwarnings =
     ignore::DeprecationWarning
-addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml
+addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fdbutil --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml

pytest.ini

@@ -4,10 +4,23 @@
 """
 from __future__ import print_function
 
+import os
+import sys
 from tabulate import tabulate
 from swsssdk import SonicV2Connector
 from natsort import natsorted
 
+# mock the redis for unit test purposes #
+try:
+    if os.environ["UTILITIES_UNIT_TESTING"] == "1":
+        modules_path = os.path.join(os.path.dirname(__file__), "..")
+        test_path = os.path.join(modules_path, "tests")
+        sys.path.insert(0, modules_path)
+        sys.path.insert(0, test_path)
+        import mock_tables.dbconnector
+except KeyError:
+    pass
+
 
 header = ['Drawer', 'LED', 'FAN', 'Speed', 'Direction', 'Presence', 'Status', 'Timestamp']
 
@@ -52,13 +65,13 @@ class FanShow(object):
 
             presence = data_dict[PRESENCE_FIELD_NAME].lower()
             presence = 'Present' if presence == 'true' else 'Not Present'
-            status = data_dict[STATUS_FIELD_NAME].lower()
-            if status == 'true':
+            
+            status = data_dict[STATUS_FIELD_NAME]
+            status_lower = status.lower()
+            if status_lower == 'true':
                 status = 'OK'
-            elif status == 'false':
+            elif status_lower == 'false':
                 status = 'Not OK'
-            else:
-                status = 'N/A'
 
             table.append((data_dict[DRAWER_FIELD_NAME], data_dict[LED_STATUS_FIELD_NAME], name, speed, data_dict[DIRECTION_FIELD_NAME], presence, status, 
                           data_dict[TIMESTAMP_FIELD_NAME]))

scripts/fanshow

@@ -18,7 +18,7 @@ DEVPATH="/usr/share/sonic/device"
 PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)
 PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin"
 LOG_SSD_HEALTH="/usr/bin/log_ssd_health"
-
+SSD_FW_UPDATE="ssd-fw-upgrade"
 # Require 100M available on the hard drive for warm reboot temp files,
 # Size is in 1K blocks:
 MIN_HD_SPACE_NEEDED=100000
@@ -456,7 +456,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
 
     FILTER_FDB_ENTRIES_RC=0
     # Filter FDB entries using MAC addresses from ARP table
-    /usr/bin/filter_fdb_entries.py -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
+    /usr/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
     if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then
         error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC"
         unload_kernel
@@ -503,9 +503,9 @@ systemctl stop nat
 debug "Stopped nat ..."
 
 # Kill radv before stopping BGP service to prevent annoucing our departure.
-debug "Stopping radv ..."
-docker kill radv &>/dev/null || [ $? == 1 ]
+debug "Stopping radv service..."
 systemctl stop radv
+debug "Stopped radv service..."
 
 # Kill bgpd to start the bgp graceful restart procedure
 debug "Stopping bgp ..."
@@ -536,12 +536,9 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
     debug "Stopped teamd ..."
 fi
 
-# Kill swss Docker container
-# We call `docker kill swss` to ensure the container stops as quickly as possible,
-# then immediately call `systemctl stop swss` to prevent the service from
-# restarting the container automatically.
-docker kill swss &> /dev/null || debug "Docker swss is not running ($?) ..."
+debug "Stopping swss service ..."
 systemctl stop swss
+debug "Stopped swss service ..."
 
 # Pre-shutdown syncd
 if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
@@ -633,6 +630,11 @@ if [ -x /sbin/hwclock ]; then
     /sbin/hwclock -w || /bin/true
 fi
 
+if [ -x ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ]; then
+    debug "updating ssd fw for${REBOOT_TYPE}"
+    ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ${REBOOT_TYPE}
+fi
+
 # Enable Watchdog Timer
 if [ -x ${WATCHDOG_UTIL} ]; then
     debug "Enabling Watchdog before ${REBOOT_TYPE}"