Update for the procedures for insertion/hot swap of Switch Fabric Module(SFM) by using "config chassis modules shutdown/startup" commands

Author: JunhongMao

config/chassis_modules.py

@@ -1,8 +1,9 @@
 #!/usr/sbin/env python
 
 import click
-
+import time
 import utilities_common.cli as clicommon
+from .fabric_module_set_admin_status import fabric_module_set_admin_status
 
 #
 # 'chassis_modules' group ('config chassis_modules ...')
@@ -17,6 +18,26 @@ def modules():
     """Configure chassis modules"""
     pass
 
+def get_config_module_state(db, chassis_module_name):
+    config_db = db.cfgdb
+    fvs = config_db.get_entry('CHASSIS_MODULE', chassis_module_name)
+    return fvs['admin_status']
+
+TIMEOUT_SECS = 10
+
+# Name: get_config_module_state_timeout
+# return: True: timeout, False: not timeout
+def get_config_module_state_timeout(ctx, db, chassis_module_name, state):
+    counter = 0
+    while  get_config_module_state(db, chassis_module_name) != state:
+        time.sleep(1)
+        counter += 1
+        if counter >= TIMEOUT_SECS:
+            ctx.fail("get_config_module_state {} timeout".format(chassis_module_name))
+            return True
+            break
+    return False
+
 #
 # 'shutdown' subcommand ('config chassis_modules shutdown ...')
 #
@@ -36,6 +57,9 @@ def shutdown_chassis_module(db, chassis_module_name):
     fvs = {'admin_status': 'down'}
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, fvs)
 
+    if not get_config_module_state_timeout(ctx, db, chassis_module_name, 'down'):
+        fabric_module_set_admin_status(chassis_module_name, 'down')
+
 #
 # 'startup' subcommand ('config chassis_modules startup ...')
 #
@@ -45,5 +69,9 @@ def shutdown_chassis_module(db, chassis_module_name):
 def startup_chassis_module(db, chassis_module_name):
     """Chassis-module startup of module"""
     config_db = db.cfgdb
-
+    ctx = click.get_current_context()
+    
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, None)
+
+    if not get_config_module_state_timeout(ctx, db, chassis_module_name, 'up'):
+        fabric_module_set_admin_status(chassis_module_name, 'up')

config/fabric_module_set_admin_status.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2024, Nokia
+# All rights reserved.
+
+import re
+import subprocess
+import time
+from swsscommon.swsscommon import SonicV2Connector
+from sonic_py_common.logger import Logger
+from platform_ndk import nokia_common
+
+sfm_hw_slot_mapping = {
+    0: 15,
+    1: 16,
+    2: 17,
+    3: 18,
+    4: 19,
+    5: 20,
+    6: 21,
+    7: 22
+}
+
+# Name: fabric_module_set_admin_status.py, version: 1.0
+# Syntax: fabric_module_set_admin_status <module_name> <up/down>
+def fabric_module_set_admin_status(module, state):
+    logger = Logger("fabric_module_set_admin_status.py")
+    logger.set_min_log_priority_info()
+   
+    if not module.startswith("FABRIC-CARD"):
+        logger.log_warning("Failed to set {} state. Admin state can only be set on Fabric module.".format(module))
+        return
+
+    if (state != "up" and state != "down"):
+        logger.log_warning("Failed to set {}. Admin state can only be set to up or down.".format(state))
+        return
+
+    num = int(re.search(r"(\d+)$", module).group())
+    chassisdb = SonicV2Connector(host="127.0.0.1")
+    chassisdb.connect("CHASSIS_STATE_DB")
+
+    if state == "down":
+        services_list = chassisdb.keys("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE*")
+        asic_list = []
+        for service in services_list:
+            name = chassisdb.get("CHASSIS_STATE_DB",service,"name")
+            if name == module:
+                asic_id = int(re.search(r"(\d+)$", service).group())
+                asic_list.append(asic_id)
+
+        logger.log_info("Shutting down chassis module {}".format(module))    
+
+        for asic in asic_list:
+            logger.log_info("Stopping swss@{} and syncd@{} ...".format(asic, asic))
+            process = subprocess.Popen(['sudo', 'systemctl', 'stop', 'swss@{}.service'.format(asic)],
+                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            outstr = stdout.decode('ascii')
+        # wait for service is down
+        time.sleep(2)
+
+        logger.log_info("Power off {} module ...".format(module))
+        hw_slot = sfm_hw_slot_mapping[num]
+        nokia_common._power_onoff_SFM(hw_slot,False)
+        logger.log_info("Chassis module {} shutdown completed".format(module))
+    else:
+        logger.log_info("Starting up chassis module {}".format(module))
+        hw_slot = sfm_hw_slot_mapping[num]
+        nokia_common._power_onoff_SFM(hw_slot,True)
+        # wait SFM HW init done.
+        time.sleep(15)
+
+        services_list = chassisdb.keys("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE*")
+        asic_list = []
+        for service in services_list:
+            name = chassisdb.get("CHASSIS_STATE_DB",service,"name")
+            if name == module:
+                asic_id = int(re.search(r"(\d+)$", service).group())
+                asic_list.append(asic_id)
+
+        for asic in asic_list:
+            logger.log_info("Start swss@{} and syncd@{} ...".format(asic, asic))
+            # Process state
+            process = subprocess.Popen(['sudo', 'systemctl', 'start', 'swss@{}.service'.format(asic)],
+                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            outstr = stdout.decode('ascii')
+        logger.log_info("Chassis module {} startup completed".format(module))
+    return