Generate PFC storm using PFC backpressure on additional Arista devices

tests/common/helpers/pfc_gen_brcm_xgs.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+
+"""
+Script to generate PFC storm.
+
+"""
+import binascii
+import sys
+import optparse
+import logging
+import logging.handlers
+import re
+import signal
+import subprocess
+import time
+
+
+logger = logging.getLogger('MyLogger')
+logger.setLevel(logging.DEBUG)
+
+
+class SignalCleanup():
+    def __init__(self, fanoutPfcStorm, endMsg):
+        self.fanoutPfcStorm = fanoutPfcStorm
+        self.endMsg = endMsg
+        signal.signal(signal.SIGTERM, self.sigHandler)
+
+    def sigHandler(self, *args):
+        self.fanoutPfcStorm.endAllPfcStorm()
+
+        logger.debug(self.endMsg)
+        sys.exit(0)
+
+
+class FanoutPfcStorm():
+    '''
+    For eos this class expects all interfaces to be in the front panel interface format
+    ex. Ethernet1/1 and not et1_1
+    For sonic, the interfaces are the default format
+    '''
+    def __init__(self, priority, chipName, os):
+        self.intfsEnabled = []
+        self.priority = priority
+        self.switchChip = chipName
+        self.os = os
+        self.intfToMmuPort, self.intfToPort = self._parseInterfaceMapFullSonic() if os == 'sonic' else self._parseInterfaceMapFull()
+
+    def _shellCmd(self, cmd):
+        output = ""
+        result = subprocess.run([f"{cmd}"], capture_output=True, text=True, shell=True)
+        if result.returncode == 0:
+            output = result.stdout
+        return output
+
+    def _cliCmd(self, cmd):
+        output = ""
+        if self.os == 'sonic':
+            result = subprocess.run([f"bcmcmd '{cmd}'"], capture_output=True, text=True, shell=True)
+        else:
+            result = subprocess.run(
+                ["Cli", "-c", f"{cmd}"], capture_output=True, text=True)
+        if result.returncode == 0:
+            output = result.stdout
+        return output
+
+    def _bcmltshellCmd(self, cmd):
+        if self.os == 'sonic':
+            return self._cliCmd(f"bsh -c \"{cmd}\"")
+        else:
+            return self._cliCmd(f"en\nplatform trident shell\nbcmltshell\n{cmd}")
+
+    def _bcmshellCmd(self, cmd):
+        return self._cliCmd(f"en\nplatform trident shell\n{cmd}")
+
+    def _parseInterfaceMapFull(self):
+        intfToMmuPort = {}
+        intfToPort = {}
+
+        output = self._cliCmd(f"en\nshow platform trident interface map full")
+
+        for line in output.splitlines():
+            mo = re.search('Intf: (?P<intf>Ethernet\S+).{1,50}Port: {1,3}(?P<port>\S+).{1,100}P2M\[ {0,3}\d+\]: {1,3}(?P<mmu>\S+)', line)
+            if mo is None:
+                continue
+            intfToMmuPort[mo.group('intf')] = mo.group('mmu')
+            intfToPort[mo.group('intf')] = mo.group('port')
+
+        return intfToMmuPort, intfToPort
+
+    def _parseInterfaceMapFullSonic(self):
+        intfToMmuPort = {}
+        intfTolPort = {}
+        lPortToIntf = {}
+
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            output = self._bcmltshellCmd('knet netif info')
+            for info in output.split("Network interface Info:"):
+                mo = re.search(r"Name: (?P<intf>Ethernet\d+)[\s\S]{1,100}Port: (?P<lport>\d+)", info)
+                if mo is None:
+                   continue
+                lPortToIntf[mo.group('lport')] = mo.group('intf')
+                intfTolPort[mo.group('intf')] = mo.group('lport')
+            output = self._cliCmd("show portmap")
+            for line in output.splitlines():
+                entries = line.split()
+                if len(entries) == 7:
+                    lport = entries[2]
+                    mmuPort = entries[4]
+                    if lport in lPortToIntf:
+                        intfToMmuPort[lPortToIntf[lport]] = mmuPort
+                    intfTolPort[mo.group('intf')] = mo.group('lport')
+        else:
+            output = self._cliCmd('knet netif show')
+            for info in output.split("Interface ID"):
+                mo = re.search(r"name=(?P<intf>Ethernet\d+)[\s\S]{1,100}port=(?P<lport>\S+)", info)
+                if mo is None:
+                   continue
+                lPortToIntf[mo.group('lport')] = mo.group('intf')
+                intfTolPort[mo.group('intf')] = mo.group('lport')
+            output = self._cliCmd("show portmap")
+            for line in output.splitlines():
+                entries = line.split()
+                if len(entries) == 9:
+                    lport = entries[0]
+                    mmuPort = entries[4]
+                    if lport in lPortToIntf:
+                        intfToMmuPort[lPortToIntf[lport]] = mmuPort
+                    intfTolPort[mo.group('intf')] = mo.group('lport')
+
+        return intfToMmuPort, intfTolPort
+
+    def _endPfcStorm(self, intf):
+        '''
+        Intf format is Ethernet1/1
+
+        The users of this class are only expected to call
+        startPfcStorm and endAllPfcStorm
+        '''
+        mmuPort = self.intfToMmuPort[intf]
+        port = self.intfToPort[intf]
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0")
+            self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0")
+        else:
+            self._bcmshellCmd(f"setreg CHFC2PFC_STATE.{port} PRI_BKP=0")
+        self._cliCmd(f"en\nconf\n\nint {intf}\nno priority-flow-control on")
+        if self.os == 'sonic':
+            for prio in range(8):
+                self._cliCmd(f"config interface pfc priority {intf} {prio} off")
+        else:
+            for prio in range(8):
+                self._cliCmd(f"en\nconf\n\nint {intf}\nno priority-flow-control priority {prio} no-drop")
+
+    def startPfcStorm(self, intf):
+        if intf in self.intfsEnabled:
+            return
+        self.intfsEnabled.append(intf)
+
+        mmuPort = self.intfToMmuPort[intf]
+        port = self.intfToPort[intf]
+        if self.os == 'sonic':
+            for prio in range(8):
+                if (1 << prio) & self.priority:
+                    self._shellCmd(f"config interface pfc priority {intf} {prio} on")
+        else:
+            self._cliCmd(f"en\nconf\n\nint {intf}\npriority-flow-control on")
+            for prio in range(8):
+                if (1 << prio) & self.priority:
+                    self._cliCmd(f"en\nconf\n\nint {intf}\npriority-flow-control priority {prio} no-drop")
+
+        if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")):
+            self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=1")
+            self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP={self.priority}")
+        else:
+            self._bcmshellCmd(f"setreg CHFC2PFC_STATE.{port} PRI_BKP={self.priority}")
+
+    def endAllPfcStorm(self):
+        for intf in self.intfsEnabled:
+            self._endPfcStorm(intf)
+
+
+def main():
+    usage = "usage: %prog [options] arg1 arg2"
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option("-i", "--interface", type="string", dest="interface",
+                      help="Interface list to send packets, separated by ','", metavar="Interface")
+    parser.add_option('-p', "--priority", type="int", dest="priority",
+                      help="PFC class enable bitmap.", metavar="Priority", default=-1)
+    parser.add_option("-r", "--rsyslog-server", type="string", dest="rsyslog_server",
+                      default="127.0.0.1", help="Rsyslog server IPv4 address", metavar="IPAddress")
+    parser.add_option("-c", "--chipName", type="string", dest="chipName", metavar="ChipName",
+                      help="Name of chip in the switch, i.e. Tomahawk5")
+    parser.add_option("-o", "--os", type="string", dest="os",
+                      help="Operating system (eos or sonic)", default="eos")
+
+    (options, args) = parser.parse_args()
+
+    if options.interface is None:
+        print("Need to specify the interface to send PFC pause frame packets.")
+        parser.print_help()
+        sys.exit(1)
+
+    if options.chipName is None:
+        print("Need to specify the ChipName to determine what cli to generate PFC pause frame packets.")
+        parser.print_help()
+        sys.exit(1)
+
+    if options.priority > 255 or options.priority < 0:
+        print("Enable class bitmap is not valid. Need to be in range 0-255.")
+        parser.print_help()
+        sys.exit(1)
+
+    # Configure logging
+    handler = logging.handlers.SysLogHandler(address=(options.rsyslog_server, 514))
+    logger.addHandler(handler)
+
+    # List of front panel kernel intfs
+    interfaces = options.interface.split(',')
+
+    fs = FanoutPfcStorm(options.priority, options.chipName, options.os)
+    fsCleanup = SignalCleanup(fs, 'PFC_STORM_END')
+
+    logger.debug('PFC_STORM_DEBUG')
+    for intf in interfaces:
+        if options.os == 'eos':
+            intf = frontPanelIntfFromKernelIntfName(intf)
+        fs.startPfcStorm(intf)
+    logger.debug('PFC_STORM_START')
+
+    # wait forever until stop
+    while True:
+       time.sleep(100)
+
+def frontPanelIntfFromKernelIntfName(intf):
+   return intf.replace("et", "Ethernet").replace("_", "/")
+
+
+if __name__ == "__main__":
+    main()

tests/common/helpers/pfc_storm.py

@@ -12,6 +12,24 @@
 
 logger = logging.getLogger(__name__)
 
+def get_chip_name_if_asic_pfc_storm_supported(fanout):
+    hwSkuInfo = {
+        "Arista DCS-7060DX5": "Tomahawk4",
+        "Arista DCS-7060PX5": "Tomahawk4",
+        "Arista DCS-7060X6": "Tomahawk5",
+        "Arista-7060X6": "Tomahawk5",
+        "Arista DCS-7060CX": "Tomahawk",
+        "Arista-7060CX": "Tomahawk",
+        "Arista DCS-7260CX3": "Tomahawk2",
+        "Arista-7260CX3": "Tomahawk2",
+        "Arista-7260QX3": "Tomahawk2",
+        }
+
+    for sku, chip in hwSkuInfo.items():
+        if fanout.startswith(sku):
+            return chip
+
+    return None
 
 class PFCStorm(object):
     """ PFC storm/start on different interfaces on a fanout connected to the DUT"""
@@ -29,6 +47,7 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs):
             fanouthosts(AnsibleHost) : fanout instance
             kwargs(dict):
                 peer_info(dict): keys are 'peerdevice', 'pfc_fanout_interface'. Optional: 'hwsku'
+                pfc_gen_chip_name(string) : chip name of switch where PFC frames are generated. default: None
                 pfc_queue_index(int) : queue on which the PFC storm should be generated. default: 3
                 pfc_frames_number(int) : Number of PFC frames to generate. default: 100000
                 pfc_gen_file(string): Script which generates the PFC traffic. default: 'pfc_gen.py'
@@ -43,6 +62,7 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs):
         self.fanout_hosts = fanouthosts
         self.pfc_gen_file = kwargs.pop('pfc_gen_file', "pfc_gen.py")
         self.pfc_gen_multiprocess = kwargs.pop('pfc_gen_multiprocess', False)
+        self.pfc_gen_chip_name = None
         self.pfc_queue_idx = kwargs.pop('pfc_queue_index', 3)
         self.pfc_frames_number = kwargs.pop('pfc_frames_number', 100000)
         self.send_pfc_frame_interval = kwargs.pop('send_pfc_frame_interval', 0)
@@ -125,13 +145,39 @@ def _create_pfc_gen(self):
         if not out['stat']['exists'] or not out['stat']['isdir']:
             self.peer_device.file(path=pfc_gen_fpath, state="touch")
 
+    def _get_eos_fanout_version(self):
+        """
+        Get version info for eos fanout device
+        """
+        cmd = 'Cli -c "show version"'
+        return self.peer_device.shell(cmd)['stdout_lines']
+
+    def _get_sonic_fanout_hwsku(self):
+        """
+        Get hwsku for sonic fanout device
+        """
+        cmd = 'show version'
+        out_lines = self.peer_device.shell(cmd)['stdout_lines']
+        for line in out_lines:
+            if line.startswith('HwSKU:'):
+                return line.split()[1]
+
     def deploy_pfc_gen(self):
         """
         Deploy the pfc generation file on the fanout
         """
         if self.asic_type == 'vs':
             return
         if self.peer_device.os in ('eos', 'sonic'):
+            chip_name = None
+            if self.peer_device.os == 'eos':
+                chip_name = get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])
+            elif self.peer_device.os == 'sonic':
+                chip_name = get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku())
+            if  self.peer_device.os == 'eos' and chip_name:
+                self.pfc_gen_file = "pfc_gen_brcm_xgs.py"
+                self.pfc_gen_file_test_name = "pfc_gen_brcm_xgs.py"
+                self.pfc_gen_chip_name = chip_name
             src_pfc_gen_file = "common/helpers/{}".format(self.pfc_gen_file)
             self._create_pfc_gen()
             if self.fanout_asic_type == 'mellanox':
@@ -185,6 +231,7 @@ def _update_template_args(self):
             "pfc_queue_index": self.pfc_queue_idx,
             "pfc_frames_number": self.pfc_frames_number,
             "pfc_fanout_interface": self.peer_info['pfc_fanout_interface'] if self.asic_type != 'vs' else "",
+            "pfc_gen_chip_name": self.pfc_gen_chip_name,
             "ansible_eth0_ipv4_addr": self.ip_addr,
             "peer_hwsku": self.peer_info['hwsku'] if self.asic_type != 'vs' else "",
             "send_pfc_frame_interval": self.send_pfc_frame_interval,
@@ -216,6 +263,10 @@ def _prepare_start_template(self):
         elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic':
             self.pfc_start_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_mlnx_{}.j2".format(self.peer_device.os))
+        elif ((self.peer_device.os == 'eos' and get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])) or
+              (self.peer_device.os == 'sonic' and get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku()))):
+            self.pfc_start_template = os.path.join(
+                TEMPLATES_DIR, "pfc_storm_arista_{}.j2".format(self.peer_device.os))
         elif self.asic_type == 'vs':
             self.pfc_start_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_eos.j2")
@@ -235,6 +286,10 @@ def _prepare_stop_template(self):
         elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic':
             self.pfc_stop_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_stop_mlnx_{}.j2".format(self.peer_device.os))
+        elif ((self.peer_device.os == 'eos' and get_chip_name_if_asic_pfc_storm_supported(self._get_eos_fanout_version()[0])) or
+              (self.peer_device.os == 'sonic' and get_chip_name_if_asic_pfc_storm_supported(self._get_sonic_fanout_hwsku()))):
+            self.pfc_stop_template = os.path.join(
+                TEMPLATES_DIR, "pfc_storm_stop_arista_{}.j2".format(self.peer_device.os))
         elif self.asic_type == 'vs':
             self.pfc_stop_template = os.path.join(
                 TEMPLATES_DIR, "pfc_storm_stop_eos.j2")

tests/common/templates/pfc_storm_arista_eos.j2

@@ -0,0 +1,9 @@
+bash
+cd {{pfc_gen_dir}}
+{% if (pfc_asym is defined) and (pfc_asym == True) %}
+{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{pfc_queue_index}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} > /dev/null 2>&1 &
+{% else %}
+{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{(1).__lshift__(pfc_queue_index)}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} > /dev/null 2>&1 &
+{% endif %}
+exit
+exit

tests/common/templates/pfc_storm_arista_sonic.j2

@@ -0,0 +1,6 @@
+cd {{pfc_gen_dir}}
+{% if (pfc_asym is defined) and (pfc_asym == True) %}
+nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{pfc_queue_index}} -o sonic -i {{pfc_fanout_interface}}" > /dev/null 2>&1 &
+{% else %}
+nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{(1).__lshift__(pfc_queue_index)}} -o sonic -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}" > /dev/null 2>&1 &
+{% endif %}

tests/common/templates/pfc_storm_stop_arista_eos.j2

@@ -0,0 +1,9 @@
+bash
+cd {{pfc_gen_dir}}
+{% if (pfc_asym is defined) and (pfc_asym == True) %}
+{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{pfc_queue_index}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}}' > /dev/null 2>&1 &
+{% else %}
+{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python3 {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{(1).__lshift__(pfc_queue_index)}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}' > /dev/null 2>&1 &
+{% endif %}
+exit
+exit

tests/common/templates/pfc_storm_stop_arista_sonic.j2

@@ -0,0 +1,6 @@
+cd {{pfc_gen_dir}}
+{% if (pfc_asym is defined) and (pfc_asym == True) %}
+nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{pfc_queue_index}} -o sonic -i {{pfc_fanout_interface}}'" > /dev/null 2>&1 &
+{% else %}
+nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} -c {{pfc_gen_chip_name}} -p {{(1).__lshift__(pfc_queue_index)}} -o sonic -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}'" > /dev/null 2>&1 &
+{% endif %}