diff --git a/ansible/roles/test/files/helpers/pfc_gen.py b/ansible/roles/test/files/helpers/pfc_gen.py deleted file mode 100755 index f1d53c650f3..00000000000 --- a/ansible/roles/test/files/helpers/pfc_gen.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -""" -Script to generate PFC packets. - -""" -import binascii -import sys -import os -import optparse -import logging -import logging.handlers -from socket import socket, AF_PACKET, SOCK_RAW -from struct import * - -my_logger = logging.getLogger('MyLogger') -my_logger.setLevel(logging.DEBUG) - -def checksum(msg): - s = 0 - - # loop taking 2 characters at a time - for i in range(0, len(msg), 2): - w = ord(msg[i]) + (ord(msg[i+1]) << 8 ) - s = s + w - - s = (s>>16) + (s & 0xffff); - s = s + (s >> 16); - - #complement and mask to 4 byte short - s = ~s & 0xffff - - return s - -def main(): - usage = "usage: %prog [options] arg1 arg2" - parser = optparse.OptionParser(usage=usage) - parser.add_option("-i", "--interface", type="string", dest="interface", help="Interface list to send packets, seperated by ','",metavar="Interface") - parser.add_option('-p', "--priority", type="int", dest="priority", help="PFC class enable bitmap.", metavar="Priority", default=-1) - parser.add_option("-t", "--time", type="int", dest="time", help="Pause time in quanta for global pause or enabled class",metavar="time") - parser.add_option("-n", "--num", type="int", dest="num", help="Number of packets to be sent",metavar="number",default=1) - parser.add_option("-r", "--rsyslog-server", type="string", dest="rsyslog_server", default="127.0.0.1", help="Rsyslog server IPv4 address",metavar="IPAddress") - parser.add_option('-g', "--global", action="store_true", dest="global_pf", help="Send global pause frames (not PFC)", default=False) - (options, args) = parser.parse_args() - - if options.interface is None: - print "Need to specify the interface to send PFC/global pause frame packets." - parser.print_help() - sys.exit(1) - - if options.time > 65535 or options.time < 0: - print "Quanta is not valid. Need to be in range 0-65535." - parser.print_help() - sys.exit(1) - - if options.global_pf: - # Send global pause frames - # -p option should not be set - if options.priority != -1: - print "'-p' option is not valid when sending global pause frames ('--global' / '-g')" - parser.print_help() - sys.exit(1) - elif options.priority > 255 or options.priority < 0: - print "Enable class bitmap is not valid. Need to be in range 0-255." - parser.print_help() - sys.exit(1) - - interfaces = options.interface.split(',') - - try: - sockets = [] - for i in range(0, len(interfaces)): - sockets.append(socket(AF_PACKET, SOCK_RAW)) - except: - print "Unable to create socket. Check your permissions" - sys.exit(1) - - # Configure logging - handler = logging.handlers.SysLogHandler(address = (options.rsyslog_server,514)) - my_logger.addHandler(handler) - - for s,interface in zip(sockets, interfaces): - s.bind((interface, 0)) - - """ - Set PFC defined fields and generate the packet - - The Ethernet Frame format for PFC packets is the following: - - Destination MAC | 01:80:C2:00:00:01 | - ------------------------- - Source MAC | Station MAC | - ------------------------- - Ethertype | 0x8808 | - ------------------------- - OpCode | 0x0101 | - ------------------------- - Class Enable V | 0x00 E7...E0 | - ------------------------- - Time Class 0 | 0x0000 | - ------------------------- - Time Class 1 | 0x0000 | - ------------------------- - ... - ------------------------- - Time Class 7 | 0x0000 | - ------------------------- - """ - """ - Set pause frame defined fields and generate the packet - - The Ethernet Frame format for pause frames is the following: - - Destination MAC | 01:80:C2:00:00:01 | - ------------------------- - Source MAC | Station MAC | - ------------------------- - Ethertype | 0x8808 | - ------------------------- - OpCode | 0x0001 | - ------------------------- - pause time | 0x0000 | - ------------------------- - """ - src_addr = "\x01\x02\x03\x04\x05\x06" - dst_addr = "\x01\x80\xc2\x00\x00\x01" - if options.global_pf: - opcode = "\x00\x01" - else: - opcode = "\x01\x01" - ethertype = "\x88\x08" - - packet = dst_addr + src_addr + ethertype + opcode - if options.global_pf: - packet = packet + binascii.unhexlify(format(options.time, '04x')) - else: - class_enable = options.priority - class_enable_field = binascii.unhexlify(format(class_enable, '04x')) - - packet = packet + class_enable_field - for p in range(0,7): - if (class_enable & (1< 0: - for s in sockets: - s.send(packet) - iteration -= 1 - my_logger.debug(pre_str + '_STORM_END') - -if __name__ == "__main__": - main() diff --git a/ansible/roles/test/files/helpers/pfc_gen.py b/ansible/roles/test/files/helpers/pfc_gen.py new file mode 120000 index 00000000000..7422b159913 --- /dev/null +++ b/ansible/roles/test/files/helpers/pfc_gen.py @@ -0,0 +1 @@ +../../../../../tests/common/helpers/pfc_gen.py \ No newline at end of file diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages deleted file mode 100644 index 7cff6813972..00000000000 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages +++ /dev/null @@ -1,11 +0,0 @@ -r, ".* Port counter .* not implemented" -r, ".* Port counter .* not supported" -r, ".* Invalid port counter .*" -r, ".* Unknown.*" -r, ".* SAI_STATUS_ATTR_NOT_SUPPORT.*" -r, ".* snmp.*" -r, ".* Trying to remove nonexisting queue from flex counter .*" -r, ".* SAI_STATUS_BUFFER_OVERFLOW" -r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" -r, ".* ERR syncd.*" -r, ".* syncd .* ERROR +HOST_INTERFACE" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages new file mode 120000 index 00000000000..829cff14df6 --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages @@ -0,0 +1 @@ +../../../../../../tests/pfcwd/templates/ignore_pfc_wd_messages \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_arista.j2 b/ansible/roles/test/templates/pfc_storm_arista.j2 deleted file mode 100644 index f79561c7456..00000000000 --- a/ansible/roles/test/templates/pfc_storm_arista.j2 +++ /dev/null @@ -1,9 +0,0 @@ -bash -cd /mnt/flash -{% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} & -{% else %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & -{% endif %} -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_arista.j2 b/ansible/roles/test/templates/pfc_storm_arista.j2 new file mode 120000 index 00000000000..4bb60df93aa --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_arista.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_arista.j2 \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_icos.j2 b/ansible/roles/test/templates/pfc_storm_icos.j2 deleted file mode 100644 index b1382b3be7b..00000000000 --- a/ansible/roles/test/templates/pfc_storm_icos.j2 +++ /dev/null @@ -1,5 +0,0 @@ -bash -cd /mnt/flash -sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{"fpti1_"~pfc_fanout_interface|replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_icos.j2 b/ansible/roles/test/templates/pfc_storm_icos.j2 new file mode 120000 index 00000000000..5e4cf45d9eb --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_icos.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_icos.j2 \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_mlnx.j2 b/ansible/roles/test/templates/pfc_storm_mlnx.j2 deleted file mode 100644 index 89902595a0f..00000000000 --- a/ansible/roles/test/templates/pfc_storm_mlnx.j2 +++ /dev/null @@ -1,16 +0,0 @@ -{% set container_name = "storm" %} - -enable -configure terminal - -docker exec {{ container_name }} /bin/bash -cd /root/ -{% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} & -{% else %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & -{% endif %} -exit - -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_mlnx.j2 b/ansible/roles/test/templates/pfc_storm_mlnx.j2 new file mode 120000 index 00000000000..da99c3b8dc8 --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_mlnx.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_mlnx.j2 \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_stop_arista.j2 b/ansible/roles/test/templates/pfc_storm_stop_arista.j2 deleted file mode 100644 index 63dfbcc8ff4..00000000000 --- a/ansible/roles/test/templates/pfc_storm_stop_arista.j2 +++ /dev/null @@ -1,5 +0,0 @@ -bash -cd /mnt/flash -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f "sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_stop_arista.j2 b/ansible/roles/test/templates/pfc_storm_stop_arista.j2 new file mode 120000 index 00000000000..5a5baa649f3 --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_stop_arista.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_stop_arista.j2 \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_stop_icos.j2 b/ansible/roles/test/templates/pfc_storm_stop_icos.j2 deleted file mode 100644 index a483cf255df..00000000000 --- a/ansible/roles/test/templates/pfc_storm_stop_icos.j2 +++ /dev/null @@ -1,5 +0,0 @@ -bash -cd /mnt/flash -sudo pkill -f {{pfc_gen_file}} -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_stop_icos.j2 b/ansible/roles/test/templates/pfc_storm_stop_icos.j2 new file mode 120000 index 00000000000..dddc70b3d67 --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_stop_icos.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_stop_icos.j2 \ No newline at end of file diff --git a/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 b/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 deleted file mode 100644 index b04ddaedd66..00000000000 --- a/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 +++ /dev/null @@ -1,17 +0,0 @@ -{% set container_name = "storm" %} -enable -configure terminal - -docker exec {{ container_name }} /bin/bash -cd /root/ - -{% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} -{% else %} -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} -{% endif %} - -exit - -exit -exit diff --git a/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 b/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 new file mode 120000 index 00000000000..73d73ca191b --- /dev/null +++ b/ansible/roles/test/templates/pfc_storm_stop_mlnx.j2 @@ -0,0 +1 @@ +../../../../tests/common/templates/pfc_storm_stop_mlnx.j2 \ No newline at end of file diff --git a/tests/common/errors.py b/tests/common/errors.py index be30d841170..d17528497cf 100644 --- a/tests/common/errors.py +++ b/tests/common/errors.py @@ -24,3 +24,6 @@ def __init__(self, msg, results=None): def __str__(self): return "{}\nAnsible Results => {}".format(self.message, dump_ansible_results(self.results)) + +class MissingInputError(Exception): + pass diff --git a/tests/common/helpers/pfc_gen.py b/tests/common/helpers/pfc_gen.py new file mode 100755 index 00000000000..f1d53c650f3 --- /dev/null +++ b/tests/common/helpers/pfc_gen.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python + +""" +Script to generate PFC packets. + +""" +import binascii +import sys +import os +import optparse +import logging +import logging.handlers +from socket import socket, AF_PACKET, SOCK_RAW +from struct import * + +my_logger = logging.getLogger('MyLogger') +my_logger.setLevel(logging.DEBUG) + +def checksum(msg): + s = 0 + + # loop taking 2 characters at a time + for i in range(0, len(msg), 2): + w = ord(msg[i]) + (ord(msg[i+1]) << 8 ) + s = s + w + + s = (s>>16) + (s & 0xffff); + s = s + (s >> 16); + + #complement and mask to 4 byte short + s = ~s & 0xffff + + return s + +def main(): + usage = "usage: %prog [options] arg1 arg2" + parser = optparse.OptionParser(usage=usage) + parser.add_option("-i", "--interface", type="string", dest="interface", help="Interface list to send packets, seperated by ','",metavar="Interface") + parser.add_option('-p', "--priority", type="int", dest="priority", help="PFC class enable bitmap.", metavar="Priority", default=-1) + parser.add_option("-t", "--time", type="int", dest="time", help="Pause time in quanta for global pause or enabled class",metavar="time") + parser.add_option("-n", "--num", type="int", dest="num", help="Number of packets to be sent",metavar="number",default=1) + parser.add_option("-r", "--rsyslog-server", type="string", dest="rsyslog_server", default="127.0.0.1", help="Rsyslog server IPv4 address",metavar="IPAddress") + parser.add_option('-g', "--global", action="store_true", dest="global_pf", help="Send global pause frames (not PFC)", default=False) + (options, args) = parser.parse_args() + + if options.interface is None: + print "Need to specify the interface to send PFC/global pause frame packets." + parser.print_help() + sys.exit(1) + + if options.time > 65535 or options.time < 0: + print "Quanta is not valid. Need to be in range 0-65535." + parser.print_help() + sys.exit(1) + + if options.global_pf: + # Send global pause frames + # -p option should not be set + if options.priority != -1: + print "'-p' option is not valid when sending global pause frames ('--global' / '-g')" + parser.print_help() + sys.exit(1) + elif options.priority > 255 or options.priority < 0: + print "Enable class bitmap is not valid. Need to be in range 0-255." + parser.print_help() + sys.exit(1) + + interfaces = options.interface.split(',') + + try: + sockets = [] + for i in range(0, len(interfaces)): + sockets.append(socket(AF_PACKET, SOCK_RAW)) + except: + print "Unable to create socket. Check your permissions" + sys.exit(1) + + # Configure logging + handler = logging.handlers.SysLogHandler(address = (options.rsyslog_server,514)) + my_logger.addHandler(handler) + + for s,interface in zip(sockets, interfaces): + s.bind((interface, 0)) + + """ + Set PFC defined fields and generate the packet + + The Ethernet Frame format for PFC packets is the following: + + Destination MAC | 01:80:C2:00:00:01 | + ------------------------- + Source MAC | Station MAC | + ------------------------- + Ethertype | 0x8808 | + ------------------------- + OpCode | 0x0101 | + ------------------------- + Class Enable V | 0x00 E7...E0 | + ------------------------- + Time Class 0 | 0x0000 | + ------------------------- + Time Class 1 | 0x0000 | + ------------------------- + ... + ------------------------- + Time Class 7 | 0x0000 | + ------------------------- + """ + """ + Set pause frame defined fields and generate the packet + + The Ethernet Frame format for pause frames is the following: + + Destination MAC | 01:80:C2:00:00:01 | + ------------------------- + Source MAC | Station MAC | + ------------------------- + Ethertype | 0x8808 | + ------------------------- + OpCode | 0x0001 | + ------------------------- + pause time | 0x0000 | + ------------------------- + """ + src_addr = "\x01\x02\x03\x04\x05\x06" + dst_addr = "\x01\x80\xc2\x00\x00\x01" + if options.global_pf: + opcode = "\x00\x01" + else: + opcode = "\x01\x01" + ethertype = "\x88\x08" + + packet = dst_addr + src_addr + ethertype + opcode + if options.global_pf: + packet = packet + binascii.unhexlify(format(options.time, '04x')) + else: + class_enable = options.priority + class_enable_field = binascii.unhexlify(format(class_enable, '04x')) + + packet = packet + class_enable_field + for p in range(0,7): + if (class_enable & (1< 0: + for s in sockets: + s.send(packet) + iteration -= 1 + my_logger.debug(pre_str + '_STORM_END') + +if __name__ == "__main__": + main() diff --git a/tests/common/helpers/pfc_storm.py b/tests/common/helpers/pfc_storm.py new file mode 100644 index 00000000000..ec40d9ddac9 --- /dev/null +++ b/tests/common/helpers/pfc_storm.py @@ -0,0 +1,261 @@ +import logging +import os +from common.errors import MissingInputError + +TEMPLATES_DIR = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../common/templates"))) +ANSIBLE_ROOT = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../../ansible"))) +RUN_PLAYBOOK = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../scripts/exec_template.yml")) + +logger = logging.getLogger(__name__) + + +class PFCStorm(object): + """ PFC storm/start on different interfaces on a fanout connected to the DUT""" + def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs): + """ + Args: + duthost(AnsibleHost) : dut instance + fanout_graph_facts(dict) : fixture that returns the fanouts connection info + fanouthosts(AnsibleHost) : fanout instance + kwargs(dict): + peer_info(dict): keys are 'peerdevice', 'pfc_fanout_interface'. Optional: 'hwsku' + pfc_queue_index(int) : queue on which the PFC storm should be generated. default: 3 + pfc_frames_number(int) : Number of PFC frames to generate. default: 100000 + pfc_gen_file(string): Script which generates the PFC traffic. default: 'pfc_gen.py' + Other keys: 'pfc_storm_defer_time', 'pfc_storm_stop_defer_time', 'pfc_asym' + """ + self.dut = duthost + dut_facts = duthost.setup()['ansible_facts'] + hostvars = self.dut.host.options['variable_manager']._hostvars[self.dut.hostname] + self.inventory = hostvars['inventory_file'].split('/')[-1] + self.ip_addr = dut_facts['ansible_eth0']['ipv4']['address'] + self.fanout_info = fanout_graph_facts + self.fanout_hosts = fanouthosts + self.pfc_gen_file = kwargs.pop('pfc_gen_file', "pfc_gen.py") + self.pfc_queue_idx = kwargs.pop('pfc_queue_index', 3) + self.pfc_frames_number = kwargs.pop('pfc_frames_number', 100000) + self.peer_info = kwargs.pop('peer_info') + self._validate_params(expected_args=['pfc_fanout_interface', 'peerdevice']) + if 'hwsku' not in self.peer_info: + self._populate_peer_hwsku() + self.platform_name = None + self.update_platform_name() + self._populate_optional_params(kwargs) + self.peer_device = self.fanout_hosts[self.peer_info['peerdevice']] + + def _populate_peer_hwsku(self): + """ + Find out the hwsku associated with the fanout + """ + peer_dev_info = self.fanout_info[self.peer_info['peerdevice']]['device_info'] + self.peer_info['hwsku'] = peer_dev_info['HwSku'] + + def _validate_params(self, **params): + """ + Validate if all the needed keys are present + """ + expected_args = params.get('expected_args') + peer_info_keys = self.peer_info.keys() + if not all(elem in peer_info_keys for elem in expected_args): + raise MissingInputError("Peer_info does not contain all the keys," + "Expected args: {}".format(expected_args)) + + def _populate_optional_params(self, kwargs): + """ + Create var and assign values if any the following keys are present + 'pfc_storm_defer_time', 'pfc_storm_stop_defer_time', 'pfc_asym' + """ + if len(kwargs) > 0: + self.__dict__.update(kwargs) + kwargs.clear() + + def _create_pfc_gen(self): + """ + Create the pfc generation file on the fanout if it does not exist + """ + out = self.peer_device.stat(path="/mnt/flash") + if not out['stat']['exists'] or not out['stat']['isdir']: + self.peer_device.file(path="/mnt/flash/{}".format(self.pfc_gen_file), state="touch") + + def deploy_pfc_gen(self): + """ + Deploy the pfc generation file on the fanout + """ + if 'arista' in self.peer_info['hwsku'].lower(): + self._create_pfc_gen() + self.peer_device.copy(src="common/helpers/{}".format(self.pfc_gen_file), + dest="/mnt/flash") + + def update_queue_index(self, q_idx): + """ + Update the queue index. Can be invoked after the class init to change the queue index + """ + self.pfc_queue_idx = q_idx + + def update_peer_info(self, peer_info): + """ + Update the fanout info. Can be invoked after the class init to change the fanout or fanout interface + """ + self._validate_params(expected_args=['peerdevice', 'pfc_fanout_interface']) + for key in peer_info: + self.peer_info[key] = peer_info[key] + if 'hwsku' not in peer_info: + self._populate_peer_hwsku() + self.update_platform_name() + self.peer_device = self.fanout_hosts[self.peer_info['peerdevice']] + + def update_platform_name(self): + """ + Identifies the fanout platform + """ + if 'arista' in self.peer_info['hwsku'].lower(): + self.platform_name = 'arista' + elif 'MLNX-OS' in self.peer_info['hwsku']: + self.platform_name = 'mlnx' + + def _update_template_args(self): + """ + Populates all the vars needed by the pfc storm templates + """ + self.extra_vars = dict() + self.extra_vars = { "pfc_gen_file": self.pfc_gen_file, + "pfc_queue_index": self.pfc_queue_idx, + "pfc_frames_number": self.pfc_frames_number, + "pfc_fanout_interface": self.peer_info['pfc_fanout_interface'], + "ansible_eth0_ipv4_addr": self.ip_addr, + "peer_hwsku": self.peer_info['hwsku'] + } + if getattr(self, "pfc_storm_defer_time", None): + self.extra_vars.update({"pfc_storm_defer_time": self.pfc_storm_defer_time}) + if getattr(self, "pfc_storm_stop_defer_time", None): + self.extra_vars.update({"pfc_storm_stop_defer_time": self.pfc_storm_stop_defer_time}) + if getattr(self, "pfc_asym", None): + self.extra_vars.update({"pfc_asym": self.pfc_asym}) + + def _prepare_start_template(self): + """ + Populates the pfc storm start template + """ + self._update_template_args() + self.pfc_start_template = os.path.join(TEMPLATES_DIR, "pfc_storm_{}.j2".format(self.platform_name)) + self.extra_vars.update({"template_path": self.pfc_start_template}) + + def _prepare_stop_template(self): + """ + Populates the pfc storm stop template + """ + self._update_template_args() + self.pfc_stop_template = os.path.join(TEMPLATES_DIR, "pfc_storm_stop_{}.j2".format(self.platform_name)) + self.extra_vars.update({"template_path": self.pfc_stop_template}) + + def start_storm(self): + """ + Starts PFC storm on the fanout interfaces + """ + self._prepare_start_template() + logger.info("--- Starting PFC storm on {} on interfaces {}" + .format(self.peer_info['peerdevice'], self.peer_info['pfc_fanout_interface'])) + # TODO will get rid of this ansible playbook execution option when Mellanox adds the necessary functionality + # to their onyx_config/onyx_command modules + self.peer_device.exec_template(ANSIBLE_ROOT, RUN_PLAYBOOK, self.inventory, **self.extra_vars) + + def stop_storm(self): + """ + Stops PFC storm on the fanout interfaces + """ + self._prepare_stop_template() + logger.info("--- Stopping PFC storm on {} on interfaces {}" + .format(self.peer_info['peerdevice'], self.peer_info['pfc_fanout_interface'])) + # TODO will get rid of this ansible playbook execution option when Mellanox adds the necessary functionality + # to their onyx_config/onyx_command modules + self.peer_device.exec_template(ANSIBLE_ROOT, RUN_PLAYBOOK, self.inventory, **self.extra_vars) + + +class PFCMultiStorm(object): + """ PFC storm start/stop on multiple fanouts connected to the DUT""" + def __init__(self, duthost, fanout_graph_facts, fanouthosts, peer_params): + """ + Args: + duthost(AnsibleHost) : dut instance + fanout_graph_facts(dict) : fixture that returns the fanouts connection info + fanouthosts(AnsibleHost) : fanout instance + peer_params(dict) : contains all the params needed for pfc storm + eg. peer_params = { 'peerdevice': { 'pfc_gen_file': pfc_gen_file, + 'pfc_frames_number': frame count sent on all intf in the inf_list, + 'pfc_queue_index': q_index for the pfc storm on all intf in the intf list, + 'intfs': [intf_1, intf_2] + } + } + pfc_queue_index(int) : queue on which the PFC storm should be generated. default: 4 + pfc_frames_number(int) : Number of PFC frames to generate. default: 100000000 + pfc_gen_file(string): Script which generates the PFC traffic. default: pfc_gen.py + storm_handle(dict): PFCStorm instance for each fanout connected to the DUT + """ + self.duthost = duthost + self.fanout_graph = fanout_graph_facts + self.fanouthosts = fanouthosts + self.peer_params = peer_params + self.pfc_queue_index = 4 + self.pfc_frames_number = 100000000 + self.pfc_gen_file = "pfc_gen.py" + self.storm_handle = dict() + + def _get_pfc_params(self, peer_dev): + """ + Populate the pfc params value with the ones in peer_params dict if available + + Args: + peer_dev(string): fanout name + + Returns: + q_idx(int): PFC queue where PFC storm should be generated on that fanout + frames_cnt(int): Number of PFC frames to be sent from the fanout + gen_file(string): Name of pfc storm generation script + """ + q_idx = self.pfc_queue_index + frames_cnt = self.pfc_frames_number + gen_file = self.pfc_gen_file + if 'pfc_frames_number' in self.peer_params[peer_dev]: + frames_cnt = self.peer_params[peer_dev]['pfc_frames_number'] + if 'pfc_queue_index' in self.peer_params[peer_dev]: + q_idx = self.peer_params[peer_dev]['pfc_queue_index'] + if 'pfc_gen_file' in self.peer_params[peer_dev]: + gen_file = self.peer_params[peer_dev]['pfc_gen_file'] + return q_idx, frames_cnt, gen_file + + def set_storm_params(self): + """ + Construct the peer info and deploy the pfc gen script on the fanouts + """ + for peer_dev in self.peer_params: + peer_dev_info = self.fanout_graph[peer_dev]['device_info'] + peer_info = {'peerdevice': peer_dev, + 'hwsku': peer_dev_info['HwSku'], + 'pfc_fanout_interface': self.peer_params[peer_dev]['intfs'] + } + + q_idx, frames_cnt, gen_file = self._get_pfc_params(peer_dev) + # get pfc storm handle + self.storm_handle[peer_dev] = PFCStorm(self.duthost, self.fanout_graph, + self.fanouthosts, + pfc_queue_index=q_idx, + pfc_frames_number=frames_cnt, + pfc_gen_file=gen_file, + peer_info=peer_info) + + self.storm_handle[peer_dev].deploy_pfc_gen() + + def start_pfc_storm(self): + """ + Start PFC storm on all fanouts connected to the DUT + """ + for hndle in self.storm_handle: + self.storm_handle[hndle].start_storm() + + def stop_pfc_storm(self): + """ + Stop PFC storm on all fanouts connected to the DUT + """ + for hndle in self.storm_handle: + self.storm_handle[hndle].stop_storm() + diff --git a/tests/common/templates/pfc_storm_arista.j2 b/tests/common/templates/pfc_storm_arista.j2 new file mode 100644 index 00000000000..f79561c7456 --- /dev/null +++ b/tests/common/templates/pfc_storm_arista.j2 @@ -0,0 +1,9 @@ +bash +cd /mnt/flash +{% if (pfc_asym is defined) and (pfc_asym == True) %} +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} & +{% else %} +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & +{% endif %} +exit +exit diff --git a/tests/common/templates/pfc_storm_icos.j2 b/tests/common/templates/pfc_storm_icos.j2 new file mode 100644 index 00000000000..b1382b3be7b --- /dev/null +++ b/tests/common/templates/pfc_storm_icos.j2 @@ -0,0 +1,5 @@ +bash +cd /mnt/flash +sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{"fpti1_"~pfc_fanout_interface|replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & +exit +exit diff --git a/tests/common/templates/pfc_storm_mlnx.j2 b/tests/common/templates/pfc_storm_mlnx.j2 new file mode 100644 index 00000000000..89902595a0f --- /dev/null +++ b/tests/common/templates/pfc_storm_mlnx.j2 @@ -0,0 +1,16 @@ +{% set container_name = "storm" %} + +enable +configure terminal + +docker exec {{ container_name }} /bin/bash +cd /root/ +{% if (pfc_asym is defined) and (pfc_asym == True) %} +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} & +{% else %} +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & +{% endif %} +exit + +exit +exit diff --git a/tests/common/templates/pfc_storm_stop_arista.j2 b/tests/common/templates/pfc_storm_stop_arista.j2 new file mode 100644 index 00000000000..63dfbcc8ff4 --- /dev/null +++ b/tests/common/templates/pfc_storm_stop_arista.j2 @@ -0,0 +1,5 @@ +bash +cd /mnt/flash +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f "sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} +exit +exit diff --git a/tests/common/templates/pfc_storm_stop_icos.j2 b/tests/common/templates/pfc_storm_stop_icos.j2 new file mode 100644 index 00000000000..a483cf255df --- /dev/null +++ b/tests/common/templates/pfc_storm_stop_icos.j2 @@ -0,0 +1,5 @@ +bash +cd /mnt/flash +sudo pkill -f {{pfc_gen_file}} +exit +exit diff --git a/tests/common/templates/pfc_storm_stop_mlnx.j2 b/tests/common/templates/pfc_storm_stop_mlnx.j2 new file mode 100644 index 00000000000..b04ddaedd66 --- /dev/null +++ b/tests/common/templates/pfc_storm_stop_mlnx.j2 @@ -0,0 +1,17 @@ +{% set container_name = "storm" %} +enable +configure terminal + +docker exec {{ container_name }} /bin/bash +cd /root/ + +{% if (pfc_asym is defined) and (pfc_asym == True) %} +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} +{% else %} +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} +{% endif %} + +exit + +exit +exit diff --git a/tests/pfcwd/__init__.py b/tests/pfcwd/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/pfcwd/conftest.py b/tests/pfcwd/conftest.py index f2d4ca639b3..02f5748ab40 100644 --- a/tests/pfcwd/conftest.py +++ b/tests/pfcwd/conftest.py @@ -19,6 +19,8 @@ def pytest_addoption(parser): """ parser.addoption('--warm-reboot', action='store', type=bool, default=False, help='Warm reboot needs to be enabled or not') + parser.addoption('--restore-time', action='store', type=int, default=3000, + help='PFC WD storm restore interval') @pytest.fixture(scope="module") def setup_pfc_test(duthost, ptfhost, conn_graph_facts): @@ -60,8 +62,13 @@ def setup_pfc_test(duthost, ptfhost, conn_graph_facts): selected_ports = select_test_ports(test_ports) setup_info = { 'test_ports': test_ports, + 'port_list': port_list, 'selected_test_ports': selected_ports, - 'pfc_timers' : set_pfc_timers() + 'pfc_timers' : set_pfc_timers(), + 'neighbors': neighbors, + 'vlan': {'addr': vlan_addr, + 'prefix': vlan_prefix, + 'dev': vlan_dev} } # set poll interval diff --git a/tests/pfcwd/files/pfcwd_helper.py b/tests/pfcwd/files/pfcwd_helper.py index c2fa58e9286..4ee06dea2ba 100644 --- a/tests/pfcwd/files/pfcwd_helper.py +++ b/tests/pfcwd/files/pfcwd_helper.py @@ -235,3 +235,17 @@ def select_test_ports(test_ports): selected_ports[random_port] = test_ports[random_port] return selected_ports + + +def start_wd_on_ports(duthost, port, restore_time, detect_time, action="drop"): + """ + Starts PFCwd on ports + + Args: + port (string): single port or space separated list of ports + restore_time (int): PFC storm restoration time + detect_time (int): PFC storm detection time + action (string): PFCwd action. values include 'drop', 'forward' + """ + duthost.command("pfcwd start --action {} --restoration-time {} {} {}" + .format(action, restore_time, port, detect_time)) diff --git a/tests/pfcwd/templates/ignore_pfc_wd_messages b/tests/pfcwd/templates/ignore_pfc_wd_messages new file mode 100644 index 00000000000..7cff6813972 --- /dev/null +++ b/tests/pfcwd/templates/ignore_pfc_wd_messages @@ -0,0 +1,11 @@ +r, ".* Port counter .* not implemented" +r, ".* Port counter .* not supported" +r, ".* Invalid port counter .*" +r, ".* Unknown.*" +r, ".* SAI_STATUS_ATTR_NOT_SUPPORT.*" +r, ".* snmp.*" +r, ".* Trying to remove nonexisting queue from flex counter .*" +r, ".* SAI_STATUS_BUFFER_OVERFLOW" +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" +r, ".* ERR syncd.*" +r, ".* syncd .* ERROR +HOST_INTERFACE" diff --git a/tests/pfcwd/test_pfcwd_function.py b/tests/pfcwd/test_pfcwd_function.py new file mode 100644 index 00000000000..aabe46a289d --- /dev/null +++ b/tests/pfcwd/test_pfcwd_function.py @@ -0,0 +1,519 @@ +import datetime +import logging +import os +import pytest +import time + +from common.fixtures.conn_graph_facts import fanout_graph_facts +from common.helpers.assertions import pytest_assert +from common.helpers.pfc_storm import PFCStorm +from common.plugins.loganalyzer.loganalyzer import LogAnalyzer +from files.pfcwd_helper import start_wd_on_ports +from ptf_runner import ptf_runner + +TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") +EXPECT_PFC_WD_DETECT_RE = ".* detected PFC storm .*" +EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*" +WD_ACTION_MSG_PFX = { "dontcare": "Verify PFCWD detection when queue buffer is not empty and proper function of drop action", + "drop": "Verify proper function of drop action", + "forward": "Verify proper function of forward action" + } + +pytestmark = [pytest.mark.disable_loganalyzer] + +logger = logging.getLogger(__name__) + +@pytest.fixture(scope='function', autouse=True) +def stop_pfcwd(duthost): + """ + Fixture that stops PFC Watchdog before each test run + + Args: + duthost(AnsibleHost) : dut instance + """ + logger.info("--- Stop Pfcwd --") + duthost.command("pfcwd stop") + +class PfcCmd(object): + @staticmethod + def counter_cmd(dut, queue_oid, attr): + """ + Retreive queue counters + + Args: + dut(AnsibleHost) : dut instance + queue_oid(string) : queue oid for which the counter value needs to be retreived + attr(string) : counter name + + Returns: + counter value(string) + """ + cmd = "redis-cli -n 2 HGET COUNTERS:{}" + return dut.command("{} {}".format(cmd.format(queue_oid), attr))['stdout'] + + @staticmethod + def get_queue_oid(dut, port, queue_num): + """ + Retreive queue oid + + Args: + dut(AnsibleHost) : dut instance + port(string) : port name + queue_num(int) : queue number + + Returns: + queue oid(string) + """ + cmd = "redis-cli -n 2 HGET COUNTERS_QUEUE_NAME_MAP {}:{}".format(port, queue_num) + return dut.command(cmd)['stdout'] + +class PfcPktCntrs(object): + """ PFCwd counter retrieval and verifications """ + def __init__(self, dut, action): + """ + Args: + dut(AnsibleHost) : dut instance + action(string): PFCwd action for traffic test + """ + self.dut = dut + self.action = action if action != "dontcare" else "drop" + if self.action != "forward": + self.pkt_cntrs_tx = ['PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS', 'PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST'] + self.pkt_cntrs_rx = ['PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS', 'PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST'] + self.err_msg_tx = [("Tx drop cnt check failed: Tx drop before: {} Tx drop after: {} " + "Expected (diff): {} Obtained: {}"), + "Tx drop last cnt check failed: Expected: {} Obtained: {}" + ] + self.err_msg_rx = [("Rx drop cnt check failed: Rx drop before: {} Rx drop after: {} " + "Expected (diff): {} Obtained: {}"), + "Rx drop last cnt check failed: Expected: {} Obtained: {}" + ] + else: + self.pkt_cntrs_tx = ['PFC_WD_QUEUE_STATS_TX_PACKETS', 'PFC_WD_QUEUE_STATS_TX_PACKETS_LAST'] + self.pkt_cntrs_rx = ['PFC_WD_QUEUE_STATS_RX_PACKETS', 'PFC_WD_QUEUE_STATS_RX_PACKETS_LAST'] + self.err_msg_tx = [("Tx forward cnt check failed: Tx forward before: {} Tx forward after: {} " + "Expected (diff): {} Obtained: {}"), + "Tx forward last cnt check failed: Expected: {} Obtained: {}" + ] + self.err_msg_rx = [("Rx forward cnt check failed: Rx forward before: {} Rx forward after: {} " + "Expected (diff): {} Obtained: {}"), + "Rx forward last cnt check failed: Expected: {} Obtained: {}" + ] + self.cntr_val = dict() + + def get_pkt_cnts(self, queue_oid, begin=True): + """ + Retrieves the PFCwd counter values before and after the test + + Args: + queue_oid(string) : queue oid + begin(bool) : if the counter collection is before or after the test + + """ + test_state = ['end', 'begin'] + state = test_state[begin] + self.cntr_val["tx_{}".format(state)] = int(PfcCmd.counter_cmd(self.dut, queue_oid, self.pkt_cntrs_tx[0])) + self.cntr_val["rx_{}".format(state)] = int(PfcCmd.counter_cmd(self.dut, queue_oid, self.pkt_cntrs_rx[0])) + + if not begin: + self.cntr_val["tx_last"] = int(PfcCmd.counter_cmd(self.dut, queue_oid, self.pkt_cntrs_tx[1])) + self.cntr_val["rx_last"] = int(PfcCmd.counter_cmd(self.dut, queue_oid, self.pkt_cntrs_rx[1])) + + def verify_pkt_cnts(self, port_type, pkt_cnt): + """ + Validate the packet cnts after the test + + Args: + port_type(string) : the type of port (eg. portchannel, vlan, interface) + pkt_cnt(int) : Number of test packets sent from the PTF + """ + logger.info("--- Checking Tx {} cntrs ---".format(self.action)) + tx_diff = self.cntr_val["tx_end"] - self.cntr_val["tx_begin"] + if (port_type in ['vlan', 'interface'] and tx_diff != pkt_cnt) or tx_diff <= 0: + err_msg = self.err_msg_tx[0].format(self.cntr_val["tx_begin"], self.cntr_val["tx_end"], pkt_cnt, tx_diff) + pytest_assert(err_msg) + + if (port_type in ['vlan', 'interface'] and self.cntr_val["tx_last"] != pkt_cnt) or self.cntr_val["tx_last"] <= 0: + err_msg = self.err_msg_tx[1].format(pkt_cnt, self.cntr_val["tx_last"]) + pytest_assert(err_msg) + + logger.info("--- Checking Rx {} cntrs ---".format(self.action)) + rx_diff = self.cntr_val["rx_end"] - self.cntr_val["rx_begin"] + if (port_type in ['vlan', 'interface'] and rx_diff != pkt_cnt) or rx_diff <= 0: + err_msg = self.err_msg_rx[0].format(self.cntr_val["rx_begin"], self.cntr_val["rx_end"], pkt_cnt, rx_diff) + pytest_assert(err_msg) + + if (port_type in ['vlan', 'interface'] and self.cntr_val["rx_last"] != pkt_cnt) or self.cntr_val["rx_last"] <= 0: + err_msg = self.err_msg_rx[1].format(pkt_cnt, self.cntr_val["rx_last"]) + pytest_assert(err_msg) + +class SetupPfcwdFunc(object): + """ Test setup per port """ + def setup_test_params(self, port, vlan, init=False): + """ + Sets up test parameters associated with a DUT port + + Args: + port(string) : DUT port + vlan(dict) : DUT vlan info + init(bool) : If the fanout needs to be initialized or not + """ + logger.info("--- Setting up test params for port {} ---".format(port)) + self.setup_port_params(port) + self.resolve_arp(vlan) + self.storm_setup(init=init) + + def setup_port_params(self, port): + """ + Gather all the parameters needed for storm generation and ptf test based off the DUT port + + Args: + port(string) : DUT port + """ + self.pfc_wd = dict() + self.pfc_wd['test_pkt_count'] = 100 + self.pfc_wd['queue_index'] = 4 + self.pfc_wd['frames_number'] = 100000000 + self.pfc_wd['test_port_ids'] = list() + self.peer_device = self.ports[port]['peer_device'] + self.pfc_wd['test_port'] = port + self.pfc_wd['rx_port'] = self.ports[port]['rx_port'] + self.pfc_wd['test_neighbor_addr'] = self.ports[port]['test_neighbor_addr'] + self.pfc_wd['rx_neighbor_addr'] = self.ports[port]['rx_neighbor_addr'] + self.pfc_wd['test_port_id'] = self.ports[port]['test_port_id'] + self.pfc_wd['rx_port_id'] = self.ports[port]['rx_port_id'] + self.pfc_wd['port_type'] = self.ports[port]['test_port_type'] + if self.pfc_wd['port_type'] == "portchannel": + self.pfc_wd['test_port_ids'] = self.ports[port]['test_portchannel_members'] + elif self.pfc_wd['port_type'] in ["vlan", "interface"]: + self.pfc_wd['test_port_ids'] = self.pfc_wd['test_port_id'] + self.queue_oid = PfcCmd.get_queue_oid(self.dut, port, self.pfc_wd['queue_index']) + + def resolve_arp(self, vlan): + """ + Populate ARP info for the DUT vlan port + + Args: + vlan(dict) : DUT vlan info + """ + if self.pfc_wd['port_type'] == "vlan": + self.ptf.script("./scripts/remove_ip.sh") + self.ptf.command("ifconfig eth{} {}".format(self.pfc_wd['test_port_id'], + self.pfc_wd['test_neighbor_addr'])) + self.ptf.command("ping {} -c 10".format(vlan['addr'])) + self.dut.command("docker exec -i swss arping {} -c 5".format(self.pfc_wd['test_neighbor_addr'])) + + def storm_setup(self, init=False): + """ + Prepare fanout for the storm generation + + Args: + init(bool): if the storm class needs to be initialized or not + """ + # new peer device + if not self.peer_dev_list or self.peer_device not in self.peer_dev_list: + peer_info = {'peerdevice': self.peer_device, + 'hwsku': self.fanout_info[self.peer_device]['device_info']['HwSku'], + 'pfc_fanout_interface': self.neighbors[self.pfc_wd['test_port']]['peerport'] + } + self.peer_dev_list[self.peer_device] = peer_info['hwsku'] + + # get pfc storm handle + if init: + self.storm_hndle = PFCStorm(self.dut, self.fanout_info, self.fanout, + pfc_queue_idx=self.pfc_wd['queue_index'], + pfc_frames_number=self.pfc_wd['frames_number'], + peer_info=peer_info) + self.storm_hndle.update_peer_info(peer_info) + self.storm_hndle.deploy_pfc_gen() + + # peer device already exists. only interface changes + else: + peer_info = {'peerdevice': self.peer_device, + 'hwsku': self.peer_dev_list[self.peer_device], + 'pfc_fanout_interface': self.neighbors[self.pfc_wd['test_port']]['peerport'] + } + + self.storm_hndle.update_peer_info(peer_info) + + +class SendVerifyTraffic(): + """ PTF test """ + def __init__(self, ptf, eth0_mac, pfc_params): + """ + Args: + ptf(AnsibleHost) : ptf instance + eth0_mac(string) : mac addr of eth0 + ptf_params(dict) : all PFC test params specific to the DUT port + """ + self.ptf = ptf + self.eth0_mac = eth0_mac + self.pfc_queue_index = pfc_params['queue_index'] + self.pfc_wd_test_pkt_count = pfc_params['test_pkt_count'] + self.pfc_wd_rx_port_id = pfc_params['rx_port_id'] + self.pfc_wd_test_port = pfc_params['test_port'] + self.pfc_wd_test_port_id = pfc_params['test_port_id'] + self.pfc_wd_test_port_ids = pfc_params['test_port_ids'] + self.pfc_wd_test_neighbor_addr = pfc_params['test_neighbor_addr'] + self.pfc_wd_rx_neighbor_addr = pfc_params['rx_neighbor_addr'] + self.port_type = pfc_params['port_type'] + + def verify_tx_egress(self, action): + """ + Send traffic with test port as the egress and verify if the packets get forwarded + or dropped based on the action + + Args: + action(string) : PTF test action + """ + logger.info("Check for egress {} on Tx port {}".format(action, self.pfc_wd_test_port)) + dst_port = "[" + str(self.pfc_wd_test_port_id) + "]" + if action == "forward" and type(self.pfc_wd_test_port_ids) == list: + dst_port = "".join(str(self.pfc_wd_test_port_ids)).replace(',', '') + ptf_params = {'router_mac': self.eth0_mac, + 'queue_index': self.pfc_queue_index, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_rx_port_id[0], + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_test_neighbor_addr, + 'port_type': self.port_type, + 'wd_action': action} + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file) + + def verify_rx_ingress(self, action): + """ + Send traffic with test port as the ingress and verify if the packets get forwarded + or dropped based on the action + + Args: + action(string) : PTF test action + """ + logger.info("Check for ingress {} on Rx port {}".format(action, self.pfc_wd_test_port)) + if type(self.pfc_wd_rx_port_id) == list: + dst_port = "".join(str(self.pfc_wd_rx_port_id)).replace(',', '') + else: + dst_port = "[ " + str(self.pfc_wd_rx_port_id) + " ]" + ptf_params = {'router_mac': self.eth0_mac, + 'queue_index': self.pfc_queue_index, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_test_port_id, + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_rx_neighbor_addr, + 'port_type': self.port_type, + 'wd_action': action} + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file) + + def verify_other_pfc_queue(self): + """ + Send traffic on the other PFC queue (not in storm) and verify that the packets get forwarded + """ + logger.info("Send packets via {} to verify other PFC queue is not affected".format(self.pfc_wd_test_port)) + if type(self.pfc_wd_test_port_ids) == list: + dst_port = "".join(str(self.pfc_wd_test_port_ids)).replace(',', '') + else: + dst_port = "[ " + str(self.pfc_wd_test_port_ids) + " ]" + ptf_params = {'router_mac': self.eth0_mac, + 'queue_index': self.pfc_queue_index - 1, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_rx_port_id[0], + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_test_neighbor_addr, + 'port_type': self.port_type, + 'wd_action': 'forward'} + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file) + + def verify_other_pfc_pg(self): + """ + Send traffic on the other PFC PG (not in storm) and verify that the packets get forwarded + """ + logger.info("Send packets to {} to verify other PFC pg is not affected".format(self.pfc_wd_test_port)) + if type(self.pfc_wd_rx_port_id) == list: + dst_port = "".join(str(self.pfc_wd_rx_port_id)).replace(',', '') + else: + dst_port = "[ " + str(self.pfc_wd_rx_port_id) + " ]" + ptf_params = {'router_mac': self.eth0_mac, + 'queue_index': self.pfc_queue_index - 1, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_test_port_id, + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_rx_neighbor_addr, + 'port_type': self.port_type, + 'wd_action': 'forward'} + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file) + + def fill_buffer(self): + """ + Send traffic to fill up the buffer. No verification + """ + logger.info("Send packets to {} to fill up the buffer".format(self.pfc_wd_test_port)) + ptf_params = {'router_mac': self.eth0_mac, + 'queue_index': self.pfc_queue_index, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_rx_port_id[0], + 'port_dst': "[" + str(self.pfc_wd_test_port_id) + "]", + 'ip_dst': self.pfc_wd_test_neighbor_addr, + 'port_type': self.port_type, + 'wd_action': 'dontcare'} + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file) + + def verify_wd_func(self, action): + """ + PTF traffic send and verify + + Args: + action(string) : PTF traffic test action + """ + logger.info("--- Verify PFCwd function for action {} ---".format(action)) + self.verify_tx_egress(action) + self.verify_rx_ingress(action) + self.verify_other_pfc_queue() + self.verify_other_pfc_pg() + + +class TestPfcwdFunc(SetupPfcwdFunc): + """ Test PFC function and supporting methods """ + def storm_detect_path(self, dut, port, action): + """ + Storm detection action and associated verifications + + Args: + dut(AnsibleHost) : DUT instance + port(string) : DUT port + action(string) : PTF test action + + Returns: + loganalyzer(Loganalyzer) : instance + """ + restore_time = self.timers['pfc_wd_restore_time_large'] + detect_time = self.timers['pfc_wd_detect_time'] + + loganalyzer = LogAnalyzer(ansible_host=self.dut, + marker_prefix="pfc_function_storm_detect_{}_port_{}".format(action, port)) + marker = loganalyzer.init() + ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages") + reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) + loganalyzer.ignore_regex.extend(reg_exp) + loganalyzer.expect_regex = [] + loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE]) + loganalyzer.match_regex = [] + + if action != "dontcare": + start_wd_on_ports(dut, port, restore_time, detect_time, action) + + self.storm_hndle.start_storm() + + if action == "dontcare": + self.traffic_inst.fill_buffer() + start_wd_on_ports(dut, port, restore_time, detect_time, "drop") + + time.sleep(5) + + # storm detect + logger.info("Verify if PFC storm is detected on port {}".format(port)) + loganalyzer.analyze(marker) + self.stats.get_pkt_cnts(self.queue_oid, begin=True) + # test pfcwd functionality on a storm + self.traffic_inst.verify_wd_func(action if action != "dontcare" else "drop") + return loganalyzer + + def storm_restore_path(self, loganalyzer, port, action): + """ + Storm restoration action and associated verifications + + Args: + loganalyzer(Loganalyzer) : loganalyzer instance + port(string) : DUT port + action(string) : PTF test action + """ + marker = loganalyzer.update_marker_prefix("pfc_function_storm_restore_{}_port_{}".format(action, port)) + ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages") + reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) + loganalyzer.ignore_regex.extend(reg_exp) + loganalyzer.expect_regex = [] + loganalyzer.expect_regex.extend([EXPECT_PFC_WD_RESTORE_RE]) + loganalyzer.match_regex = [] + + self.storm_hndle.stop_storm() + time.sleep(self.timers['pfc_wd_wait_for_restore_time']) + # storm restore + logger.info("Verify if PFC storm is restored on port {}".format(port)) + loganalyzer.analyze(marker) + self.stats.get_pkt_cnts(self.queue_oid, begin=False) + + def run_test(self, dut, port, action): + """ + Test method that invokes the storm detection and restoration path which includes the traffic + test and associated counter verifications + + Args: + dut(AnsibleHost) : DUT instance + port(string) : DUT port + action(string) : PTF test action + """ + logger.info("--- Storm detection path for port {} ---".format(port)) + loganalyzer = self.storm_detect_path(dut, port, action) + logger.info("--- Storm restoration path for port {} ---".format(port)) + self.storm_restore_path(loganalyzer, port, action) + logger.info("--- Verify PFCwd counters for port {} ---".format(port)) + self.stats.verify_pkt_cnts(self.pfc_wd['port_type'], self.pfc_wd['test_pkt_count']) + + def test_pfcwd_actions(self, request, setup_pfc_test, fanout_graph_facts, ptfhost, duthost, fanouthosts): + """ + PFCwd functional test + + Args: + request(object) : pytest request object + setup_pfc_test(fixture) : Module scoped autouse fixture for PFCwd + fanout_graph_facts(fixture) : fanout graph info + ptfhost(AnsibleHost) : ptf host instance + duthost(AnsibleHost) : DUT instance + fanouthosts(AnsibleHost): fanout instance + """ + setup_info = setup_pfc_test + self.fanout_info = fanout_graph_facts + self.ptf = ptfhost + self.dut = duthost + self.fanout = fanouthosts + self.timers = setup_info['pfc_timers'] + self.ports = setup_info['selected_test_ports'] + self.neighbors = setup_info['neighbors'] + dut_facts = self.dut.setup()['ansible_facts'] + self.peer_dev_list = dict() + + for idx, port in enumerate(self.ports): + logger.info("") + logger.info("--- Testing various Pfcwd actions on {} ---".format(port)) + self.setup_test_params(port, setup_info['vlan'], init=not idx) + self.traffic_inst = SendVerifyTraffic(self.ptf, dut_facts['ansible_eth0']['macaddress'], self.pfc_wd) + pfc_wd_restore_time_large = request.config.getoption("--restore-time") + # wait time before we check the logs for the 'restore' signature. 'pfc_wd_restore_time_large' is in ms. + self.timers['pfc_wd_wait_for_restore_time'] = int(pfc_wd_restore_time_large / 1000 * 2) + for action in ['dontcare', 'drop', 'forward']: + try: + self.stats = PfcPktCntrs(self.dut, action) + logger.info("{} on port {}".format(WD_ACTION_MSG_PFX[action], port)) + self.run_test(self.dut, port, action) + except Exception as e: + pytest.fail(str(e)) + + finally: + if self.storm_hndle: + logger.info("--- Stop pfc storm on port {}".format(port)) + self.storm_hndle.stop_storm() + logger.info("--- Stop PFC WD ---") + self.dut.command("pfcwd stop")