diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index c43447850c3..deebc0dd08c 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -157,7 +157,7 @@ callback_whitelist = profile_tasks # current IP information. fact_caching = jsonfile fact_caching_connection = ~/.ansible/cache -fact_caching_timeout = 600 +fact_caching_timeout = 1200 # retry files diff --git a/ansible/group_vars/sonic/sku-sensors-data.yml b/ansible/group_vars/sonic/sku-sensors-data.yml index f888b088bb1..5f32c2d7b47 100644 --- a/ansible/group_vars/sonic/sku-sensors-data.yml +++ b/ansible/group_vars/sonic/sku-sensors-data.yml @@ -49,14 +49,14 @@ sensors_checks: compares: fan: [] power: - - - SMF_S6100_ON-isa-0000/Psu1 Input/power1_input - - SMF_S6100_ON-isa-0000/Psu1 Input/power1_max - - - SMF_S6100_ON-isa-0000/Psu1 Output/power2_input - - SMF_S6100_ON-isa-0000/Psu1 Output/power2_max - - - SMF_S6100_ON-isa-0000/Psu2 Input/power3_input - - SMF_S6100_ON-isa-0000/Psu2 Input/power3_max - - - SMF_S6100_ON-isa-0000/Psu2 Output/power4_input - - SMF_S6100_ON-isa-0000/Psu2 Output/power4_max + - - SMF_S6100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_S6100_ON-isa-0000/PSU1 Input Power/power1_max + - - SMF_S6100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_S6100_ON-isa-0000/PSU1 Output Power/power2_max + - - SMF_S6100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_S6100_ON-isa-0000/PSU2 Input Power/power3_max + - - SMF_S6100_ON-isa-0000/PSU2 Output Power/power4_input + - SMF_S6100_ON-isa-0000/PSU2 Output Power/power4_max temp: - - coretemp-isa-0000/Core 0/temp2_input - coretemp-isa-0000/Core 0/temp2_crit @@ -78,10 +78,10 @@ sensors_checks: - SMF_S6100_ON-isa-0000/Front GE/temp10_crit - - SMF_S6100_ON-isa-0000/Front SFP+/temp11_input - SMF_S6100_ON-isa-0000/Front SFP+/temp11_crit - - - SMF_S6100_ON-isa-0000/PSU 1/temp14_input - - SMF_S6100_ON-isa-0000/PSU 1/temp14_crit - - - SMF_S6100_ON-isa-0000/PSU 2/temp15_input - - SMF_S6100_ON-isa-0000/PSU 2/temp15_crit + - - SMF_S6100_ON-isa-0000/PSU1 Temp/temp14_input + - SMF_S6100_ON-isa-0000/PSU1 Temp/temp14_max + - - SMF_S6100_ON-isa-0000/PSU2 Temp/temp15_input + - SMF_S6100_ON-isa-0000/PSU2 Temp/temp15_max non_zero: fan: - SMF_S6100_ON-isa-0000/Tray1 Fan1/fan1_input @@ -121,10 +121,10 @@ sensors_checks: - SMF_S6100_ON-isa-0000/PSU1 VOUT/in30_input - SMF_S6100_ON-isa-0000/PSU2 VIN/in31_input - SMF_S6100_ON-isa-0000/PSU2 VOUT/in32_input - - SMF_S6100_ON-isa-0000/Psu1 Input/power1_input - - SMF_S6100_ON-isa-0000/Psu1 Output/power2_input - - SMF_S6100_ON-isa-0000/Psu2 Input/power3_input - - SMF_S6100_ON-isa-0000/Psu2 Output/power4_input + - SMF_S6100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_S6100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_S6100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_S6100_ON-isa-0000/PSU2 Output Power/power4_input - SMF_S6100_ON-isa-0000/XP1R0V/curr21_input - SMF_S6100_ON-isa-0000/XP1R0V_ROV/curr22_input temp: @@ -139,14 +139,12 @@ sensors_checks: - SMF_S6100_ON-isa-0000/U2 Switch board?/temp9_input - SMF_S6100_ON-isa-0000/Front GE/temp10_input - SMF_S6100_ON-isa-0000/Front SFP+/temp11_input - - SMF_S6100_ON-isa-0000/BCM Internal/temp12_input - - SMF_S6100_ON-isa-0000/CPU Internal/temp13_input - - SMF_S6100_ON-isa-0000/PSU 1/temp14_input - - SMF_S6100_ON-isa-0000/PSU 2/temp15_input + - SMF_S6100_ON-isa-0000/PSU1 Temp/temp14_input + - SMF_S6100_ON-isa-0000/PSU2 Temp/temp15_input psu_skips: {} - Force10-Z9100: + Force10-Z9100-C32: alarms: fan: - SMF_Z9100_ON-isa-0000/Tray1 Fan1/fan1_alarm @@ -217,14 +215,14 @@ sensors_checks: - - coretemp-isa-0000/Core 3/temp5_input - coretemp-isa-0000/Core 3/temp5_crit power: - - - SMF_Z9100_ON-isa-0000/Psu1 Input/power1_input - - SMF_Z9100_ON-isa-0000/Psu1 Input/power1_max - - - SMF_Z9100_ON-isa-0000/Psu1 Output/power2_input - - SMF_Z9100_ON-isa-0000/Psu1 Output/power2_max - - - SMF_Z9100_ON-isa-0000/Psu2 Input/power3_input - - SMF_Z9100_ON-isa-0000/Psu2 Input/power3_max - - - SMF_Z9100_ON-isa-0000/Psu2 Output/power4_input - - SMF_Z9100_ON-isa-0000/Psu2 Output/power4_max + - - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_max + - - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_max + - - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_max + - - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_input + - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_max fan: [] non_zero: fan: @@ -241,10 +239,10 @@ sensors_checks: - SMF_Z9100_ON-isa-0000/Psu1 Fan/fan11_input - SMF_Z9100_ON-isa-0000/Psu2 Fan/fan12_input power: - - SMF_Z9100_ON-isa-0000/Psu1 Input/power1_input - - SMF_Z9100_ON-isa-0000/Psu1 Output/power2_input - - SMF_Z9100_ON-isa-0000/Psu2 Input/power3_input - - SMF_Z9100_ON-isa-0000/Psu2 Output/power4_input + - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_input - SMF_Z9100_ON-isa-0000/PSU1 VIN/in29_input - SMF_Z9100_ON-isa-0000/PSU1 VOUT/in30_input - SMF_Z9100_ON-isa-0000/PSU2 VIN/in31_input @@ -262,8 +260,135 @@ sensors_checks: - SMF_Z9100_ON-isa-0000/Front BCM On-Board (U2)/temp4_input - "SMF_Z9100_ON-isa-0000/BCM Switch On-Board #1 (U38)/temp6_input" - SMF_Z9100_ON-isa-0000/Rear (U2900)/temp9_input - - SMF_Z9100_ON-isa-0000/PSU 1/temp14_input - - SMF_Z9100_ON-isa-0000/PSU 2/temp15_input + - SMF_Z9100_ON-isa-0000/PSU1 Temp/temp14_input + - SMF_Z9100_ON-isa-0000/PSU2 Temp/temp15_input + + psu_skips: {} + + Force10-Z9100-C8D48: + alarms: + fan: + - SMF_Z9100_ON-isa-0000/Tray1 Fan1/fan1_alarm + - SMF_Z9100_ON-isa-0000/Tray1 Fan1/fan1_fault + - SMF_Z9100_ON-isa-0000/Tray1 Fan2/fan2_alarm + - SMF_Z9100_ON-isa-0000/Tray1 Fan2/fan2_fault + - SMF_Z9100_ON-isa-0000/Tray2 Fan1/fan3_alarm + - SMF_Z9100_ON-isa-0000/Tray2 Fan1/fan3_fault + - SMF_Z9100_ON-isa-0000/Tray2 Fan2/fan4_alarm + - SMF_Z9100_ON-isa-0000/Tray2 Fan2/fan4_fault + - SMF_Z9100_ON-isa-0000/Tray3 Fan1/fan5_alarm + - SMF_Z9100_ON-isa-0000/Tray3 Fan1/fan5_fault + - SMF_Z9100_ON-isa-0000/Tray3 Fan2/fan6_alarm + - SMF_Z9100_ON-isa-0000/Tray3 Fan2/fan6_fault + - SMF_Z9100_ON-isa-0000/Tray4 Fan1/fan7_alarm + - SMF_Z9100_ON-isa-0000/Tray4 Fan1/fan7_fault + - SMF_Z9100_ON-isa-0000/Tray4 Fan2/fan8_alarm + - SMF_Z9100_ON-isa-0000/Tray4 Fan2/fan8_fault + - SMF_Z9100_ON-isa-0000/Tray5 Fan1/fan9_alarm + - SMF_Z9100_ON-isa-0000/Tray5 Fan1/fan9_fault + - SMF_Z9100_ON-isa-0000/Tray5 Fan2/fan10_alarm + - SMF_Z9100_ON-isa-0000/Tray5 Fan2/fan10_fault + - SMF_Z9100_ON-isa-0000/Psu1 Fan/fan11_alarm + - SMF_Z9100_ON-isa-0000/Psu1 Fan/fan11_fault + - SMF_Z9100_ON-isa-0000/Psu2 Fan/fan12_alarm + - SMF_Z9100_ON-isa-0000/Psu2 Fan/fan12_fault + temp: + - coretemp-isa-0000/Core 0/temp2_crit_alarm + - coretemp-isa-0000/Core 1/temp3_crit_alarm + - coretemp-isa-0000/Core 2/temp4_crit_alarm + - coretemp-isa-0000/Core 3/temp5_crit_alarm + power: + - SMF_Z9100_ON-isa-0000/CPU XP3R3V_EARLY/in1_alarm + - SMF_Z9100_ON-isa-0000/CPU XP5R0V_CP/in2_alarm + - SMF_Z9100_ON-isa-0000/CPU XP3R3V_STD/in3_alarm + - SMF_Z9100_ON-isa-0000/CPU XP3R3V_CP /in4_alarm + - SMF_Z9100_ON-isa-0000/CPU XP3R3V_STD/in3_alarm + - SMF_Z9100_ON-isa-0000/CPU XP3R3V_CP /in4_alarm + - SMF_Z9100_ON-isa-0000/CPU XP0R75V_VTT_A/in5_alarm + - SMF_Z9100_ON-isa-0000/CPU XP0R75V_VTT_B/in6_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R07V_CPU/in7_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R0V_CPU/in8_alarm + - SMF_Z9100_ON-isa-0000/CPU XP12R0V/in9_alarm + - SMF_Z9100_ON-isa-0000/CPU VDDR_CPU_2/in10_alarm + - SMF_Z9100_ON-isa-0000/CPU VDDR_CPU_1/in11_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R5V_CLK/in12_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R35V_CPU/in13_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R8V_CPU/in14_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R0V_CPU_VNN/in15_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R0V_CPU_VCC/in16_alarm + - SMF_Z9100_ON-isa-0000/CPU XP1R5V_EARLY/in17_alarm + - SMF_Z9100_ON-isa-0000/SW XP3R3V_MON/in19_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R8V_MON/in20_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R25V_MON/in21_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R2V_MON/in22_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R0V_SW_MON/in23_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R0V_ROV_SW_MON/in24_alarm + - SMF_Z9100_ON-isa-0000/SW XP5V_MB_MON/in25_alarm + - SMF_Z9100_ON-isa-0000/SW XP1R8V_FPGA_MON/in26_alarm + - SMF_Z9100_ON-isa-0000/SW XP3R3V_FPGA_MON/in27_alarm + - SMF_Z9100_ON-isa-0000/SW XP3R3V_EARLY_MON/in28_alarm + + compares: + temp: + - - coretemp-isa-0000/Core 0/temp2_input + - coretemp-isa-0000/Core 0/temp2_crit + - - coretemp-isa-0000/Core 1/temp3_input + - coretemp-isa-0000/Core 1/temp3_crit + - - coretemp-isa-0000/Core 2/temp4_input + - coretemp-isa-0000/Core 2/temp4_crit + - - coretemp-isa-0000/Core 3/temp5_input + - coretemp-isa-0000/Core 3/temp5_crit + power: + - - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_max + - - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_max + - - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_max + - - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_input + - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_max + fan: [] + non_zero: + fan: + - SMF_Z9100_ON-isa-0000/Tray1 Fan1/fan1_input + - SMF_Z9100_ON-isa-0000/Tray1 Fan2/fan2_input + - SMF_Z9100_ON-isa-0000/Tray2 Fan1/fan3_input + - SMF_Z9100_ON-isa-0000/Tray2 Fan2/fan4_input + - SMF_Z9100_ON-isa-0000/Tray3 Fan1/fan5_input + - SMF_Z9100_ON-isa-0000/Tray3 Fan2/fan6_input + - SMF_Z9100_ON-isa-0000/Tray4 Fan1/fan7_input + - SMF_Z9100_ON-isa-0000/Tray4 Fan2/fan8_input + - SMF_Z9100_ON-isa-0000/Tray5 Fan1/fan9_input + - SMF_Z9100_ON-isa-0000/Tray5 Fan2/fan10_input + - SMF_Z9100_ON-isa-0000/Psu1 Fan/fan11_input + - SMF_Z9100_ON-isa-0000/Psu2 Fan/fan12_input + power: + - SMF_Z9100_ON-isa-0000/PSU1 Input Power/power1_input + - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_Z9100_ON-isa-0000/PSU1 Output Power/power2_input + - SMF_Z9100_ON-isa-0000/PSU2 Input Power/power3_input + - SMF_Z9100_ON-isa-0000/PSU2 Output Power/power4_input + - SMF_Z9100_ON-isa-0000/PSU1 VIN/in29_input + - SMF_Z9100_ON-isa-0000/PSU1 VOUT/in30_input + - SMF_Z9100_ON-isa-0000/PSU2 VIN/in31_input + - SMF_Z9100_ON-isa-0000/PSU2 VOUT/in32_input + - SMF_Z9100_ON-isa-0000/XP1R0V/curr21_input + - SMF_Z9100_ON-isa-0000/XP1R0V_ROV/curr22_input + + temp: + - coretemp-isa-0000/Core 0/temp2_input + - coretemp-isa-0000/Core 1/temp3_input + - coretemp-isa-0000/Core 2/temp4_input + - coretemp-isa-0000/Core 3/temp5_input + - SMF_Z9100_ON-isa-0000/CPU On-board (U2900)/temp1_input + - "SMF_Z9100_ON-isa-0000/BCM Switch On-Board #1 (U44)/temp2_input" + - SMF_Z9100_ON-isa-0000/Front BCM On-Board (U4)/temp3_input + - SMF_Z9100_ON-isa-0000/Front BCM On-Board (U2)/temp4_input + - "SMF_Z9100_ON-isa-0000/BCM Switch On-Board #1 (U38)/temp6_input" + - SMF_Z9100_ON-isa-0000/Rear (U2900)/temp9_input + - SMF_Z9100_ON-isa-0000/PSU1 Temp/temp14_input + - SMF_Z9100_ON-isa-0000/PSU2 Temp/temp15_input psu_skips: {} diff --git a/ansible/library/minigraph_facts.py b/ansible/library/minigraph_facts.py index 7a19490eac5..18b28e242a2 100644 --- a/ansible/library/minigraph_facts.py +++ b/ansible/library/minigraph_facts.py @@ -478,7 +478,7 @@ def parse_xml(filename, hostname): s100G_ports = [x for x in range(24, 40, 4)] + [x for x in range(88, 104, 4)] for i in s50G_ports: - alias = "etp%d" % (i / 4 + 1) + "a" if i % 4 == 0 else "b" + alias = "etp%d" % (i / 4 + 1) + ("a" if i % 4 == 0 else "b") port_alias_map[alias] = "Ethernet%d" % i for i in s100G_ports: alias = "etp%d" % (i / 4 + 1) diff --git a/ansible/linkstate/testbed_inv.py b/ansible/linkstate/testbed_inv.py index 3d3540bf463..9baa13a1798 100755 --- a/ansible/linkstate/testbed_inv.py +++ b/ansible/linkstate/testbed_inv.py @@ -25,7 +25,7 @@ def read_config(): def parse_testbed_configuration(filename, target): with open(filename) as fp: for line in fp: - if line.startswith(target): + if line.startswith(target + ','): splitted_line = line.split(",") ptf_name = splitted_line[1] topo_name = splitted_line[2] diff --git a/ansible/plugins/connection/switch.py b/ansible/plugins/connection/switch.py index 0678f170bd7..1e53641210b 100644 --- a/ansible/plugins/connection/switch.py +++ b/ansible/plugins/connection/switch.py @@ -71,7 +71,7 @@ def _spawn_connect(self): self._display.vvv("SSH: EXEC {0}".format(' '.join(cmd)), host=self.host) last_user = user - client = pexpect.spawn(' '.join(cmd), env={'TERM': 'dumb'}) + client = pexpect.spawn(' '.join(cmd), env={'TERM': 'dumb'}, timeout=60) i = client.expect(['[Pp]assword:', pexpect.EOF]) if i == 1: self._display.vvv("Server closed the connection, retry in %d seconds" % self.connection_retry_interval, host=self.host) diff --git a/ansible/roles/eos/handlers/main.yml b/ansible/roles/eos/handlers/main.yml deleted file mode 100644 index ddd945ccf1b..00000000000 --- a/ansible/roles/eos/handlers/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Restart the box - command: /sbin/shutdown -r now "Ansible updates triggered" - diff --git a/ansible/roles/eos/tasks/main.yml b/ansible/roles/eos/tasks/main.yml index 09cb3a56542..b3e432749c9 100644 --- a/ansible/roles/eos/tasks/main.yml +++ b/ansible/roles/eos/tasks/main.yml @@ -50,6 +50,22 @@ - name: build a startup config template: src="{{ topo }}-{{ props.swrole }}.j2" dest=/mnt/flash/startup-config - when: configuration is defined - notify: - - Restart the box + when: hostname in configuration + +- name: Restart the box + command: /sbin/shutdown -r now "Ansible updates triggered" + when: hostname in configuration + +- name: Pause for reboot + pause: seconds=30 + when: hostname in configuration + +- name: Wait for VM to come up + wait_for: + host: "{{ ansible_ssh_host }}" + port: 22 + state: started + delay: 10 + timeout: 600 + connection: local + when: hostname in configuration diff --git a/ansible/roles/eos/templates/t0-64-leaf.j2 b/ansible/roles/eos/templates/t0-64-leaf.j2 index bd0d6f5be58..48fbcf4e400 100644 --- a/ansible/roles/eos/templates/t0-64-leaf.j2 +++ b/ansible/roles/eos/templates/t0-64-leaf.j2 @@ -38,6 +38,8 @@ route-map DEFAULT_ROUTES permit {% for podset in range(0, props.podset_number) %} {% for tor in range(0, props.tor_number) %} {% for subnet in range(0, props.tor_subnet_number) %} +{# Skip tor 0 podset 0 #} +{% if podset != 0 or tor != 0 %} {% set suffix = ( (podset * props.tor_number * props.max_tor_subnet_number * props.tor_subnet_size) + (tor * props.max_tor_subnet_number * props.tor_subnet_size) + (subnet * props.tor_subnet_size) ) %} @@ -49,6 +51,7 @@ route-map DEFAULT_ROUTES permit {% set prefixlen_v4 = (32 - ((props.tor_subnet_size | log(2))) | int) %} ip route {{ octet1 }}.{{ octet2 }}.{{ octet3 }}.{{ octet4 }}/{{ prefixlen_v4 }} {{ props.nhipv4 }} ipv6 route {{ '20%02x' % octet1 }}:{{ '%02X%02X' % (octet2, octet3) }}:0:{{ '%02X' % octet4 }}::/64 {{ props.nhipv6 }} +{% endif %} {% endfor %} {% endfor %} {% endfor %} diff --git a/ansible/roles/fanout/tasks/fanout_mlnx.yml b/ansible/roles/fanout/tasks/fanout_mlnx.yml new file mode 100644 index 00000000000..4e2de20b6f1 --- /dev/null +++ b/ansible/roles/fanout/tasks/fanout_mlnx.yml @@ -0,0 +1,51 @@ +############################################################################################## +### playbook to deploy the fanout swtich +### Use this playbook to deploy the VLAN configurations of fanout leaf switch in SONiC testbed +### This playbook will run based on hardware flatform. Each fanout switch hardware type has its +### own unique feature and configuration command or format. Unless you use the hardware swtich +### specified in this playbook, you would need to come up with your own fanout switch deployment +### playbook +################################################################################################ + # Gather minigraph facts +- name: Gathering lab graph facts about the device + conn_graph_facts: host={{ inventory_hostname }} + connection: local + tags: always + +- name: prepare fanout switch admin login info + set_fact: ansible_ssh_user={{ fanout_admin_user }} ansible_ssh_pass={{ fanout_admin_password }} peer_hwsku={{device_info['HwSku']}} + tags: always + +- set_fact: + fanout_root_user: "user" + fanout_root_pass: "password" + tags: deploy,pfcwd_config,check_pfcwd_config + + ########################################################## + # deploy tasks to deploy default configuration on fanout # + ########################################################## +- block: + - debug: msg={{ inventory_hostname }} + - name: build fanout startup config for fanout mlnx-os-switch + action: apswitch template=mlnx_fanout.j2 + connection: switch + args: + login: "{{ switch_login['MLNX-OS'] }}" + vars: + action_variable: "deploy" + when: peer_hwsku == "MLNX-OS" + tags: deploy + + ################################################################### + # build, deploy and start docker images for the PFC WD test # + ################################################################### +- include: mlnx/deploy_pfcwd_fanout.yml + when: peer_hwsku == "MLNX-OS" + tags: deploy,pfcwd_config + + ################################################################### + # check and recover docker images for the PFC WD test # + ################################################################### +- include: mlnx/check_pfcwd_fanout.yml + when: peer_hwsku == "MLNX-OS" + tags: check_pfcwd_config diff --git a/ansible/roles/fanout/tasks/main.yml b/ansible/roles/fanout/tasks/main.yml index 610a20fd538..3ed4aae8a7e 100644 --- a/ansible/roles/fanout/tasks/main.yml +++ b/ansible/roles/fanout/tasks/main.yml @@ -1,32 +1,36 @@ -############################################################################################## -### playbook to deploy the fanout swtich -### Use this playbook to deploy the VLAN configurations of fanout switch in SONiC testbed -### This playbook will run based on hardware flatform. Each fanout switch hardware type has its -### own unique feature and configuration command or format. Unless you use the hardware swtich -### specified in this playbook, you would need to come up with your own fanout switch deployment -### playbook -################################################################################################ -# Deploy fanout switch -- name: Gathering lab graph facts about the device - conn_graph_facts: host={{ inventory_hostname }} - connection: local - -- set_fact: sw_type="{{ device_info['Type'] }}" - -- set_fact: os='eos' - when: os is not defined - -- include: fanout_eos.yml - when: os == 'eos' - -- include: fanout_sonic.yml - when: os == 'sonic' - -- block: - - set_fact: - leaf_name: "{{ inventory_hostname }}" - leaf: "{{ ansible_host }}" - - - include: rootfanout_connect.yml - deploy_leaf=true - when: sw_type == 'FanoutLeaf' +############################################################################################## +### playbook to deploy the fanout switch +### Use this playbook to deploy the VLAN configurations of fanout switch in SONiC testbed +### This playbook will run based on hardware platform. Each fanout switch hardware type has its +### own unique feature and configuration command or format. Unless you use the hardware switch +### specified in this playbook, you would need to come up with your own fanout switch deployment +### playbook +################################################################################################ +# Deploy fanout switch +- name: Gathering lab graph facts about the device + conn_graph_facts: host={{ inventory_hostname }} + connection: local + +- set_fact: sw_type="{{ device_info['Type'] }}" + +- set_fact: os='eos' + when: os is not defined + +- include: fanout_eos.yml + when: os == 'eos' + +- include: fanout_sonic.yml + when: os == 'sonic' + +- include: fanout_mlnx.yml + when: os == 'mellanox' + +- block: + - set_fact: + leaf_name: "{{ inventory_hostname }}" + leaf: "{{ ansible_host }}" + + - include: rootfanout_connect.yml + deploy_leaf=true + when: sw_type == 'FanoutLeaf' + diff --git a/ansible/roles/fanout/tasks/mlnx/check_pfcwd_fanout.yml b/ansible/roles/fanout/tasks/mlnx/check_pfcwd_fanout.yml new file mode 100644 index 00000000000..56ebefadb9e --- /dev/null +++ b/ansible/roles/fanout/tasks/mlnx/check_pfcwd_fanout.yml @@ -0,0 +1,22 @@ +############################################################################################## +### sub-playbook to deploy the docker images needed for the pfcwd test to fanout swtich +### to run separately: +### ansible-playbook -i lab fanout.yml -l ${FANOUT} --become --tags check_pfcwd_config -vvvv +################################################################################################ + +- name: Load and start dockers + action: apswitch template=mlnx_check_pfcwd_fanout.j2 + connection: switch + register: output + args: + login: "{{ switch_login['MLNX-OS'] }}" + +- set_fact: + dockers_running: "{{output.stdout|search(\"args *storm_args\")|bool}}" + dockers_installed: "{{output.stdout|search(\"pfc_storm\") and output.stdout|search(\"storm_args\")|bool}}" + +- debug: + msg: "Dockers installed{{':'}} {{dockers_installed}}" + +- debug: + msg: "Dockers running {{':'}} {{dockers_running}}" diff --git a/ansible/roles/fanout/tasks/mlnx/deploy_pfcwd_fanout.yml b/ansible/roles/fanout/tasks/mlnx/deploy_pfcwd_fanout.yml new file mode 100644 index 00000000000..ef1d47ff825 --- /dev/null +++ b/ansible/roles/fanout/tasks/mlnx/deploy_pfcwd_fanout.yml @@ -0,0 +1,54 @@ +############################################################################################## +### sub-playbook to deploy the docker images needed for the pfcwd test to fanout swtich +### to run separately: +### ansible-playbook -i lab fanout.yml -l ${FANOUT} --become --tags pfcwd_config -vvvv +### Optionally "-e pfcwd_dockers_url=" can be specified to fetch dockers without +### building them. This is useful to save time or run task in sonic-mgmt docker. +### E.g. +### ansible-playbook -i lab fanout.yml -l ${FANOUT} -e pfcwd_dockers_url=http://arc-build-server/sonic/ --become --tags pfcwd_config -vvvv +################################################################################################ + +- set_fact: + fanout_addr: "{{device_info['mgmtip']}}" + ansible_ssh_user: "{{fanout_root_user}}" + ansible_ssh_pass: "{{fanout_root_pass}}" + pfcwd_dockers: "['roles/test/files/mlnx/docker-tests-pfcgen/pfc_storm.tgz', 'roles/test/files/mlnx/docker-tests-saveargs/storm_args.tgz']" + fanout_img_path: "/var/opt/tms/images/" + +- name: Build containers to save storm arguments and to run storm + command: make + args: + chdir: "{{item | dirname}}" + with_items: pfcwd_dockers + delegate_to: localhost + when: pfcwd_dockers_url is not defined + +- name: Copy test match and ignore files to switch + copy: + src: "{{ item }}" + dest: "{{fanout_img_path}}" + with_items: pfcwd_dockers + when: pfcwd_dockers_url is not defined + +- name: Download pre-built pfcwd dockers if path specified + get_url: url={{pfcwd_dockers_url}}{{item | basename}} dest={{fanout_img_path}}/{{item | basename}} + with_items: pfcwd_dockers + when: pfcwd_dockers_url is defined + +- block: + - name: Mount FS to read-write + command: mount -o remount, rw / + + - name: Update storage driver for Docker + command: 'sed -i s/\"storage-driver\":\ \"vfs\",/\"storage-driver\":\ \"devicemapper\",\\n\ \ \ \ \"storage-opts\":\ [\\n\ \ \ \ \ \ \ \ \"dm.fs=ext4\"\\n\ \ \ \ ],\/g /opt/tms/bin/docker_config.json' + + always: + - name: Remount FS back to read-only + command: mount -r -o remount / + +- name: Load and start dockers + action: apswitch template=mlnx_deploy_pfcwd_fanout.j2 + connection: switch + args: + timeout: 600 + login: "{{ switch_login['MLNX-OS'] }}" diff --git a/ansible/roles/fanout/templates/mlnx_check_pfcwd_fanout.j2 b/ansible/roles/fanout/templates/mlnx_check_pfcwd_fanout.j2 new file mode 100644 index 00000000000..61532916d40 --- /dev/null +++ b/ansible/roles/fanout/templates/mlnx_check_pfcwd_fanout.j2 @@ -0,0 +1,4 @@ +config t + +show docker ps +show docker images diff --git a/ansible/roles/fanout/templates/mlnx_deploy_pfcwd_fanout.j2 b/ansible/roles/fanout/templates/mlnx_deploy_pfcwd_fanout.j2 new file mode 100644 index 00000000000..be7e0675f1e --- /dev/null +++ b/ansible/roles/fanout/templates/mlnx_deploy_pfcwd_fanout.j2 @@ -0,0 +1,9 @@ +config t + +docker no shutdown +ping -c 5 8.8.8.8 +docker label storm +docker load pfc_storm.tgz +docker load storm_args.tgz +docker start storm_args latest args init label storm privileged network sdk +docker start storm_args latest args now label storm privileged network sdk diff --git a/ansible/roles/fanout/templates/mlnx_fanout.j2 b/ansible/roles/fanout/templates/mlnx_fanout.j2 new file mode 100644 index 00000000000..81f36278e5f --- /dev/null +++ b/ansible/roles/fanout/templates/mlnx_fanout.j2 @@ -0,0 +1,165 @@ +{% set management_interface = 'mgmt0' %} +{% set ofpvid_present = 4096 %} +{% set vlanid_offset = 100 %} +{% set vlan_range = '101-132' %} +{% set open_flow_tableid = '0' %} +{% set flowid_offset = 160 %} +{% set last_flowid = '193' %} +{% set server_to_dut_flow_priority = '100' %} +{% set dut_to_server_flow_priority = '101' %} +{% set low_priority = '1' %} +{% set eth_typ_lldp = '0x88CC' %} +{% set eth_typ_slow = '0x8809' %} +{% set eth_typ_arp = '0x0806' %} +{% set eth_typ_vlan = '0x8100' %} +{% set eth_typ_test = '0x1234' %} +{% set MTU = '9216' %} +{% set trunk_port = '1/32' %} + +{# ----------------------------Start of default port configuration------------------------ #} + +{% set default_eth_ports = ['ERR', '1/1', '1/2', '1/3', '1/4', '1/5', '1/6', '1/7', '1/8', '1/9', '1/10', '1/11', '1/12', '1/13', '1/14', '1/15', '1/16', '1/17', '1/18', '1/19', '1/20', '1/21', '1/22', '1/23', '1/24', '1/25', '1/26', '1/27', '1/28', '1/29', '1/30', '1/31/1', '1/31/2', '1/32'] %} +{% set default_of_ports = [-1, 125, 127, 121, 123, 117, 119, 113, 115, 109, 111, 105, 107, 101, 103, 97, 99, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 94, 95] -%} +{% set uplink_port_id = 33 %} + +{# ----------------------------End of default port configuration-------------------------- #} + +{# ----------------------------Start of port speed configuration-------------------------- #} + +{% set mellanox-fanout_port_speed = [] %} + +{# -----------------------------End of speed configuration-------------------------------- #} + +{# -----------------------------Start of params dictionary-------------------------------- #} + +{% set qsfp_split_2_dict = {'mellanox-fanout':['1/31']} %} + +{% set qsfp_split_4_dict = {'mellanox-fanout':['1/9', '1/11', '1/13', '1/15', '1/17', '1/19', '1/21', '1/23']} %} + +{% set eth_ports_dict = {'mellanox-fanout':default_eth_ports} %} + +{% set of_ports_dict = {'mellanox-fanout':default_of_ports} %} + +{% set port_speed_dict = {'mellanox-fanout':mellanox-fanout_port_speed} %} + +{% set qsfp_split_2 = qsfp_split_2_dict[inventory_hostname] %} +{% set qsfp_split_4 = qsfp_split_4_dict[inventory_hostname] %} +{% set eth_ports = eth_ports_dict[inventory_hostname] %} +{% set of_ports = of_ports_dict[inventory_hostname] %} +{% set port_speed = port_speed_dict[inventory_hostname] %} + +{# ------------------------------End of params dictionary-------------------------------------- #} + +{# ------------------------------Start of fanout deploy function ------------------------------ #} + +{% macro fanout_deploy() %} +conf t +no lldp +no spanning-tree +ip routing +interface {{ management_interface }} dhcp +hostname {{ inventory_hostname }} +no fae trap-group TRAP_GROUP_LLDP policer bind +no fae trap-group TRAP_GROUP_OF_CONTROLLER policer bind + +{% for i in range(0, qsfp_split_2|length) %} +interface ethernet {{ qsfp_split_2[i] }} module-type qsfp-split-2 force +{% endfor %} + +{% for i in range(0, qsfp_split_4|length) %} +interface ethernet {{ qsfp_split_4[i] }} module-type qsfp-split-4 force +{% endfor %} + +{% for i in range(1, port_speed|length) %} +interface ethernet {{ eth_ports[i] }} speed {{ port_speed[i] }} force +{% endfor %} + +{% for i in range(1, eth_ports|length) %} +interface ethernet {{ eth_ports[i] }} shutdown +interface ethernet {{ eth_ports[i] }} mtu {{ MTU }} +interface ethernet {{ eth_ports[i] }} no shutdown +{% endfor %} + +interface ethernet {{ trunk_port }} switchport mode trunk +vlan {{ vlan_range }} +ex +interface ethernet {{ trunk_port }} switchport trunk allowed-vlan none + +{% for i in range(1,eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +interface ethernet {{ eth_ports[i] }} switchport mode dot1q-tunnel +interface ethernet {{ eth_ports[i] }} switchport access vlan {{ vlanid }} +{% endfor %} + +interface ethernet {{ trunk_port }} switchport trunk allowed-vlan add {{ vlan_range }} +interface ethernet {{ trunk_port }} switchport trunk allowed-vlan remove 1 + +protocol openflow + +{% for i in range(1, eth_ports|length) %} +interface ethernet {{ eth_ports[i] }} openflow mode hybrid +{% endfor %} + +{% set of_counter = 0 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ server_to_dut_flow_priority }},dl_type={{ eth_typ_lldp }},in_port={{ of_ports[uplink_port_id] }},dl_vlan={{ vlanid }},actions=strip_vlan,output:{{ of_ports[i] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (ofpvid_present + vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ dut_to_server_flow_priority }},dl_type={{ eth_typ_lldp }},in_port={{ of_ports[i] }},actions=push_vlan:{{ eth_typ_vlan }},set_field:{{ vlanid }}->vlan_vid,output:{{ of_ports[uplink_port_id] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ server_to_dut_flow_priority }},dl_type={{ eth_typ_slow }},in_port={{ of_ports[uplink_port_id] }},dl_vlan={{ vlanid }},actions=strip_vlan,output:{{ of_ports[i] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (ofpvid_present + vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ dut_to_server_flow_priority }},dl_type={{ eth_typ_slow }},in_port={{ of_ports[i] }},actions=push_vlan:{{ eth_typ_vlan }},set_field:{{ vlanid }}->vlan_vid,output:{{ of_ports[uplink_port_id] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ server_to_dut_flow_priority }},dl_type={{ eth_typ_arp }},in_port={{ of_ports[uplink_port_id] }},dl_vlan={{ vlanid }},actions=output:{{ of_ports[i] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ dut_to_server_flow_priority }},dl_type={{ eth_typ_arp }},in_port={{ of_ports[i] }},actions=output:{{ of_ports[uplink_port_id] }} +{% endfor %} + +{% set of_counter = of_counter + eth_ports|length-2 -%} + +{% for i in range(1, eth_ports|length-1) %} +{% set vlanid = (vlanid_offset + i)|string %} +openflow add-flows {{ of_counter + i }} table={{ open_flow_tableid }},priority={{ dut_to_server_flow_priority }},dl_type={{ eth_typ_test }},in_port={{ of_ports[i] }},actions=output:{{ of_ports[uplink_port_id] }} +{% endfor %} + +openflow add-flows {{ last_flowid }} table={{ open_flow_tableid }},priority={{ low_priority }},actions=normal + +docker +no shutdown +exit +write memory + +{% endmacro %} + +{# ------------------------------End of fanout deploy function -------------------------------- #} + +{% if action_variable == "deploy" %} + {{ fanout_deploy() }} +{% endif %} diff --git a/ansible/roles/test/files/acstests/everflow_policer_test.py b/ansible/roles/test/files/acstests/everflow_policer_test.py new file mode 100644 index 00000000000..b09e30a74e5 --- /dev/null +++ b/ansible/roles/test/files/acstests/everflow_policer_test.py @@ -0,0 +1,149 @@ +''' +Description: This file contains the EVERFLOW policer test + +Usage: Examples of how to use: + ptf --test-dir acstests everflow_policer_test.EverflowPolicerTest --platform remote -t 'router_mac="00:02:03:04:05:00";src_port="20";dst_ports="21,22";verbose=True' --relax +''' + + +import ptf +import ptf.packet as scapy +import ptf.dataplane as dataplane +import ptf.testutils as testutils +from ptf.base_tests import BaseTest +from ptf.mask import Mask + +class EverflowPolicerTest(BaseTest): + + GRE_PROTOCOL_NUMBER = 47 + NUM_OF_TOTAL_PACKETS = 200 + + + def __init__(self): + ''' + @summary: constructor + ''' + BaseTest.__init__(self) + self.test_params = testutils.test_params_get() + + + def greFilter(self, pkt_str): + ''' + @summaty: Filter GRE packets + ''' + try: + pkt = scapy.Ether(pkt_str) + + if scapy.IP not in pkt: + return False + + return pkt[scapy.IP].proto == self.GRE_PROTOCOL_NUMBER + except: + return False + + + def setUp(self): + ''' + @summary: Setup the test + ''' + print "" + + self.dataplane = ptf.dataplane_instance + self.hwsku = self.test_params['hwsku'] + self.asic_type = self.test_params['asic_type'] + self.router_mac = self.test_params['router_mac'] + self.session_src_ip = "1.1.1.1" + self.session_dst_ip = "2.2.2.2" + self.session_ttl = 1 + self.session_dscp = 8 + self.src_port = int(self.test_params['src_port']) + self.dst_mirror_ports = [int(p) for p in self.test_params['dst_mirror_ports'].split(",") if p] + self.dst_ports = [int(p) for p in self.test_params['dst_ports'].split(",")] + + self.base_pkt = testutils.simple_tcp_packet( + eth_dst = self.router_mac, + eth_src = self.dataplane.get_mac(0, 0), + ip_src = "20.0.0.1", + ip_dst = "30.0.0.1", + tcp_sport = 0x1234, + tcp_dport = 0x50, + ip_dscp = 9, + ip_ttl = 64) + + def checkOriginalFlow(self): + """ + @summary: Send traffic & check how many original packets are received + @return: count: number of original packets received + """ + exp_pkt = self.base_pkt.copy() + exp_pkt['Ethernet'].src = self.router_mac + exp_pkt['IP'].ttl = self.base_pkt['IP'].ttl - 1 + + masked_exp_pkt = Mask(exp_pkt) + masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") + + self.dataplane.flush() + + count = 0 + for i in range(0, self.NUM_OF_TOTAL_PACKETS): + testutils.send_packet(self, self.src_port, self.base_pkt) + (rcv_device, rcv_port, rcv_pkt, pkt_time) = testutils.dp_poll(self, timeout=0.1, exp_pkt=masked_exp_pkt) + if rcv_pkt is not None: + count += 1 + elif count == 0: + print "The first original packet is not recieved" + assert False # Fast failure without waiting for full iteration + print "Recieved " + str(count) + " original packets" + return count + + def checkMirroredFlow(self): + """ + @summary: Send traffic & check how many mirrored packets are received + @return: count: number of mirrored packets received + """ + exp_pkt = testutils.simple_gre_packet( + eth_src = self.router_mac, + ip_src = self.session_src_ip, + ip_dst = self.session_dst_ip, + ip_dscp = self.session_dscp, + ip_id = 0, + #ip_flags = 0x10, # need to upgrade ptf version to support it + ip_ttl = self.session_ttl, + inner_frame = self.base_pkt) + + exp_pkt['GRE'].proto = 0x88be + + masked_exp_pkt = Mask(exp_pkt) + masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") + masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "flags") + masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "chksum") + + self.dataplane.flush() + + count = 0 + for i in range(0,self.NUM_OF_TOTAL_PACKETS): + testutils.send_packet(self, self.src_port, self.base_pkt) + (rcv_device, rcv_port, rcv_pkt, pkt_time) = testutils.dp_poll(self, timeout=0.1, exp_pkt=masked_exp_pkt) + if rcv_pkt is not None: + count += 1 + elif count == 0: + print "The first mirrored packet is not recieved" + assert False # Fast failure without waiting for full iteration + print "Received " + str(count) + " mirrored packets after rate limiting" + return count + + + def runTest(self): + """ + @summary: Run EVERFLOW Policer Test + """ + + # Send traffic and verify the original traffic is not rate limited + count = self.checkOriginalFlow() + assert count == self.NUM_OF_TOTAL_PACKETS + + testutils.add_filter(self.greFilter) + + # Send traffic and verify the mirroed traffic is rate limited + count = self.checkMirroredFlow() + assert count > 100 and count < self.NUM_OF_TOTAL_PACKETS # cbs = cir = 100 diff --git a/ansible/roles/test/files/helpers/arp_responder.py b/ansible/roles/test/files/helpers/arp_responder.py index 62c52603dfc..badbee6b91f 100644 --- a/ansible/roles/test/files/helpers/arp_responder.py +++ b/ansible/roles/test/files/helpers/arp_responder.py @@ -76,6 +76,7 @@ def poll(self): class ARPResponder(object): ARP_PKT_LEN = 60 + ARP_OP_REQUEST = 1 def __init__(self, ip_sets): self.arp_chunk = binascii.unhexlify('08060001080006040002') # defines a part of the packet for ARP Reply self.arp_pad = binascii.unhexlify('00' * 18) @@ -89,7 +90,11 @@ def action(self, interface): if len(data) != self.ARP_PKT_LEN: return - remote_mac, remote_ip, request_ip = self.extract_arp_info(data) + remote_mac, remote_ip, request_ip, op_type = self.extract_arp_info(data) + + # Don't send ARP response if the ARP op code is not request + if op_type != self.ARP_OP_REQUEST: + return request_ip_str = socket.inet_ntoa(request_ip) if request_ip_str not in self.ip_sets[interface.name()]: @@ -101,7 +106,8 @@ def action(self, interface): return def extract_arp_info(self, data): - return data[6:12], data[28:32], data[38:42] # remote_mac, remote_ip, request_ip + # remote_mac, remote_ip, request_ip, op_type + return data[6:12], data[28:32], data[38:42], (ord(data[20]) * 256 + ord(data[21])) def generate_arp_reply(self, local_mac, remote_mac, local_ip, remote_ip): return remote_mac + local_mac + self.arp_chunk + local_mac + local_ip + remote_mac + remote_ip + self.arp_pad diff --git a/ansible/roles/test/files/helpers/mirror_session.py b/ansible/roles/test/files/helpers/mirror_session.py deleted file mode 100644 index 04b3f4e393a..00000000000 --- a/ansible/roles/test/files/helpers/mirror_session.py +++ /dev/null @@ -1,50 +0,0 @@ -import click -from swsssdk import ConfigDBConnector - - -@click.group() -def cli(): - pass - - -@cli.command() -@click.argument('session_name', type=click.STRING, required=True) -@click.argument('src_ip', type=click.STRING, required=True) -@click.argument('dst_ip', type=click.STRING, required=True) -@click.argument('gre_type', type=click.STRING, required=True) -@click.argument('dscp', type=click.STRING, required=True) -@click.argument('ttl', type=click.STRING, required=True) -@click.argument('queue', type=click.STRING, required=True) -def create(session_name, src_ip, dst_ip, gre_type, dscp, ttl, queue): - """ - Create mirror session. - """ - configdb = ConfigDBConnector() - configdb.connect() - - session_info = { - "src_ip": src_ip, - "dst_ip": dst_ip, - "gre_type": gre_type, - "dscp": dscp, - "ttl": ttl, - "queue": queue - } - - configdb.set_entry("MIRROR_SESSION", session_name, session_info) - - -@cli.command() -@click.argument('session_name', type=click.STRING, required=False) -def delete(session_name): - """ - Delete mirror session. - """ - configdb = ConfigDBConnector() - configdb.connect() - - configdb.set_entry("MIRROR_SESSION", session_name, None) - - -if __name__ == "__main__": - cli() diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Dockerfile b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Dockerfile new file mode 100644 index 00000000000..666d7fc18ce --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Dockerfile @@ -0,0 +1,5 @@ +FROM python:2.7-jessie + +COPY ["start.sh", "pfc_gen.py", "/root/"] + +CMD ["/root/start.sh"] diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Makefile b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Makefile new file mode 100644 index 00000000000..62c0ff80daf --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/Makefile @@ -0,0 +1,9 @@ +all: save + +build: Dockerfile + cp ../../helpers/pfc_gen.py . + docker build -t pfc_storm . + rm ./pfc_gen.py + +save: build + docker save pfc_storm:latest | gzip >pfc_storm.tgz diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen/start.sh b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/start.sh new file mode 100755 index 00000000000..250a670d687 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/root/pfc_gen.py `cat /storm/args` diff --git a/ansible/roles/test/files/mlnx/docker-tests-saveargs/Dockerfile b/ansible/roles/test/files/mlnx/docker-tests-saveargs/Dockerfile new file mode 100644 index 00000000000..9a84e876df8 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-saveargs/Dockerfile @@ -0,0 +1,3 @@ +FROM debian:jessie + +COPY ./save_args.sh /root/ diff --git a/ansible/roles/test/files/mlnx/docker-tests-saveargs/Makefile b/ansible/roles/test/files/mlnx/docker-tests-saveargs/Makefile new file mode 100644 index 00000000000..d6b834e804a --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-saveargs/Makefile @@ -0,0 +1,7 @@ +all: save + +build: Dockerfile + docker build -t storm_args . + +save: build + docker save storm_args:latest | gzip >storm_args.tgz diff --git a/ansible/roles/test/files/mlnx/docker-tests-saveargs/save_args.sh b/ansible/roles/test/files/mlnx/docker-tests-saveargs/save_args.sh new file mode 100755 index 00000000000..dab1533c029 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-saveargs/save_args.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo $@ >/storm/args diff --git a/ansible/roles/test/files/mlnx/upgrade_mlnx_fw.sh b/ansible/roles/test/files/mlnx/upgrade_mlnx_fw.sh new file mode 100644 index 00000000000..d517ef941ef --- /dev/null +++ b/ansible/roles/test/files/mlnx/upgrade_mlnx_fw.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +EXIT_FS_MOUNT_MOUNT_FAILED=111 +EXIT_FS_MOUNT_UNMOUNT_FAILED=112 +EXIT_MLNX_QUERY_FAILED=113 +EXIT_NO_FW_INFO=114 +EXIT_MLNX_FW_UPGRADE_FAILED=115 + +upgradeMLNXFW() { + FS_PATH="/host/image-${TARGET_FW#SONiC-OS-}/fs.squashfs" + FS_MOUNTPOINT="/tmp/image-${TARGET_FW#SONiC-OS-}-fs" + + mkdir -p "${FS_MOUNTPOINT}" + mount -t squashfs "${FS_PATH}" "${FS_MOUNTPOINT}" || exit ${EXIT_FS_MOUNT_FAILED} + + FW_FILE="${FS_MOUNTPOINT}/etc/mlnx/fw-SPC.mfa" + FW_QUERY="/tmp/mlnxfwmanager-query.txt" + + mlxfwmanager --query -i "${FW_FILE}" > "${FW_QUERY}" || exit ${EXIT_MLNX_QUERY_FAILED} + + FW_INFO="$(grep FW ${FW_QUERY})" + FW_CURRENT="$(echo ${FW_INFO} | cut -f2 -d' ')" + FW_AVAILABLE="$(echo ${FW_INFO} | cut -f3 -d' ')" + + [ -z "${FW_CURRENT}" -o -z "${FW_AVAILABLE}" ] && exit ${EXIT_NO_FW_INFO} + + if [ "${FW_CURRENT}" == "${FW_AVAILABLE}" ]; then + echo "Mellanox firmware is up to date" + else + echo "Mellanox firmware upgrade is required. Installing compatible version..." + mlxfwmanager -i "${FW_FILE}" -u -f -y || exit ${EXIT_MLNX_FW_UPGRADE_FAILED} + fi + + umount -rf "${FS_MOUNTPOINT}" || exit ${EXIT_FS_UNMOUNT_FAILED} + rm -rf "${FS_MOUNTPOINT}" +} + +TARGET_FW=$(sonic_installer list | grep "Next: " | cut -d ' ' -f 2) + +upgradeMLNXFW diff --git a/ansible/roles/test/files/ptftests/advanced-reboot.py b/ansible/roles/test/files/ptftests/advanced-reboot.py index 025da12a256..71943b390d0 100644 --- a/ansible/roles/test/files/ptftests/advanced-reboot.py +++ b/ansible/roles/test/files/ptftests/advanced-reboot.py @@ -62,348 +62,57 @@ import pickle from operator import itemgetter import scapy.all as scapyall +import itertools +from arista import Arista +import sad_path as sp -class Arista(object): - DEBUG = False - def __init__(self, ip, queue, test_params, login='admin', password='123456'): - self.ip = ip - self.queue = queue - self.login = login - self.password = password - self.conn = None - self.hostname = None - self.v4_routes = [test_params['vlan_ip_range'], test_params['lo_prefix']] - self.v6_routes = [test_params['lo_v6_prefix']] - self.fails = set() - self.info = set() - self.min_bgp_gr_timeout = int(test_params['min_bgp_gr_timeout']) - def __del__(self): - self.disconnect() +class StateMachine(): + def __init__(self, init_state='init'): + self.state_lock = threading.RLock() + self.state_time = {} # Recording last time when entering a state + self.state = None + self.flooding = False + self.set(init_state) - def connect(self): - self.conn = paramiko.SSHClient() - self.conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self.conn.connect(self.ip, username=self.login, password=self.password, allow_agent=False, look_for_keys=False) - self.shell = self.conn.invoke_shell() - first_prompt = self.do_cmd(None, prompt = '>') - self.hostname = self.extract_hostname(first_prompt) + def set(self, state): + with self.state_lock: + self.state = state + self.state_time[state] = datetime.datetime.now() - self.do_cmd('enable') - self.do_cmd('terminal length 0') - return self.shell + def get(self): + with self.state_lock: + cur_state = self.state + return cur_state - def extract_hostname(self, first_prompt): - lines = first_prompt.split('\n') - prompt = lines[-1] - return prompt.strip().replace('>', '#') - def do_cmd(self, cmd, prompt = None): - if prompt == None: - prompt = self.hostname + def get_state_time(self, state): + with self.state_lock: + time = self.state_time[state] + return time - if cmd is not None: - self.shell.send(cmd + '\n') - input_buffer = '' - while prompt not in input_buffer: - input_buffer += self.shell.recv(16384) + def set_flooding(self, flooding): + with self.state_lock: + self.flooding = flooding - return input_buffer - def disconnect(self): - if self.conn is not None: - self.conn.close() - self.conn = None + def is_flooding(self): + with self.state_lock: + flooding = self.flooding - return + return flooding - def run(self): - data = {} - debug_data = {} - run_once = False - log_first_line = None - quit_enabled = False - v4_routing_ok = False - v6_routing_ok = False - routing_works = True - self.connect() - - cur_time = time.time() - sample = {} - samples = {} - portchannel_output = self.do_cmd("show interfaces po1 | json") - portchannel_output = "\n".join(portchannel_output.split("\r\n")[1:-1]) - sample["po_changetime"] = json.loads(portchannel_output, strict=False)['interfaces']['Port-Channel1']['lastStatusChangeTimestamp'] - samples[cur_time] = sample - - while not (quit_enabled and v4_routing_ok and v6_routing_ok): - cmd = self.queue.get() - if cmd == 'quit': - quit_enabled = True - continue - cur_time = time.time() - info = {} - debug_info = {} - lacp_output = self.do_cmd('show lacp neighbor') - info['lacp'] = self.parse_lacp(lacp_output) - bgp_neig_output = self.do_cmd('show ip bgp neighbors') - info['bgp_neig'] = self.parse_bgp_neighbor(bgp_neig_output) - - bgp_route_v4_output = self.do_cmd('show ip route bgp | json') - v4_routing_ok = self.parse_bgp_route(bgp_route_v4_output, self.v4_routes) - info['bgp_route_v4'] = v4_routing_ok - - bgp_route_v6_output = self.do_cmd("show ipv6 route bgp | json") - v6_routing_ok = self.parse_bgp_route(bgp_route_v6_output, self.v6_routes) - info["bgp_route_v6"] = v6_routing_ok - - portchannel_output = self.do_cmd("show interfaces po1 | json") - portchannel_output = "\n".join(portchannel_output.split("\r\n")[1:-1]) - sample["po_changetime"] = json.loads(portchannel_output, strict=False)['interfaces']['Port-Channel1']['lastStatusChangeTimestamp'] - - if not run_once: - self.ipv4_gr_enabled, self.ipv6_gr_enabled, self.gr_timeout = self.parse_bgp_neighbor_once(bgp_neig_output) - if self.gr_timeout is not None: - log_first_line = "session_begins_%f" % cur_time - self.do_cmd("send log message %s" % log_first_line) - run_once = True - - data[cur_time] = info - samples[cur_time] = sample - if self.DEBUG: - debug_data[cur_time] = { - 'show lacp neighbor' : lacp_output, - 'show ip bgp neighbors' : bgp_neig_output, - 'show ip route bgp' : bgp_route_v4_output, - 'show ipv6 route bgp' : bgp_route_v6_output, - } - - attempts = 60 - for _ in range(attempts): - log_output = self.do_cmd("show log | begin %s" % log_first_line) - log_lines = log_output.split("\r\n")[1:-1] - log_data = self.parse_logs(log_lines) - if len(log_data) != 0: - break - time.sleep(1) # wait until logs are populated - - if len(log_data) == 0: - log_data['error'] = 'Incomplete output' - - self.disconnect() - - # save data for troubleshooting - with open("/tmp/%s.data.pickle" % self.ip, "w") as fp: - pickle.dump(data, fp) - - # save debug data for troubleshooting - if self.DEBUG: - with open("/tmp/%s.raw.pickle" % self.ip, "w") as fp: - pickle.dump(debug_data, fp) - with open("/tmp/%s.logging" % self.ip, "w") as fp: - fp.write("\n".join(log_lines)) - - self.check_gr_peer_status(data) - cli_data = {} - cli_data['lacp'] = self.check_series_status(data, "lacp", "LACP session") - cli_data['bgp_v4'] = self.check_series_status(data, "bgp_route_v4", "BGP v4 routes") - cli_data['bgp_v6'] = self.check_series_status(data, "bgp_route_v6", "BGP v6 routes") - cli_data['po'] = self.check_change_time(samples, "po_changetime", "PortChannel interface") - - return self.fails, self.info, cli_data, log_data - - def extract_from_logs(self, regexp, data): - raw_data = [] - result = defaultdict(list) - initial_time = -1 - re_compiled = re.compile(regexp) - for line in data: - m = re_compiled.match(line) - if not m: - continue - raw_data.append((datetime.datetime.strptime(m.group(1), "%b %d %X"), m.group(2), m.group(3))) - - if len(raw_data) > 0: - initial_time = raw_data[0][0] - for when, what, status in raw_data: - offset = (when - initial_time if when > initial_time else initial_time - when).seconds - result[what].append((offset, status)) - - return result, initial_time - - def parse_logs(self, data): - result = {} - bgp_r = r'^(\S+\s+\d+\s+\S+) \S+ Rib: %BGP-5-ADJCHANGE: peer (\S+) .+ (\S+)$' - result_bgp, initial_time_bgp = self.extract_from_logs(bgp_r, data) - if_r = r'^(\S+\s+\d+\s+\S+) \S+ Ebra: %LINEPROTO-5-UPDOWN: Line protocol on Interface (\S+), changed state to (\S+)$' - result_if, initial_time_if = self.extract_from_logs(if_r, data) - - if initial_time_bgp == -1 or initial_time_if == -1: - return result - - for events in result_bgp.values(): - if events[-1][1] != 'Established': - return result - - # first state is Idle, last state is Established - for events in result_bgp.values(): - if len(events) > 1: - assert(events[0][1] != 'Established') - - assert(events[-1][1] == 'Established') - - # first state is down, last state is up - for events in result_if.values(): - assert(events[0][1] == 'down') - assert(events[-1][1] == 'up') - - po_name = [ifname for ifname in result_if.keys() if 'Port-Channel' in ifname][0] - neigh_ipv4 = [neig_ip for neig_ip in result_bgp.keys() if '.' in neig_ip][0] - - result['PortChannel was down (seconds)'] = result_if[po_name][-1][0] - result_if[po_name][0][0] - for if_name in sorted(result_if.keys()): - result['Interface %s was down (times)' % if_name] = map(itemgetter(1), result_if[if_name]).count("down") - - for neig_ip in result_bgp.keys(): - key = "BGP IPv6 was down (seconds)" if ':' in neig_ip else "BGP IPv4 was down (seconds)" - result[key] = result_bgp[neig_ip][-1][0] - result_bgp[neig_ip][0][0] - - for neig_ip in result_bgp.keys(): - key = "BGP IPv6 was down (times)" if ':' in neig_ip else "BGP IPv4 was down (times)" - result[key] = map(itemgetter(1), result_bgp[neig_ip]).count("Idle") - - bgp_po_offset = (initial_time_if - initial_time_bgp if initial_time_if > initial_time_bgp else initial_time_bgp - initial_time_if).seconds - result['PortChannel went down after bgp session was down (seconds)'] = bgp_po_offset + result_if[po_name][0][0] - - for neig_ip in result_bgp.keys(): - key = "BGP IPv6 was gotten up after Po was up (seconds)" if ':' in neig_ip else "BGP IPv4 was gotten up after Po was up (seconds)" - result[key] = result_bgp[neig_ip][-1][0] - bgp_po_offset - result_if[po_name][-1][0] - - return result - - def parse_lacp(self, output): - return output.find('Bundled') != -1 - - def parse_bgp_neighbor_once(self, output): - is_gr_ipv4_enabled = False - is_gr_ipv6_enabled = False - restart_time = None - for line in output.split('\n'): - if ' Restart-time is' in line: - restart_time = int(line.replace(' Restart-time is ', '')) - continue - - if 'is enabled, Forwarding State is' in line: - if 'IPv6' in line: - is_gr_ipv6_enabled = True - elif 'IPv4' in line: - is_gr_ipv4_enabled = True - - return is_gr_ipv4_enabled, is_gr_ipv6_enabled, restart_time - - def parse_bgp_neighbor(self, output): - gr_active = None - gr_timer = None - for line in output.split('\n'): - if 'Restart timer is' in line: - gr_active = 'is active' in line - gr_timer = str(line[-9:-1]) - - return gr_active, gr_timer - - def parse_bgp_route(self, output, expects): - prefixes = set() - data = "\n".join(output.split("\r\n")[1:-1]) - obj = json.loads(data) - - if "vrfs" in obj and "default" in obj["vrfs"]: - obj = obj["vrfs"]["default"] - for prefix, attrs in obj["routes"].items(): - if "routeAction" not in attrs or attrs["routeAction"] != "forward": - continue - if all("Port-Channel" in via["interface"] for via in attrs["vias"]): - prefixes.add(prefix) - - return set(expects) == prefixes - - def check_gr_peer_status(self, output): - # [0] True 'ipv4_gr_enabled', [1] doesn't matter 'ipv6_enabled', [2] should be >= 120 - if not self.ipv4_gr_enabled: - self.fails.add("bgp ipv4 graceful restart is not enabled") - if not self.ipv6_gr_enabled: - pass # ToDo: - if self.gr_timeout < 120: # bgp graceful restart timeout less then 120 seconds - self.fails.add("bgp graceful restart timeout is less then 120 seconds") - - for when, other in sorted(output.items(), key = lambda x : x[0]): - gr_active, timer = other['bgp_neig'] - # wnen it's False, it's ok, wnen it's True, check that inactivity timer not less then self.min_bgp_gr_timeout seconds - if gr_active and datetime.datetime.strptime(timer, '%H:%M:%S') < datetime.datetime(1900, 1, 1, second = self.min_bgp_gr_timeout): - self.fails.add("graceful restart timer is almost finished. Less then %d seconds left" % self.min_bgp_gr_timeout) - - def check_series_status(self, output, entity, what): - # find how long anything was down - # Input parameter is a dictionary when:status - # constraints: - # entity must be down just once - # entity must be up when the test starts - # entity must be up when the test stops - - sorted_keys = sorted(output.keys()) - if not output[sorted_keys[0]][entity]: - self.fails.add("%s must be up when the test starts" % what) - return 0, 0 - if not output[sorted_keys[-1]][entity]: - self.fails.add("%s must be up when the test stops" % what) - return 0, 0 - - start = sorted_keys[0] - cur_state = True - res = defaultdict(list) - for when in sorted_keys[1:]: - if cur_state != output[when][entity]: - res[cur_state].append(when - start) - start = when - cur_state = output[when][entity] - res[cur_state].append(when - start) - - is_down_count = len(res[False]) - - if is_down_count > 1: - self.info.add("%s must be down just for once" % what) - - return is_down_count, sum(res[False]) # summary_downtime - - def check_change_time(self, output, entity, what): - # find last changing time updated, if no update, the entity is never changed - # Input parameter is a dictionary when:last_changing_time - # constraints: - # the dictionary `output` cannot be empty - sorted_keys = sorted(output.keys()) - if not output: - self.fails.add("%s cannot be empty" % what) - return 0, 0 - - start = sorted_keys[0] - prev_time = output[start] - change_count = 0 - for when in sorted_keys[1:]: - if prev_time != output[when][entity]: - prev_time = output[when][entity] - change_count += 1 - - if change_count > 0: - self.info.add("%s state changed %d times" % (what, change_count)) - - # Note: the first item is a placeholder - return 0, change_count class ReloadTest(BaseTest): TIMEOUT = 0.5 + VLAN_BASE_MAC_PATTERN = '72060001{:04}' + LAG_BASE_MAC_PATTERN = '5c010203{:04}' + SOCKET_RECV_BUFFER_SIZE = 10 * 1024 * 1024 + def __init__(self): BaseTest.__init__(self) self.fails = {} @@ -411,36 +120,47 @@ def __init__(self): self.cli_info = {} self.logs_info = {} self.log_lock = threading.RLock() + self.vm_handle = None + self.pre_handle = None self.test_params = testutils.test_params_get() - self.check_param('verbose', False, required = False) - self.check_param('dut_username', '', required = True) - self.check_param('dut_hostname', '', required = True) - self.check_param('reboot_limit_in_seconds', 30, required = False) - self.check_param('reboot_type', 'fast-reboot', required = False) - self.check_param('graceful_limit', 180, required = False) - self.check_param('portchannel_ports_file', '', required = True) - self.check_param('vlan_ports_file', '', required = True) - self.check_param('ports_file', '', required = True) - self.check_param('dut_mac', '', required = True) - self.check_param('dut_vlan_ip', '', required = True) - self.check_param('default_ip_range', '', required = True) - self.check_param('vlan_ip_range', '', required = True) - self.check_param('lo_prefix', '10.1.0.32/32', required = False) - self.check_param('lo_v6_prefix', 'fc00:1::/64', required = False) - self.check_param('arista_vms', [], required = True) - self.check_param('min_bgp_gr_timeout', 15, required = False) - self.check_param('warm_up_timeout_secs', 180, required = False) - self.check_param('dut_stabilize_secs', 20, required = False) - - self.log_file_name = '/tmp/%s.log' % self.test_params['reboot_type'] + self.check_param('verbose', False, required=False) + self.check_param('dut_username', '', required=True) + self.check_param('dut_hostname', '', required=True) + self.check_param('reboot_limit_in_seconds', 30, required=False) + self.check_param('reboot_type', 'fast-reboot', required=False) + self.check_param('graceful_limit', 180, required=False) + self.check_param('portchannel_ports_file', '', required=True) + self.check_param('vlan_ports_file', '', required=True) + self.check_param('ports_file', '', required=True) + self.check_param('dut_mac', '', required=True) + self.check_param('dut_vlan_ip', '', required=True) + self.check_param('default_ip_range', '', required=True) + self.check_param('vlan_ip_range', '', required=True) + self.check_param('lo_prefix', '10.1.0.32/32', required=False) + self.check_param('lo_v6_prefix', 'fc00:1::/64', required=False) + self.check_param('arista_vms', [], required=True) + self.check_param('min_bgp_gr_timeout', 15, required=False) + self.check_param('warm_up_timeout_secs', 300, required=False) + self.check_param('dut_stabilize_secs', 30, required=False) + self.check_param('preboot_files', None, required = False) + self.check_param('preboot_oper', None, required = False) + self.check_param('allow_vlan_flooding', False, required = False) + if not self.test_params['preboot_oper'] or self.test_params['preboot_oper'] == 'None': + self.test_params['preboot_oper'] = None + + if self.test_params['preboot_oper'] is not None: + self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.test_params['preboot_oper']) + else: + self.log_file_name = '/tmp/%s.log' % self.test_params['reboot_type'] self.log_fp = open(self.log_file_name, 'w') # Default settings - self.ping_dut_pkts = 10 - self.nr_pc_pkts = 100 - self.nr_tests = 3 - self.reboot_delay = 10 - self.task_timeout = 300 # Wait up to 5 minutes for tasks to complete + self.ping_dut_pkts = 10 + self.arp_ping_pkts = 1 + self.nr_pc_pkts = 100 + self.nr_tests = 3 + self.reboot_delay = 10 + self.task_timeout = 300 # Wait up to 5 minutes for tasks to complete self.max_nr_vl_pkts = 500 # FIXME: should be 1000. # But ptf is not fast enough + swss is slow for FDB and ARP entries insertions self.timeout_thr = None @@ -449,25 +169,30 @@ def __init__(self): # Inter-packet interval, to be used in send_in_background method. # Improve this interval to gain more precision of disruptions. self.send_interval = 0.0035 - self.packets_to_send = min(int(self.time_to_listen / (self.send_interval + 0.0015)), 45000) # How many packets to be sent in send_in_background method + self.packets_to_send = min(int(self.time_to_listen / (self.send_interval + 0.0015)), 45000) # How many packets to be sent in send_in_background method + + # Thread pool for background watching operations + self.pool = ThreadPool(processes=3) # State watcher attributes - self.cpu_state_lock = threading.RLock() - self.asic_state_lock = threading.RLock() self.watching = False - self.cpu_state = None - self.asic_state = None - self.cpu_state_time = {} # Recording last cpu state entering time + self.cpu_state = StateMachine('init') + self.asic_state = StateMachine('init') + self.vlan_state = StateMachine('init') + self.vlan_lock = threading.RLock() self.asic_state_time = {} # Recording last asic state entering time self.asic_vlan_reach = [] # Recording asic vlan reachability self.recording = False # Knob for recording asic_vlan_reach - self.set_asic_state('init') - self.set_cpu_state('init') - self.asic_flooding = False # light_probe: # True : when one direction probe fails, don't probe another. # False: when one direction probe fails, continue probe another. self.light_probe = False + # We have two data plane traffic generators which are mutualy exclusive + # one is the reachability_watcher thread + # second is the fast send_in_background + self.dataplane_io_lock = threading.Lock() + + self.allow_vlan_flooding = bool(self.test_params['allow_vlan_flooding']) return @@ -478,9 +203,9 @@ def read_json(self, name): return content def read_port_indices(self): - self.port_indices = self.read_json('ports_file') + port_indices = self.read_json('ports_file') - return + return port_indices def read_portchannel_ports(self): content = self.read_json('portchannel_ports_file') @@ -530,39 +255,147 @@ def log(self, message, verbose=False): print "%s : %s" % (current_time, message) self.log_fp.write("%s : %s\n" % (current_time, message)) - def timeout(self, seconds, message): - def timeout_exception(self, message): - self.log('Timeout is reached: %s' % message) - self.tearDown() - os.kill(os.getpid(), signal.SIGINT) + def timeout(self, func, seconds, message): + async_res = self.pool.apply_async(func) + try: + res = async_res.get(timeout=seconds) + except Exception as err: + # TimeoutError and Exception's from func + # captured here + raise type(err)(message) + return res - if self.timeout_thr is None: - self.timeout_thr = threading.Timer(seconds, timeout_exception, args=(self, message)) - self.timeout_thr.start() - else: - raise Exception("Timeout already set") + def generate_vlan_servers(self): + vlan_host_map = defaultdict(dict) + vlan_ip_range = self.test_params['vlan_ip_range'] + + _, mask = vlan_ip_range.split('/') + n_hosts = min(2**(32 - int(mask)) - 3, self.max_nr_vl_pkts) + + for counter, i in enumerate(xrange(2, n_hosts + 2)): + mac = self.VLAN_BASE_MAC_PATTERN.format(counter) + port = self.vlan_ports[i % len(self.vlan_ports)] + addr = self.host_ip(vlan_ip_range, i) - def cancel_timeout(self): - if self.timeout_thr is not None: - self.timeout_thr.cancel() - self.timeout_thr = None + vlan_host_map[port][addr] = mac + + self.nr_vl_pkts = n_hosts + + return vlan_host_map + + def generate_arp_responder_conf(self, vlan_host_map): + arp_responder_conf = {} + for port in vlan_host_map: + arp_responder_conf['eth{}'.format(port)] = vlan_host_map[port] + + return arp_responder_conf + + def dump_arp_responder_config(self, dump): + # save data for arp_replay process + filename = "/tmp/from_t1.json" if self.preboot_oper is None else "/tmp/from_t1_%s.json" % self.preboot_oper + with open(filename, "w") as fp: + json.dump(dump, fp) + + def get_peer_dev_info(self): + content = self.read_json('peer_dev_info') + for key in content.keys(): + if 'ARISTA' in key: + self.vm_dut_map[key] = dict() + self.vm_dut_map[key]['mgmt_addr'] = content[key]['mgmt_addr'] + # initialize all the port mapping + self.vm_dut_map[key]['dut_ports'] = [] + self.vm_dut_map[key]['neigh_ports'] = [] + self.vm_dut_map[key]['ptf_ports'] = [] + + def get_portchannel_info(self): + content = self.read_json('portchannel_ports_file') + for key in content.keys(): + for member in content[key]['members']: + for vm_key in self.vm_dut_map.keys(): + if member in self.vm_dut_map[vm_key]['dut_ports']: + self.vm_dut_map[vm_key]['dut_portchannel'] = key + self.vm_dut_map[vm_key]['neigh_portchannel'] = 'Port-Channel1' + break + + def get_neigh_port_info(self): + content = self.read_json('neigh_port_info') + for key in content.keys(): + if content[key]['name'] in self.vm_dut_map.keys(): + self.vm_dut_map[content[key]['name']]['dut_ports'].append(key) + self.vm_dut_map[content[key]['name']]['neigh_ports'].append(content[key]['port']) + self.vm_dut_map[content[key]['name']]['ptf_ports'].append(self.port_indices[key]) + + def build_peer_mapping(self): + ''' + Builds a map of the form + 'ARISTA01T1': {'mgmt_addr': + 'neigh_portchannel' + 'dut_portchannel' + 'neigh_ports' + 'dut_ports' + 'ptf_ports' + } + ''' + self.vm_dut_map = {} + for file in self.test_params['preboot_files'].split(','): + self.test_params[file] = '/tmp/' + file + '.json' + self.get_peer_dev_info() + self.get_neigh_port_info() + self.get_portchannel_info() def setUp(self): - self.read_port_indices() + self.fails['dut'] = set() + self.port_indices = self.read_port_indices() self.portchannel_ports = self.read_portchannel_ports() - vlan_ip_range = self.test_params['vlan_ip_range'] self.vlan_ports = self.read_vlan_ports() + if self.test_params['preboot_oper'] is not None: + self.build_peer_mapping() + + self.vlan_ip_range = self.test_params['vlan_ip_range'] + self.default_ip_range = self.test_params['default_ip_range'] self.limit = datetime.timedelta(seconds=self.test_params['reboot_limit_in_seconds']) self.reboot_type = self.test_params['reboot_type'] + self.preboot_oper = self.test_params['preboot_oper'] if self.reboot_type not in ['fast-reboot', 'warm-reboot']: raise ValueError('Not supported reboot_type %s' % self.reboot_type) self.dut_ssh = self.test_params['dut_username'] + '@' + self.test_params['dut_hostname'] self.dut_mac = self.test_params['dut_mac'] - # - self.generate_from_t1() - self.generate_from_vlan() - self.generate_ping_dut_lo() + + # get VM info + arista_vms = self.test_params['arista_vms'][1:-1].split(",") + self.ssh_targets = [] + for vm in arista_vms: + if (vm.startswith("'") or vm.startswith('"')) and (vm.endswith("'") or vm.endswith('"')): + self.ssh_targets.append(vm[1:-1]) + else: + self.ssh_targets.append(vm) + + self.log("Converted addresses VMs: %s" % str(self.ssh_targets)) + if self.preboot_oper is not None: + self.log("Preboot Operations:") + self.pre_handle = sp.PrebootTest(self.preboot_oper, self.ssh_targets, self.portchannel_ports, self.vm_dut_map, self.test_params, self.dut_ssh) + (self.ssh_targets, self.portchannel_ports, self.neigh_vm), (log_info, fails_dut, fails_vm) = self.pre_handle.setup() + self.fails['dut'] |= fails_dut + self.fails[self.neigh_vm] = fails_vm + for log in log_info: + self.log(log) + log_info, fails_dut, fails_vm = self.pre_handle.verify() + self.fails['dut'] |= fails_dut + self.fails[self.neigh_vm] |= fails_vm + for log in log_info: + self.log(log) + self.log(" ") + + self.vlan_host_map = self.generate_vlan_servers() + arp_responder_conf = self.generate_arp_responder_conf(self.vlan_host_map) + self.dump_arp_responder_config(arp_responder_conf) + + self.random_vlan = random.choice(self.vlan_ports) + self.from_server_src_port = self.random_vlan + self.from_server_src_addr = random.choice(self.vlan_host_map[self.random_vlan].keys()) + self.from_server_dst_addr = self.random_ip(self.test_params['default_ip_range']) + self.from_server_dst_ports = self.portchannel_ports self.log("Test params:") self.log("DUT ssh: %s" % self.dut_ssh) @@ -578,7 +411,13 @@ def setUp(self): self.log("Reboot type is %s" % self.reboot_type) + self.generate_from_t1() + self.generate_from_vlan() + self.generate_ping_dut_lo() + self.generate_arp_ping_packet() + if self.reboot_type == 'warm-reboot': + self.log("Preboot Oper: %s" % self.preboot_oper) # Pre-generate list of packets to be sent in send_in_background method. generate_start = datetime.datetime.now() self.generate_bidirectional() @@ -587,7 +426,7 @@ def setUp(self): self.dataplane = ptf.dataplane_instance for p in self.dataplane.ports.values(): port = p.get_packet_source() - port.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1000000) + port.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.SOCKET_RECV_BUFFER_SIZE) self.dataplane.flush() if config["log_dir"] != None: @@ -621,38 +460,34 @@ def get_mac(self, iff): SIOCGIFHWADDR = 0x8927 # Get hardware address return ':'.join(['%02x' % ord(char) for char in self.get_if(iff, SIOCGIFHWADDR)[18:24]]) + @staticmethod + def hex_to_mac(hex_mac): + return ':'.join(hex_mac[i:i+2] for i in range(0, len(hex_mac), 2)) + def generate_from_t1(self): self.from_t1 = [] - vlan_ip_range = self.test_params['vlan_ip_range'] + # for each server host create a packet destinating server IP + for counter, host_port in enumerate(self.vlan_host_map): + src_addr = self.random_ip(self.default_ip_range) + src_port = self.random_port(self.portchannel_ports) - _, mask = vlan_ip_range.split('/') - n_hosts = min(2**(32 - int(mask)) - 3, self.max_nr_vl_pkts) + for server_ip in self.vlan_host_map[host_port]: + dst_addr = server_ip - dump = defaultdict(dict) - counter = 0 - for i in xrange(2, n_hosts + 2): - from_t1_src_addr = self.random_ip(self.test_params['default_ip_range']) - from_t1_src_port = self.random_port(self.portchannel_ports) - from_t1_dst_addr = self.host_ip(vlan_ip_range, i) - from_t1_dst_port = self.vlan_ports[i % len(self.vlan_ports)] - from_t1_if_name = "eth%d" % from_t1_dst_port - from_t1_if_addr = "%s/%s" % (from_t1_dst_addr, vlan_ip_range.split('/')[1]) - vlan_mac_hex = '72060001%04x' % counter - lag_mac_hex = '5c010203%04x' % counter - mac_addr = ':'.join(lag_mac_hex[i:i+2] for i in range(0, len(lag_mac_hex), 2)) - packet = simple_tcp_packet( - eth_src=mac_addr, - eth_dst=self.dut_mac, - ip_src=from_t1_src_addr, - ip_dst=from_t1_dst_addr, - ip_ttl=255, - tcp_dport=5000 - ) - self.from_t1.append((from_t1_src_port, str(packet))) - dump[from_t1_if_name][from_t1_dst_addr] = vlan_mac_hex - counter += 1 + # generate source MAC address for traffic based on LAG_BASE_MAC_PATTERN + mac_addr = self.hex_to_mac(self.LAG_BASE_MAC_PATTERN.format(counter)) + + packet = simple_tcp_packet(eth_src=mac_addr, + eth_dst=self.dut_mac, + ip_src=src_addr, + ip_dst=dst_addr, + ip_ttl=255, + tcp_dport=5000) + self.from_t1.append((src_port, str(packet))) + + # expect any packet with dport 5000 exp_packet = simple_tcp_packet( ip_src="0.0.0.0", ip_dst="0.0.0.0", @@ -668,20 +503,6 @@ def generate_from_t1(self): self.from_t1_exp_packet.set_do_not_care_scapy(scapy.TCP, "chksum") self.from_t1_exp_packet.set_do_not_care_scapy(scapy.IP, "ttl") - # save data for arp_replay process - with open("/tmp/from_t1.json", "w") as fp: - json.dump(dump, fp) - - random_vlan_iface = random.choice(dump.keys()) - self.from_server_src_port = int(random_vlan_iface.replace('eth','')) - self.from_server_src_addr = random.choice(dump[random_vlan_iface].keys()) - self.from_server_dst_addr = self.random_ip(self.test_params['default_ip_range']) - self.from_server_dst_ports = self.portchannel_ports - - self.nr_vl_pkts = n_hosts - - return - def generate_from_vlan(self): packet = simple_tcp_packet( eth_dst=self.dut_mac, @@ -697,13 +518,11 @@ def generate_from_vlan(self): ) self.from_vlan_exp_packet = Mask(exp_packet) - self.from_vlan_exp_packet.set_do_not_care_scapy(scapy.Ether,"src") - self.from_vlan_exp_packet.set_do_not_care_scapy(scapy.Ether,"dst") + self.from_vlan_exp_packet.set_do_not_care_scapy(scapy.Ether, "src") + self.from_vlan_exp_packet.set_do_not_care_scapy(scapy.Ether, "dst") self.from_vlan_packet = str(packet) - return - def generate_ping_dut_lo(self): dut_lo_ipv4 = self.test_params['lo_prefix'].split('/')[0] packet = simple_icmp_packet(eth_dst=self.dut_mac, @@ -723,50 +542,54 @@ def generate_ping_dut_lo(self): self.ping_dut_packet = str(packet) - def generate_bidirectional(self, packets_to_send = None): + def generate_arp_ping_packet(self): + vlan_ip_range = self.test_params['vlan_ip_range'] + + vlan_port_canadiates = range(len(self.vlan_ports)) + vlan_port_canadiates.remove(0) # subnet prefix + vlan_port_canadiates.remove(1) # subnet IP on dut + src_idx = random.choice(vlan_port_canadiates) + vlan_port_canadiates.remove(src_idx) + dst_idx = random.choice(vlan_port_canadiates) + src_port = self.vlan_ports[src_idx] + dst_port = self.vlan_ports[dst_idx] + src_mac = self.get_mac('eth%d' % src_port) + src_addr = self.host_ip(vlan_ip_range, src_idx) + dst_addr = self.host_ip(vlan_ip_range, dst_idx) + packet = simple_arp_packet(eth_src=src_mac, arp_op=1, ip_snd=src_addr, ip_tgt=dst_addr, hw_snd=src_mac) + expect = simple_arp_packet(eth_dst=src_mac, arp_op=2, ip_snd=dst_addr, ip_tgt=src_addr, hw_tgt=src_mac) + self.log("ARP ping: src idx %d port %d mac %s addr %s" % (src_idx, src_port, src_mac, src_addr)) + self.log("ARP ping: dst idx %d port %d addr %s" % (dst_idx, dst_port, dst_addr)) + self.arp_ping = str(packet) + self.arp_resp = Mask(expect) + self.arp_resp.set_do_not_care_scapy(scapy.Ether, 'src') + self.arp_resp.set_do_not_care_scapy(scapy.ARP, 'hwtype') + self.arp_resp.set_do_not_care_scapy(scapy.ARP, 'hwsrc') + self.arp_src_port = src_port + + def generate_bidirectional(self): """ This method is used to pre-generate packets to be sent in background thread. Packets are composed into a list, and present a bidirectional flow as next: five packet from T1, one packet from vlan. - Each packet has sequential UDP Payload - to be identified later. + Each packet has sequential TCP Payload - to be identified later. """ - if packets_to_send: - self.packets_to_send = packets_to_send - self.send_interval = self.time_to_listen / self.packets_to_send - else: - packets_to_send = self.packets_to_send - vlan_ip_range = self.test_params['vlan_ip_range'] - _, mask = vlan_ip_range.split('/') - n_hosts = min(2**(32 - int(mask)) - 3, self.max_nr_vl_pkts) - counter = 0 - self.packets_list = list() - for i in xrange(packets_to_send): + + self.send_interval = self.time_to_listen / self.packets_to_send + self.packets_list = [] + from_t1_iter = itertools.cycle(self.from_t1) + + for i in xrange(self.packets_to_send): payload = '0' * 60 + str(i) if (i % 5) == 0 : # From vlan to T1. - packet = simple_udp_packet( - eth_dst = self.dut_mac, - ip_src = self.from_server_src_addr, - ip_dst = self.from_server_dst_addr, - udp_sport = 1234, - udp_dport = 5000, - udp_payload = payload) + packet = scapyall.Ether(self.from_vlan_packet) + packet.load = payload from_port = self.from_server_src_port else: # From T1 to vlan. - from_t1_src_addr = self.random_ip(self.test_params['default_ip_range']) - from_t1_src_port = self.random_port(self.portchannel_ports) - from_t1_dst_addr = self.host_ip(vlan_ip_range, (counter%(n_hosts-2))+2) - lag_mac_hex = '5c010203%04x' % counter - mac_addr = ':'.join(lag_mac_hex[i:i+2] for i in range(0, len(lag_mac_hex), 2)) - counter += 1 - packet = simple_udp_packet( - eth_src = mac_addr, - eth_dst = self.dut_mac, - ip_src = from_t1_src_addr, - ip_dst = from_t1_dst_addr, - ip_ttl = 255, - udp_dport = 5000, - udp_payload = payload) - from_port = from_t1_src_port + src_port, packet = next(from_t1_iter) + packet = scapyall.Ether(packet) + packet.load = payload + from_port = src_port self.packets_list.append((from_port, str(packet))) def runTest(self): @@ -774,19 +597,11 @@ def runTest(self): no_routing_start = None no_routing_stop = None no_cp_replies = None - - arista_vms = self.test_params['arista_vms'][1:-1].split(",") - ssh_targets = [] - for vm in arista_vms: - if (vm.startswith("'") or vm.startswith('"')) and (vm.endswith("'") or vm.endswith('"')): - ssh_targets.append(vm[1:-1]) - else: - ssh_targets.append(vm) - - self.log("Converted addresses VMs: %s" % str(ssh_targets)) + upper_replies = [] + routing_always = False self.ssh_jobs = [] - for addr in ssh_targets: + for addr in self.ssh_targets: q = Queue.Queue() thr = threading.Thread(target=self.peer_state_check, kwargs={'ip': addr, 'queue': q}) thr.setDaemon(True) @@ -797,32 +612,27 @@ def runTest(self): thr.setDaemon(True) try: - self.fails['dut'] = set() - - pool = ThreadPool(processes=3) self.log("Starting reachability state watch thread...") self.watching = True self.light_probe = False - watcher = pool.apply_async(self.reachability_watcher) self.watcher_is_stopped = threading.Event() # Waiter Event for the Watcher state is stopped. self.watcher_is_running = threading.Event() # Waiter Event for the Watcher state is running. self.watcher_is_stopped.set() # By default the Watcher is not running. self.watcher_is_running.clear() # By default its required to wait for the Watcher started. # Give watch thread some time to wind up + watcher = self.pool.apply_async(self.reachability_watcher) time.sleep(5) self.log("Check that device is alive and pinging") - self.fails['dut'].add('DUT is not ready for test') - self.assertTrue(self.wait_dut_to_warm_up(), 'DUT is not stable') + self.fails['dut'].add("DUT is not ready for test") + self.wait_dut_to_warm_up() self.fails['dut'].clear() self.log("Schedule to reboot the remote switch in %s sec" % self.reboot_delay) thr.start() self.log("Wait until Control plane is down") - self.timeout(self.task_timeout, "DUT hasn't shutdown in %d seconds" % self.task_timeout) - self.wait_until_cpu_port_down() - self.cancel_timeout() + self.timeout(self.wait_until_cpu_port_down, self.task_timeout, "DUT hasn't shutdown in {} seconds".format(self.task_timeout)) if self.reboot_type == 'fast-reboot': self.light_probe = True @@ -832,15 +642,15 @@ def runTest(self): if self.reboot_type == 'fast-reboot': self.log("Check that device is still forwarding data plane traffic") - self.fails['dut'].add('Data plane has a forwarding problem') - self.assertTrue(self.check_alive(), 'DUT is not stable') + self.fails['dut'].add("Data plane has a forwarding problem after CPU went down") + self.check_alive() self.fails['dut'].clear() self.log("Wait until control plane up") - async_cpu_up = pool.apply_async(self.wait_until_cpu_port_up) + async_cpu_up = self.pool.apply_async(self.wait_until_cpu_port_up) self.log("Wait until data plane stops") - async_forward_stop = pool.apply_async(self.check_forwarding_stop) + async_forward_stop = self.pool.apply_async(self.check_forwarding_stop) try: async_cpu_up.get(timeout=self.task_timeout) @@ -854,24 +664,29 @@ def runTest(self): self.log("Data plane was stopped, Waiting until it's up. Stop time: %s" % str(no_routing_start)) except TimeoutError: self.log("Data plane never stop") - no_routing_start = datetime.datetime.min + routing_always = True + upper_replies = [self.nr_vl_pkts] if no_routing_start is not None: - self.timeout(self.task_timeout, "DUT hasn't started to work for %d seconds" % self.task_timeout) - no_routing_stop, _ = self.check_forwarding_resume() - self.cancel_timeout() + no_routing_stop, _ = self.timeout(self.check_forwarding_resume, + self.task_timeout, + "DUT hasn't started to work for %d seconds" % self.task_timeout) else: no_routing_stop = datetime.datetime.min + no_routing_start = datetime.datetime.min # Stop watching DUT self.watching = False if self.reboot_type == 'warm-reboot': + self.send_and_sniff() + # Stop watching DUT self.watching = False self.log("Stopping reachability state watch thread.") self.watcher_is_stopped.wait(timeout = 10) # Wait for the Watcher stopped. - self.send_and_sniff() + + self.save_sniffed_packets() examine_start = datetime.datetime.now() self.log("Packet flow examine started %s after the reboot" % str(examine_start - self.reboot_start)) @@ -892,15 +707,16 @@ def runTest(self): for _, q in self.ssh_jobs: q.put('quit') - self.timeout(self.task_timeout, "SSH threads haven't finished for %d seconds" % self.task_timeout) - while any(thr.is_alive() for thr, _ in self.ssh_jobs): - for _, q in self.ssh_jobs: - q.put('go') - time.sleep(self.TIMEOUT) + def wait_for_ssh_threads(): + while any(thr.is_alive() for thr, _ in self.ssh_jobs): + for _, q in self.ssh_jobs: + q.put('go') + time.sleep(self.TIMEOUT) - for thr, _ in self.ssh_jobs: - thr.join() - self.cancel_timeout() + for thr, _ in self.ssh_jobs: + thr.join() + + self.timeout(wait_for_ssh_threads, self.task_timeout, "SSH threads haven't finished for %d seconds" % self.task_timeout) self.log("Data plane works again. Start time: %s" % str(no_routing_stop)) self.log("") @@ -916,10 +732,28 @@ def runTest(self): if self.reboot_type == 'fast-reboot' and no_cp_replies < 0.95 * self.nr_vl_pkts: self.fails['dut'].add("Dataplane didn't route to all servers, when control-plane was down: %d vs %d" % (no_cp_replies, self.nr_vl_pkts)) + if self.reboot_type == 'warm-reboot' and self.preboot_oper is not None: + if self.pre_handle is not None: + self.log("Postboot checks:") + log_info, fails_dut, fails_vm = self.pre_handle.verify(pre_check=False) + self.fails[self.neigh_vm] |= fails_vm + self.fails['dut'] |= fails_dut + for log in log_info: + self.log(log) + self.log(" ") + + except Exception as e: + self.fails['dut'].add(e) finally: # Stop watching DUT self.watching = False + # revert to pretest state + if self.preboot_oper is not None and self.pre_handle is not None: + self.log("Revert to preboot state:") + self.pre_handle.revert() + self.log(" ") + # Generating report self.log("="*50) self.log("Report:") @@ -940,7 +774,7 @@ def runTest(self): for ip in sorted(self.logs_info.keys()): self.log("Extracted log info from %s" % ip) for msg in sorted(self.logs_info[ip].keys()): - if msg != 'error': + if not msg in [ 'error', 'route_timeout' ]: self.log(" %s : %d" % (msg, self.logs_info[ip][msg])) else: self.log(" %s" % self.logs_info[ip][msg]) @@ -951,7 +785,8 @@ def runTest(self): if no_routing_stop: self.log("Downtime was %s" % str(no_routing_stop - no_routing_start)) - self.log("Reboot time was %s" % str(no_routing_stop - self.reboot_start)) + reboot_time = "0:00:00" if routing_always else str(no_routing_stop - self.reboot_start) + self.log("Reboot time was %s" % reboot_time) self.log("Expected downtime is less then %s" % self.limit) if self.reboot_type == 'fast-reboot' and no_cp_replies: @@ -1033,7 +868,7 @@ def wait_until_cpu_port_down(self): while True: for _, q in self.ssh_jobs: q.put('go') - if self.get_cpu_state() == 'down': + if self.cpu_state.get() == 'down': break time.sleep(self.TIMEOUT) @@ -1041,10 +876,15 @@ def wait_until_cpu_port_up(self): while True: for _, q in self.ssh_jobs: q.put('go') - if self.get_cpu_state() == 'up': + if self.cpu_state.get() == 'up': break time.sleep(self.TIMEOUT) + def apply_filter_all_ports(self, filter_expression): + for p in self.dataplane.ports.values(): + port = p.get_packet_source() + scapyall.attach_filter(port.socket, filter_expression) + def send_in_background(self, packets_list = None, interval = None): """ This method sends predefined list of packets with predefined interval. @@ -1054,26 +894,33 @@ def send_in_background(self, packets_list = None, interval = None): if not packets_list: packets_list = self.packets_list self.sniffer_started.wait(timeout=10) - sender_start = datetime.datetime.now() - self.log("Sender started at %s" % str(sender_start)) - for entry in packets_list: - time.sleep(interval) - testutils.send_packet(self, *entry) - self.log("Sender has been running for %s" % str(datetime.datetime.now() - sender_start)) + with self.dataplane_io_lock: + # While running fast data plane sender thread there are two reasons for filter to be applied + # 1. filter out data plane traffic which is tcp to free up the load on PTF socket (sniffer thread is using a different one) + # 2. during warm neighbor restoration DUT will send a lot of ARP requests which we are not interested in + # This is essential to get stable results + self.apply_filter_all_ports('not (arp and ether src {}) and not tcp'.format(self.test_params['dut_mac'])) + sender_start = datetime.datetime.now() + self.log("Sender started at %s" % str(sender_start)) + for entry in packets_list: + time.sleep(interval) + testutils.send_packet(self, *entry) + self.log("Sender has been running for %s" % str(datetime.datetime.now() - sender_start)) + # Remove filter + self.apply_filter_all_ports('') def sniff_in_background(self, wait = None): """ - This function listens on all ports, in both directions, for the UDP src=1234 dst=5000 packets, until timeout. + This function listens on all ports, in both directions, for the TCP src=1234 dst=5000 packets, until timeout. Once found, all packets are dumped to local pcap file, and all packets are saved to self.packets as scapy type. The native scapy.snif() is used as a background thread, to allow delayed start for the send_in_background(). """ if not wait: - wait = self.time_to_listen + 30 + wait = self.time_to_listen + 60 sniffer_start = datetime.datetime.now() self.log("Sniffer started at %s" % str(sniffer_start)) - filename = '/tmp/capture.pcap' - sniff_filter = "udp and udp dst port 5000 and udp src port 1234 and not icmp" + sniff_filter = "tcp and tcp dst port 5000 and tcp src port 1234 and not icmp" scapy_sniffer = threading.Thread(target=self.scapy_sniff, kwargs={'wait': wait, 'sniff_filter': sniff_filter}) scapy_sniffer.start() time.sleep(2) # Let the scapy sniff initialize completely. @@ -1081,6 +928,9 @@ def sniff_in_background(self, wait = None): scapy_sniffer.join() self.log("Sniffer has been running for %s" % str(datetime.datetime.now() - sniffer_start)) self.sniffer_started.clear() + + def save_sniffed_packets(self): + filename = "/tmp/capture_%s.pcap" % self.preboot_oper if self.preboot_oper is not None else "/tmp/capture.pcap" if self.packets: scapyall.wrpcap(filename, self.packets) self.log("Pcap file dumped to %s" % filename) @@ -1106,13 +956,13 @@ def send_and_sniff(self): self.sniff_thr.join() self.sender_thr.join() - def check_udp_payload(self, packet): + def check_tcp_payload(self, packet): """ This method is used by examine_flow() method. - It returns True if a packet is not corrupted and has a valid UDP sequential UDP Payload, as created by generate_bidirectional() method'. + It returns True if a packet is not corrupted and has a valid TCP sequential TCP Payload, as created by generate_bidirectional() method'. """ try: - int(str(packet[scapyall.UDP].payload)) in range(self.packets_to_send) + int(str(packet[scapyall.TCP].payload)) in range(self.packets_to_send) return True except Exception as err: return False @@ -1121,9 +971,9 @@ def no_flood(self, packet): """ This method filters packets which are unique (i.e. no floods). """ - if (not int(str(packet[scapyall.UDP].payload)) in self.unique_id) and (packet[scapyall.Ether].src == self.dut_mac): + if (not int(str(packet[scapyall.TCP].payload)) in self.unique_id) and (packet[scapyall.Ether].src == self.dut_mac): # This is a unique (no flooded) received packet. - self.unique_id.append(int(str(packet[scapyall.UDP].payload))) + self.unique_id.append(int(str(packet[scapyall.TCP].payload))) return True elif packet[scapyall.Ether].dst == self.dut_mac: # This is a sent packet. @@ -1134,7 +984,7 @@ def no_flood(self, packet): def examine_flow(self, filename = None): """ This method examines pcap file (if given), or self.packets scapy file. - The method compares UDP payloads of the packets one by one (assuming all payloads are consecutive integers), + The method compares TCP payloads of the packets one by one (assuming all payloads are consecutive integers), and the losses if found - are treated as disruptions in Dataplane forwarding. All disruptions are saved to self.lost_packets dictionary, in format: disrupt_start_id = (missing_packets_count, disrupt_time, disrupt_start_timestamp, disrupt_stop_timestamp) @@ -1150,15 +1000,15 @@ def examine_flow(self, filename = None): # Filter out packets and remove floods: self.unique_id = list() # This list will contain all unique Payload ID, to filter out received floods. filtered_packets = [ pkt for pkt in all_packets if - scapyall.UDP in pkt and + scapyall.TCP in pkt and not scapyall.ICMP in pkt and - pkt[scapyall.UDP].sport == 1234 and - pkt[scapyall.UDP].dport == 5000 and - self.check_udp_payload(pkt) and + pkt[scapyall.TCP].sport == 1234 and + pkt[scapyall.TCP].dport == 5000 and + self.check_tcp_payload(pkt) and self.no_flood(pkt) ] # Re-arrange packets, if delayed, by Payload ID and Timestamp: - packets = sorted(filtered_packets, key = lambda packet: (int(str(packet[scapyall.UDP].payload)), packet.time )) + packets = sorted(filtered_packets, key = lambda packet: (int(str(packet[scapyall.TCP].payload)), packet.time )) self.lost_packets = dict() self.max_disrupt, self.total_disruption = 0, 0 sent_packets = dict() @@ -1173,13 +1023,13 @@ def examine_flow(self, filename = None): for packet in packets: if packet[scapyall.Ether].dst == self.dut_mac: # This is a sent packet - keep track of it as payload_id:timestamp. - sent_payload = int(str(packet[scapyall.UDP].payload)) + sent_payload = int(str(packet[scapyall.TCP].payload)) sent_packets[sent_payload] = packet.time continue if packet[scapyall.Ether].src == self.dut_mac: # This is a received packet. received_time = packet.time - received_payload = int(str(packet[scapyall.UDP].payload)) + received_payload = int(str(packet[scapyall.TCP].payload)) received_counter += 1 if not (received_payload and received_time): # This is the first valid received packet. @@ -1214,7 +1064,7 @@ def examine_flow(self, filename = None): self.log("Gaps in forwarding not found.") self.log("Total incoming packets captured %d" % received_counter) if packets: - filename = '/tmp/capture_filtered.pcap' + filename = '/tmp/capture_filtered.pcap' if self.preboot_oper is None else "/tmp/capture_filtered_%s.pcap" % self.preboot_oper scapyall.wrpcap(filename, packets) self.log("Filtered pcap dumped to %s" % filename) @@ -1222,7 +1072,7 @@ def check_forwarding_stop(self): self.asic_start_recording_vlan_reachability() while True: - state = self.get_asic_state() + state = self.asic_state.get() for _, q in self.ssh_jobs: q.put('go') if state == 'down': @@ -1231,16 +1081,16 @@ def check_forwarding_stop(self): self.asic_stop_recording_vlan_reachability() - return self.get_asic_state_time(state), self.get_asic_vlan_reachability() + return self.asic_state.get_state_time(state), self.get_asic_vlan_reachability() def check_forwarding_resume(self): while True: - state = self.get_asic_state() + state = self.asic_state.get() if state != 'down': break time.sleep(self.TIMEOUT) - return self.get_asic_state_time(state), self.get_asic_vlan_reachability() + return self.asic_state.get_state_time(state), self.get_asic_vlan_reachability() def ping_data_plane(self, light_probe=True): replies_from_servers = self.pingFromServers() @@ -1256,21 +1106,31 @@ def wait_dut_to_warm_up(self): # up towards PTF docker. In practice, I've seen this warm up taking # up to ~70 seconds. + fail = None + dut_stabilize_secs = int(self.test_params['dut_stabilize_secs']) warm_up_timeout_secs = int(self.test_params['warm_up_timeout_secs']) start_time = datetime.datetime.now() + up_time = None # First wait until DUT data/control planes are up while True: - dataplane = self.get_asic_state() - ctrlplane = self.get_cpu_state() + dataplane = self.asic_state.get() + ctrlplane = self.cpu_state.get() elapsed = (datetime.datetime.now() - start_time).total_seconds() - if dataplane == 'up' and ctrlplane == 'up' and elapsed > dut_stabilize_secs: - break; + if dataplane == 'up' and ctrlplane == 'up': + if not up_time: + up_time = datetime.datetime.now() + up_secs = (datetime.datetime.now() - up_time).total_seconds() + if up_secs > dut_stabilize_secs: + break; + else: + # reset up_time + up_time = None + if elapsed > warm_up_timeout_secs: - # Control plane didn't come up within warm up timeout - return False + raise Exception("Control plane didn't come up within warm up timeout") time.sleep(1) # check until flooding is over. Flooding happens when FDB entry of @@ -1279,29 +1139,33 @@ def wait_dut_to_warm_up(self): uptime = datetime.datetime.now() while True: elapsed = (datetime.datetime.now() - start_time).total_seconds() - if not self.asic_flooding and elapsed > dut_stabilize_secs: + if not self.asic_state.is_flooding() and elapsed > dut_stabilize_secs: break if elapsed > warm_up_timeout_secs: - # Control plane didn't stop flooding within warm up timeout - return False + if self.allow_vlan_flooding: + break + raise Exception("Data plane didn't stop flooding within warm up timeout") time.sleep(1) - dataplane = self.get_asic_state() - ctrlplane = self.get_cpu_state() - if not dataplane == 'up' or not ctrlplane == 'up': - # Either control or data plane went down while we were waiting - # for the flooding to stop. - return False + dataplane = self.asic_state.get() + ctrlplane = self.cpu_state.get() + if not dataplane == 'up': + fail = "Data plane" + elif not ctrlplane == 'up': + fail = "Control plane" - if (self.get_asic_state_time('up') > uptime or - self.get_cpu_state_time('up') > uptime): - # Either control plane or data plane flapped while we were - # waiting for the warm up. - return False + if fail is not None: + raise Exception("{} went down while waiting for flooding to stop".format(fail)) - # Everything is good - return True + if self.asic_state.get_state_time('up') > uptime: + fail = "Data plane" + elif self.cpu_state.get_state_time('up') > uptime: + fail = "Control plane" + + if fail is not None: + raise Exception("{} flapped while waiting for the warm up".format(fail)) + # Everything is good def check_alive(self): # This function checks that DUT routes the packets in the both directions. @@ -1317,40 +1181,23 @@ def check_alive(self): uptime = None for counter in range(self.nr_tests * 2): - state = self.get_asic_state() + state = self.asic_state.get() if state == 'up': if not uptime: - uptime = self.get_asic_state_time(state) + uptime = self.asic_state.get_state_time(state) else: if uptime: - return False # Stopped working after it working for sometime? + raise Exception("Data plane stopped working") time.sleep(2) # wait, until FDB entries are populated for _ in range(self.nr_tests * 10): # wait for some time - if not self.asic_flooding: - return True - time.sleep(2) - - return False # we still see extra replies - - - def get_asic_state(self): - with self.asic_state_lock: - state = self.asic_state - return state - - - def get_asic_state_time(self, state): - with self.asic_state_lock: - time = self.asic_state_time[state] - return time - - - def set_asic_state(self, state, reachability=0): - with self.asic_state_lock: - self.asic_state = state - self.asic_state_time[state] = datetime.datetime.now() + if self.asic_state.is_flooding(): + time.sleep(2) + else: + break + else: + raise Exception("DUT is flooding") def get_asic_vlan_reachability(self): @@ -1358,65 +1205,65 @@ def get_asic_vlan_reachability(self): def asic_start_recording_vlan_reachability(self): - with self.asic_state_lock: + with self.vlan_lock: self.asic_vlan_reach = [] self.recording = True def asic_stop_recording_vlan_reachability(self): - with self.asic_state_lock: + with self.vlan_lock: self.recording = False def try_record_asic_vlan_recachability(self, t1_to_vlan): - with self.asic_state_lock: + with self.vlan_lock: if self.recording: self.asic_vlan_reach.append(t1_to_vlan) - def get_cpu_state(self): - with self.cpu_state_lock: - state = self.cpu_state - return state + def log_asic_state_change(self, reachable, partial=False, t1_to_vlan=0, flooding=False): + old = self.asic_state.get() - def get_cpu_state_time(self, state): - with self.cpu_state_lock: - time = self.cpu_state_time[state] - return time + if reachable: + state = 'up' if not partial else 'partial' + else: + state = 'down' + self.try_record_asic_vlan_recachability(t1_to_vlan) + + self.asic_state.set_flooding(flooding) - def set_cpu_state(self, state): - with self.cpu_state_lock: - self.cpu_state = state - self.cpu_state_time[state] = datetime.datetime.now() + if old != state: + self.log("Data plane state transition from %s to %s (%d)" % (old, state, t1_to_vlan)) + self.asic_state.set(state) - def log_asic_state_change(self, reachable, partial=False, t1_to_vlan=0): - old = self.get_asic_state() + def log_cpu_state_change(self, reachable, partial=False, flooding=False): + old = self.cpu_state.get() if reachable: state = 'up' if not partial else 'partial' else: state = 'down' - self.try_record_asic_vlan_recachability(t1_to_vlan) + self.cpu_state.set_flooding(flooding) if old != state: - self.log("Data plane state transition from %s to %s (%d)" % (old, state, t1_to_vlan)) - self.set_asic_state(state) + self.log("Control plane state transition from %s to %s" % (old, state)) + self.cpu_state.set(state) - def log_cpu_state_change(self, reachable, partial=False): - old = self.get_cpu_state() + def log_vlan_state_change(self, reachable): + old = self.vlan_state.get() if reachable: - state = 'up' if not partial else 'partial' + state = 'up' else: state = 'down' if old != state: - self.log("Control plane state transition from %s to %s" % (old, state)) - self.set_cpu_state(state) + self.log("VLAN ARP state transition from %s to %s" % (old, state)) + self.vlan_state.set(state) def reachability_watcher(self): @@ -1424,21 +1271,26 @@ def reachability_watcher(self): # changes for future analysis self.watcher_is_stopped.clear() # Watcher is running. while self.watching: - vlan_to_t1, t1_to_vlan = self.ping_data_plane(self.light_probe) - reachable = (t1_to_vlan > self.nr_vl_pkts * 0.7 and - vlan_to_t1 > self.nr_pc_pkts * 0.7) - partial = (reachable and - (t1_to_vlan < self.nr_vl_pkts or - vlan_to_t1 < self.nr_pc_pkts)) - self.asic_flooding = (reachable and - (t1_to_vlan > self.nr_vl_pkts or - vlan_to_t1 > self.nr_pc_pkts)) - self.log_asic_state_change(reachable, partial, t1_to_vlan) + if self.dataplane_io_lock.acquire(False): + vlan_to_t1, t1_to_vlan = self.ping_data_plane(self.light_probe) + reachable = (t1_to_vlan > self.nr_vl_pkts * 0.7 and + vlan_to_t1 > self.nr_pc_pkts * 0.7) + partial = (reachable and + (t1_to_vlan < self.nr_vl_pkts or + vlan_to_t1 < self.nr_pc_pkts)) + flooding = (reachable and + (t1_to_vlan > self.nr_vl_pkts or + vlan_to_t1 > self.nr_pc_pkts)) + self.log_asic_state_change(reachable, partial, t1_to_vlan, flooding) + self.dataplane_io_lock.release() total_rcv_pkt_cnt = self.pingDut() reachable = total_rcv_pkt_cnt > 0 and total_rcv_pkt_cnt > self.ping_dut_pkts * 0.7 partial = total_rcv_pkt_cnt > 0 and total_rcv_pkt_cnt < self.ping_dut_pkts - self.cpu_flooding = reachable and total_rcv_pkt_cnt > self.ping_dut_pkts - self.log_cpu_state_change(reachable, partial) + flooding = reachable and total_rcv_pkt_cnt > self.ping_dut_pkts + self.log_cpu_state_change(reachable, partial, flooding) + total_rcv_pkt_cnt = self.arpPing() + reachable = total_rcv_pkt_cnt >= self.arp_ping_pkts + self.log_vlan_state_change(reachable) self.watcher_is_running.set() # Watcher is running. self.watcher_is_stopped.set() # Watcher has stopped. self.watcher_is_running.clear() # Watcher has stopped. @@ -1473,3 +1325,10 @@ def pingDut(self): self.log("Send %5d Received %5d ping DUT" % (self.ping_dut_pkts, total_rcv_pkt_cnt), True) return total_rcv_pkt_cnt + + def arpPing(self): + for i in xrange(self.arp_ping_pkts): + testutils.send_packet(self, self.arp_src_port, self.arp_ping) + total_rcv_pkt_cnt = testutils.count_matched_packets_all_ports(self, self.arp_resp, [self.arp_src_port], timeout=self.TIMEOUT) + self.log("Send %5d Received %5d arp ping" % (self.arp_ping_pkts, total_rcv_pkt_cnt), True) + return total_rcv_pkt_cnt diff --git a/ansible/roles/test/files/ptftests/arista.py b/ansible/roles/test/files/ptftests/arista.py new file mode 100644 index 00000000000..e0bef368a54 --- /dev/null +++ b/ansible/roles/test/files/ptftests/arista.py @@ -0,0 +1,487 @@ +import ptf +from ptf.base_tests import BaseTest +from ptf import config +import ptf.testutils as testutils +from ptf.testutils import * +from ptf.dataplane import match_exp_pkt +import datetime +import _strptime # workaround python bug ref: https://stackoverflow.com/a/22476843/2514803 +import time +import subprocess +from ptf.mask import Mask +import socket +import ptf.packet as scapy +import thread +import threading +from multiprocessing.pool import ThreadPool, TimeoutError +import os +import signal +import random +import struct +import socket +from pprint import pprint +from fcntl import ioctl +import sys +import json +import re +from collections import defaultdict +import json +import paramiko +import Queue +import pickle +from operator import itemgetter +import scapy.all as scapyall +import enum + +class Arista(object): + DEBUG = False + def __init__(self, ip, queue, test_params, login='admin', password='123456'): + self.ip = ip + self.queue = queue + self.login = login + self.password = password + self.conn = None + self.arista_prompt = None + self.v4_routes = [test_params['vlan_ip_range'], test_params['lo_prefix']] + self.v6_routes = [test_params['lo_v6_prefix']] + self.fails = set() + self.info = set() + self.min_bgp_gr_timeout = int(test_params['min_bgp_gr_timeout']) + self.reboot_type = test_params['reboot_type'] + + def __del__(self): + self.disconnect() + + def connect(self): + self.conn = paramiko.SSHClient() + self.conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.conn.connect(self.ip, username=self.login, password=self.password, allow_agent=False, look_for_keys=False) + self.shell = self.conn.invoke_shell() + + first_prompt = self.do_cmd(None, prompt = '>') + self.arista_prompt = self.get_arista_prompt(first_prompt) + + self.do_cmd('enable') + self.do_cmd('terminal length 0') + + return self.shell + + def get_arista_prompt(self, first_prompt): + lines = first_prompt.split('\n') + prompt = lines[-1] + # match all modes - A#, A(config)#, A(config-if)# + return prompt.strip().replace('>', '.*#') + + def do_cmd(self, cmd, prompt = None): + if prompt == None: + prompt = self.arista_prompt + + if cmd is not None: + self.shell.send(cmd + '\n') + + input_buffer = '' + while re.search(prompt, input_buffer) is None: + input_buffer += self.shell.recv(16384) + + return input_buffer + + def disconnect(self): + if self.conn is not None: + self.conn.close() + self.conn = None + + return + + def run(self): + data = {} + debug_data = {} + run_once = False + log_first_line = None + quit_enabled = False + v4_routing_ok = False + v6_routing_ok = False + routing_works = True + self.connect() + + cur_time = time.time() + sample = {} + samples = {} + portchannel_output = self.do_cmd("show interfaces po1 | json") + portchannel_output = "\n".join(portchannel_output.split("\r\n")[1:-1]) + sample["po_changetime"] = json.loads(portchannel_output, strict=False)['interfaces']['Port-Channel1']['lastStatusChangeTimestamp'] + samples[cur_time] = sample + + while not (quit_enabled and v4_routing_ok and v6_routing_ok): + cmd = self.queue.get() + if cmd == 'quit': + quit_enabled = True + continue + cur_time = time.time() + info = {} + debug_info = {} + lacp_output = self.do_cmd('show lacp neighbor') + info['lacp'] = self.parse_lacp(lacp_output) + bgp_neig_output = self.do_cmd('show ip bgp neighbors') + info['bgp_neig'] = self.parse_bgp_neighbor(bgp_neig_output) + + bgp_route_v4_output = self.do_cmd('show ip route bgp | json') + v4_routing_ok = self.parse_bgp_route(bgp_route_v4_output, self.v4_routes) + info['bgp_route_v4'] = v4_routing_ok + + bgp_route_v6_output = self.do_cmd("show ipv6 route bgp | json") + v6_routing_ok = self.parse_bgp_route(bgp_route_v6_output, self.v6_routes) + info["bgp_route_v6"] = v6_routing_ok + + portchannel_output = self.do_cmd("show interfaces po1 | json") + portchannel_output = "\n".join(portchannel_output.split("\r\n")[1:-1]) + sample["po_changetime"] = json.loads(portchannel_output, strict=False)['interfaces']['Port-Channel1']['lastStatusChangeTimestamp'] + + if not run_once: + self.ipv4_gr_enabled, self.ipv6_gr_enabled, self.gr_timeout = self.parse_bgp_neighbor_once(bgp_neig_output) + if self.gr_timeout is not None: + log_first_line = "session_begins_%f" % cur_time + self.do_cmd("send log message %s" % log_first_line) + run_once = True + + data[cur_time] = info + samples[cur_time] = sample + if self.DEBUG: + debug_data[cur_time] = { + 'show lacp neighbor' : lacp_output, + 'show ip bgp neighbors' : bgp_neig_output, + 'show ip route bgp' : bgp_route_v4_output, + 'show ipv6 route bgp' : bgp_route_v6_output, + } + + attempts = 60 + log_present = False + for _ in range(attempts): + log_output = self.do_cmd("show log | begin %s" % log_first_line) + log_lines = log_output.split("\r\n")[1:-1] + log_data = self.parse_logs(log_lines) + if (self.reboot_type == 'fast-reboot' and \ + any(k.startswith('BGP') for k in log_data) and any(k.startswith('PortChannel') for k in log_data)) \ + or (self.reboot_type == 'warm-reboot' and any(k.startswith('BGP') for k in log_data)): + log_present = True + break + time.sleep(1) # wait until logs are populated + + if not log_present: + log_data['error'] = 'Incomplete output' + + self.disconnect() + + # save data for troubleshooting + with open("/tmp/%s.data.pickle" % self.ip, "w") as fp: + pickle.dump(data, fp) + + # save debug data for troubleshooting + if self.DEBUG: + with open("/tmp/%s.raw.pickle" % self.ip, "w") as fp: + pickle.dump(debug_data, fp) + with open("/tmp/%s.logging" % self.ip, "w") as fp: + fp.write("\n".join(log_lines)) + + self.check_gr_peer_status(data) + cli_data = {} + cli_data['lacp'] = self.check_series_status(data, "lacp", "LACP session") + cli_data['bgp_v4'] = self.check_series_status(data, "bgp_route_v4", "BGP v4 routes") + cli_data['bgp_v6'] = self.check_series_status(data, "bgp_route_v6", "BGP v6 routes") + cli_data['po'] = self.check_change_time(samples, "po_changetime", "PortChannel interface") + + route_timeout = log_data['route_timeout'] + cli_data['route_timeout'] = route_timeout + + # {'10.0.0.38': [(0, '4200065100)')], 'fc00::2d': [(0, '4200065100)')]} + for nei in route_timeout.keys(): + asn = route_timeout[nei][0][-1] + msg = 'BGP route GR timeout: neighbor %s (ASN %s' % (nei, asn) + self.fails.add(msg) + + return self.fails, self.info, cli_data, log_data + + def extract_from_logs(self, regexp, data): + raw_data = [] + result = defaultdict(list) + initial_time = -1 + re_compiled = re.compile(regexp) + for line in data: + m = re_compiled.match(line) + if not m: + continue + raw_data.append((datetime.datetime.strptime(m.group(1), "%b %d %X"), m.group(2), m.group(3))) + + if len(raw_data) > 0: + initial_time = raw_data[0][0] + for when, what, status in raw_data: + offset = (when - initial_time if when > initial_time else initial_time - when).seconds + result[what].append((offset, status)) + + return result, initial_time + + def parse_logs(self, data): + result = {} + bgp_r = r'^(\S+\s+\d+\s+\S+) \S+ Rib: %BGP-5-ADJCHANGE: peer (\S+) .+ (\S+)$' + result_bgp, initial_time_bgp = self.extract_from_logs(bgp_r, data) + if_r = r'^(\S+\s+\d+\s+\S+) \S+ Ebra: %LINEPROTO-5-UPDOWN: Line protocol on Interface (\S+), changed state to (\S+)$' + result_if, initial_time_if = self.extract_from_logs(if_r, data) + + route_r = r'^(\S+\s+\d+\s+\S+) \S+ Rib: %BGP-5-BGP_GRACEFUL_RESTART_TIMEOUT: Deleting stale routes from peer (\S+) .+ (\S+)$' + result_rt, initial_time_rt = self.extract_from_logs(route_r, data) + + result['route_timeout'] = result_rt + + # for fast-reboot, we expect to have both the bgp and portchannel events in the logs. for warm-reboot, portchannel events might not be present in the logs all the time. + if self.reboot_type == 'fast-reboot' and (initial_time_bgp == -1 or initial_time_if == -1): + return result + elif self.reboot_type == 'warm-reboot' and initial_time_bgp == -1: + return result + + for events in result_bgp.values(): + if events[-1][1] != 'Established': + return result + + # first state is Idle, last state is Established + for events in result_bgp.values(): + if len(events) > 1: + assert(events[0][1] != 'Established') + + assert(events[-1][1] == 'Established') + + # first state is down, last state is up + for events in result_if.values(): + assert(events[0][1] == 'down') + assert(events[-1][1] == 'up') + + neigh_ipv4 = [neig_ip for neig_ip in result_bgp.keys() if '.' in neig_ip][0] + for neig_ip in result_bgp.keys(): + key = "BGP IPv6 was down (seconds)" if ':' in neig_ip else "BGP IPv4 was down (seconds)" + result[key] = result_bgp[neig_ip][-1][0] - result_bgp[neig_ip][0][0] + + for neig_ip in result_bgp.keys(): + key = "BGP IPv6 was down (times)" if ':' in neig_ip else "BGP IPv4 was down (times)" + result[key] = map(itemgetter(1), result_bgp[neig_ip]).count("Idle") + + if initial_time_if != -1: + po_name = [ifname for ifname in result_if.keys() if 'Port-Channel' in ifname][0] + result['PortChannel was down (seconds)'] = result_if[po_name][-1][0] - result_if[po_name][0][0] + for if_name in sorted(result_if.keys()): + result['Interface %s was down (times)' % if_name] = map(itemgetter(1), result_if[if_name]).count("down") + + bgp_po_offset = (initial_time_if - initial_time_bgp if initial_time_if > initial_time_bgp else initial_time_bgp - initial_time_if).seconds + result['PortChannel went down after bgp session was down (seconds)'] = bgp_po_offset + result_if[po_name][0][0] + + for neig_ip in result_bgp.keys(): + key = "BGP IPv6 was gotten up after Po was up (seconds)" if ':' in neig_ip else "BGP IPv4 was gotten up after Po was up (seconds)" + result[key] = result_bgp[neig_ip][-1][0] - bgp_po_offset - result_if[po_name][-1][0] + + return result + + def parse_lacp(self, output): + return output.find('Bundled') != -1 + + def parse_bgp_neighbor_once(self, output): + is_gr_ipv4_enabled = False + is_gr_ipv6_enabled = False + restart_time = None + for line in output.split('\n'): + if ' Restart-time is' in line: + restart_time = int(line.replace(' Restart-time is ', '')) + continue + + if 'is enabled, Forwarding State is' in line: + if 'IPv6' in line: + is_gr_ipv6_enabled = True + elif 'IPv4' in line: + is_gr_ipv4_enabled = True + + return is_gr_ipv4_enabled, is_gr_ipv6_enabled, restart_time + + def parse_bgp_info(self, output): + neigh_bgp = None + dut_bgp = None + asn = None + for line in output.split('\n'): + if 'BGP neighbor is' in line: + dut_bgp = re.findall('BGP neighbor is (.*?),', line)[0] + elif 'Local AS is' in line: + asn = re.findall('Local AS is (\d+?),', line)[0] + elif 'Local TCP address is' in line: + neigh_bgp = re.findall('Local TCP address is (.*?),', line)[0] + break + + return neigh_bgp, dut_bgp, asn + + def parse_bgp_neighbor(self, output): + gr_active = None + gr_timer = None + for line in output.split('\n'): + if 'Restart timer is' in line: + gr_active = 'is active' in line + gr_timer = str(line[-9:-1]) + + return gr_active, gr_timer + + def parse_bgp_route(self, output, expects): + prefixes = set() + data = "\n".join(output.split("\r\n")[1:-1]) + obj = json.loads(data) + + if "vrfs" in obj and "default" in obj["vrfs"]: + obj = obj["vrfs"]["default"] + for prefix, attrs in obj["routes"].items(): + if "routeAction" not in attrs or attrs["routeAction"] != "forward": + continue + if all("Port-Channel" in via["interface"] for via in attrs["vias"]): + prefixes.add(prefix) + + return set(expects) == prefixes + + def get_bgp_info(self): + # Retreive BGP info (peer addr, AS) for the dut and neighbor + neigh_bgp = {} + dut_bgp = {} + for cmd, ver in [('show ip bgp neighbors', 'v4'), ('show ipv6 bgp neighbors', 'v6')]: + output = self.do_cmd(cmd) + if ver == 'v6': + neigh_bgp[ver], dut_bgp[ver], neigh_bgp['asn'] = self.parse_bgp_info(output) + else: + neigh_bgp[ver], dut_bgp[ver], neigh_bgp['asn'] = self.parse_bgp_info(output) + + return neigh_bgp, dut_bgp + + def change_bgp_neigh_state(self, asn, is_up=True): + state = ['shut', 'no shut'] + self.do_cmd('configure') + self.do_cmd('router bgp %s' % asn) + self.do_cmd('%s' % state[is_up]) + self.do_cmd('exit') + self.do_cmd('exit') + + def verify_bgp_neigh_state(self, dut=None, state="Active"): + bgp_state = {} + bgp_state['v4'] = bgp_state['v6'] = False + for cmd, ver in [('show ip bgp summary | json', 'v4'), ('show ipv6 bgp summary | json', 'v6')]: + output = self.do_cmd(cmd) + data = '\n'.join(output.split('\r\n')[1:-1]) + obj = json.loads(data) + + if state == 'down': + if 'vrfs' in obj: + # return True when obj['vrfs'] is empty which is the case when the bgp state is 'down' + bgp_state[ver] = not obj['vrfs'] + else: + self.fails.add('Verify BGP %s neighbor: Object missing in output' % ver) + else: + if 'vrfs' in obj and 'default' in obj['vrfs']: + obj = obj['vrfs']['default'] + if 'peers' in obj: + bgp_state[ver] = (obj['peers'][dut[ver]]['peerState'] == state) + else: + self.fails.add('Verify BGP %S neighbor: Peer attribute missing in output' % ver) + else: + self.fails.add('Verify BGP %s neighbor: Object missing in output' % ver) + return self.fails, bgp_state + + def change_neigh_lag_state(self, lag, is_up=True): + state = ['shut', 'no shut'] + self.do_cmd('configure') + is_match = re.match('(Port-Channel|Ethernet)\d+', lag) + if is_match: + output = self.do_cmd('interface %s' % lag) + if 'Invalid' not in output: + self.do_cmd(state[is_up]) + self.do_cmd('exit') + self.do_cmd('exit') + + def verify_neigh_lag_state(self, lag, state="connected", pre_check=True): + lag_state = False + msg_prefix = ['Postboot', 'Preboot'] + is_match = re.match('(Port-Channel|Ethernet)\d+', lag) + if is_match: + output = self.do_cmd('show interfaces %s | json' % lag) + if 'Invalid' not in output: + data = '\n'.join(output.split('\r\n')[1:-1]) + obj = json.loads(data) + + if 'interfaces' in obj and lag in obj['interfaces']: + lag_state = (obj['interfaces'][lag]['interfaceStatus'] == state) + else: + self.fails.add('%s: Verify LAG %s: Object missing in output' % (msg_prefix[pre_check], lag)) + return self.fails, lag_state + + self.fails.add('%s: Invalid interface name' % msg_prefix[pre_check]) + return self.fails, lag_state + + def check_gr_peer_status(self, output): + # [0] True 'ipv4_gr_enabled', [1] doesn't matter 'ipv6_enabled', [2] should be >= 120 + if not self.ipv4_gr_enabled: + self.fails.add("bgp ipv4 graceful restart is not enabled") + if not self.ipv6_gr_enabled: + pass # ToDo: + if self.gr_timeout < 120: # bgp graceful restart timeout less then 120 seconds + self.fails.add("bgp graceful restart timeout is less then 120 seconds") + + for when, other in sorted(output.items(), key = lambda x : x[0]): + gr_active, timer = other['bgp_neig'] + # wnen it's False, it's ok, wnen it's True, check that inactivity timer not less then self.min_bgp_gr_timeout seconds + if gr_active and datetime.datetime.strptime(timer, '%H:%M:%S') < datetime.datetime(1900, 1, 1, second = self.min_bgp_gr_timeout): + self.fails.add("graceful restart timer is almost finished. Less then %d seconds left" % self.min_bgp_gr_timeout) + + def check_series_status(self, output, entity, what): + # find how long anything was down + # Input parameter is a dictionary when:status + # constraints: + # entity must be down just once + # entity must be up when the test starts + # entity must be up when the test stops + + sorted_keys = sorted(output.keys()) + if not output[sorted_keys[0]][entity]: + self.fails.add("%s must be up when the test starts" % what) + return 0, 0 + if not output[sorted_keys[-1]][entity]: + self.fails.add("%s must be up when the test stops" % what) + return 0, 0 + + start = sorted_keys[0] + cur_state = True + res = defaultdict(list) + for when in sorted_keys[1:]: + if cur_state != output[when][entity]: + res[cur_state].append(when - start) + start = when + cur_state = output[when][entity] + res[cur_state].append(when - start) + + is_down_count = len(res[False]) + + if is_down_count > 1: + self.info.add("%s must be down just for once" % what) + + return is_down_count, sum(res[False]) # summary_downtime + + def check_change_time(self, output, entity, what): + # find last changing time updated, if no update, the entity is never changed + # Input parameter is a dictionary when:last_changing_time + # constraints: + # the dictionary `output` cannot be empty + sorted_keys = sorted(output.keys()) + if not output: + self.fails.add("%s cannot be empty" % what) + return 0, 0 + + start = sorted_keys[0] + prev_time = output[start] + change_count = 0 + for when in sorted_keys[1:]: + if prev_time != output[when][entity]: + prev_time = output[when][entity] + change_count += 1 + + if change_count > 0: + self.info.add("%s state changed %d times" % (what, change_count)) + + # Note: the first item is a placeholder + return 0, change_count + diff --git a/ansible/roles/test/files/ptftests/dhcp_relay_test.py b/ansible/roles/test/files/ptftests/dhcp_relay_test.py index 6ae050df941..358d811805a 100644 --- a/ansible/roles/test/files/ptftests/dhcp_relay_test.py +++ b/ansible/roles/test/files/ptftests/dhcp_relay_test.py @@ -214,6 +214,46 @@ def create_dhcp_offer_packet(self): dhcp_lease=self.LEASE_TIME, padding_bytes=0) + def create_dhcp_offer_relayed_packet(self): + my_chaddr = ''.join([chr(int(octet, 16)) for octet in self.client_mac.split(':')]) + + # Relay modifies the DHCPOFFER message in the following ways: + # 1.) Replaces the source MAC with the MAC of the interface it received it on + # 2.) Replaces the destination MAC with boradcast (ff:ff:ff:ff:ff:ff) + # 3.) Replaces the source IP with the IP of the interface which the relay + # received it on + # 4.) Replaces the destination IP with broadcast (255.255.255.255) + # 5.) Replaces the destination port with the DHCP client port (68) + ether = scapy.Ether(dst=self.BROADCAST_MAC, src=self.relay_iface_mac, type=0x0800) + ip = scapy.IP(src=self.relay_iface_ip, dst=self.BROADCAST_IP, len=290, ttl=64) + udp = scapy.UDP(sport=self.DHCP_SERVER_PORT, dport=self.DHCP_CLIENT_PORT, len=262) + bootp = scapy.BOOTP(op=2, + htype=1, + hlen=6, + hops=0, + xid=0, + secs=0, + flags=0, + ciaddr=self.DEFAULT_ROUTE_IP, + yiaddr=self.client_ip, + siaddr=self.server_ip, + giaddr=self.relay_iface_ip, + chaddr=my_chaddr) + bootp /= scapy.DHCP(options=[('message-type', 'offer'), + ('server_id', self.server_ip), + ('lease_time', self.LEASE_TIME), + ('subnet_mask', self.client_subnet), + ('end')]) + + # TODO: Need to add this to the packet creation functions in PTF code first! + # If our bootp layer is too small, pad it + #pad_bytes = self.DHCP_PKT_BOOTP_MIN_LEN - len(bootp) + #if pad_bytes > 0: + # bootp /= scapy.PADDING('\x00' * pad_bytes) + + pkt = ether / ip / udp / bootp + return pkt + def create_dhcp_request_packet(self): return testutils.dhcp_request_packet(eth_client=self.client_mac, ip_server=self.server_ip, @@ -272,6 +312,47 @@ def create_dhcp_ack_packet(self): dhcp_lease=self.LEASE_TIME, padding_bytes=0) + def create_dhcp_ack_relayed_packet(self): + my_chaddr = ''.join([chr(int(octet, 16)) for octet in self.client_mac.split(':')]) + + # Relay modifies the DHCPACK message in the following ways: + # 1.) Replaces the source MAC with the MAC of the interface it received it on + # 2.) Replaces the destination MAC with boradcast (ff:ff:ff:ff:ff:ff) + # 3.) Replaces the source IP with the IP of the interface which the relay + # received it on + # 4.) Replaces the destination IP with broadcast (255.255.255.255) + # 5.) Replaces the destination port with the DHCP client port (68) + ether = scapy.Ether(dst=self.BROADCAST_MAC, src=self.relay_iface_mac, type=0x0800) + ip = scapy.IP(src=self.relay_iface_ip, dst=self.BROADCAST_IP, len=290, ttl=64) + udp = scapy.UDP(sport=self.DHCP_SERVER_PORT, dport=self.DHCP_CLIENT_PORT, len=262) + bootp = scapy.BOOTP(op=2, + htype=1, + hlen=6, + hops=0, + xid=0, + secs=0, + flags=0, + ciaddr=self.DEFAULT_ROUTE_IP, + yiaddr=self.client_ip, + siaddr=self.server_ip, + giaddr=self.relay_iface_ip, + chaddr=my_chaddr) + bootp /= scapy.DHCP(options=[('message-type', 'ack'), + ('server_id', self.server_ip), + ('lease_time', self.LEASE_TIME), + ('subnet_mask', self.client_subnet), + ('end')]) + + # TODO: Need to add this to the packet creation functions in PTF code first! + # If our bootp layer is too small, pad it + #pad_bytes = self.DHCP_PKT_BOOTP_MIN_LEN - len(bootp) + #if pad_bytes > 0: + # bootp /= scapy.PADDING('\x00' * pad_bytes) + + pkt = ether / ip / udp / bootp + return pkt + + """ Send/receive functions @@ -318,9 +399,10 @@ def verify_relayed_discover(self): masked_discover.set_do_not_care_scapy(scapy.PADDING, "load") # Count the number of these packets received on the ports connected to our leaves + num_expected_packets = self.num_dhcp_servers discover_count = testutils.count_matched_packets_all_ports(self, masked_discover, self.server_port_indices) - self.assertTrue(discover_count == self.num_dhcp_servers, - "Failed: Discover count of %d != %d (num_dhcp_servers)" % (discover_count, self.num_dhcp_servers)) + self.assertTrue(discover_count == num_expected_packets, + "Failed: Discover count of %d != %d" % (discover_count, num_expected_packets)) # Simulate a DHCP server sending a DHCPOFFER message to client. # We do this by injecting a DHCPOFFER message on the link connected to one @@ -331,24 +413,31 @@ def server_send_offer(self): # Verify that the DHCPOFFER would be received by our simulated client def verify_offer_received(self): - dhcp_offer = self.create_dhcp_offer_packet() + dhcp_offer = self.create_dhcp_offer_relayed_packet() masked_offer = Mask(dhcp_offer) - masked_offer.set_do_not_care_scapy(scapy.Ether, "src") - masked_offer.set_do_not_care_scapy(scapy.Ether, "dst") + masked_offer.set_do_not_care_scapy(scapy.IP, "version") + masked_offer.set_do_not_care_scapy(scapy.IP, "ihl") + masked_offer.set_do_not_care_scapy(scapy.IP, "tos") + masked_offer.set_do_not_care_scapy(scapy.IP, "len") + masked_offer.set_do_not_care_scapy(scapy.IP, "id") + masked_offer.set_do_not_care_scapy(scapy.IP, "flags") + masked_offer.set_do_not_care_scapy(scapy.IP, "frag") + masked_offer.set_do_not_care_scapy(scapy.IP, "ttl") + masked_offer.set_do_not_care_scapy(scapy.IP, "proto") masked_offer.set_do_not_care_scapy(scapy.IP, "chksum") - masked_offer.set_do_not_care_scapy(scapy.IP, "src") - masked_offer.set_do_not_care_scapy(scapy.IP, "dst") + masked_offer.set_do_not_care_scapy(scapy.IP, "options") + masked_offer.set_do_not_care_scapy(scapy.UDP, "len") masked_offer.set_do_not_care_scapy(scapy.UDP, "chksum") - masked_offer.set_do_not_care_scapy(scapy.UDP, "dport") - # Mask out lease time since it can change depending on when the server receives the request - # Lease time in ack can be slightly different than in offer, since lease time varies slightly - # We also want to ignore the checksums since they will vary a bit depending on the timestamp - # Offset is byte 292, 6 byte field, set_do_not_care() expects values in bits - masked_offer.set_do_not_care((self.DHCP_LEASE_TIME_OFFSET * 8), (self.DHCP_LEASE_TIME_LEN * 8)) + masked_offer.set_do_not_care_scapy(scapy.BOOTP, "sname") + masked_offer.set_do_not_care_scapy(scapy.BOOTP, "file") + + masked_offer.set_do_not_care_scapy(scapy.DHCP, "lease_time") + + #masked_offer.set_do_not_care_scapy(scapy.PADDING, "load") # NOTE: verify_packet() will fail for us via an assert, so no need to check a return value here testutils.verify_packet(self, masked_offer, self.client_port_index) @@ -390,9 +479,10 @@ def verify_relayed_request(self): masked_request.set_do_not_care_scapy(scapy.BOOTP, "file") # Count the number of these packets received on the ports connected to our leaves + num_expected_packets = self.num_dhcp_servers request_count = testutils.count_matched_packets_all_ports(self, masked_request, self.server_port_indices) - self.assertTrue(request_count == self.num_dhcp_servers, - "Failed: Request count of %d != %d (num_dhcp_servers)" % (request_count, self.num_dhcp_servers)) + self.assertTrue(request_count == num_expected_packets, + "Failed: Request count of %d != %d" % (request_count, num_expected_packets)) # Simulate a DHCP server sending a DHCPOFFER message to client from one of our leaves def server_send_ack(self): @@ -401,23 +491,29 @@ def server_send_ack(self): # Verify that the DHCPACK would be received by our simulated client def verify_ack_received(self): - dhcp_ack = self.create_dhcp_ack_packet() + dhcp_ack = self.create_dhcp_ack_relayed_packet() - # Mask out lease time, ip checksum, udp checksum (explanation above) masked_ack = Mask(dhcp_ack) - masked_ack.set_do_not_care_scapy(scapy.Ether, "src") - masked_ack.set_do_not_care_scapy(scapy.Ether, "dst") - + masked_ack.set_do_not_care_scapy(scapy.IP, "version") + masked_ack.set_do_not_care_scapy(scapy.IP, "ihl") + masked_ack.set_do_not_care_scapy(scapy.IP, "tos") + masked_ack.set_do_not_care_scapy(scapy.IP, "len") + masked_ack.set_do_not_care_scapy(scapy.IP, "id") + masked_ack.set_do_not_care_scapy(scapy.IP, "flags") + masked_ack.set_do_not_care_scapy(scapy.IP, "frag") + masked_ack.set_do_not_care_scapy(scapy.IP, "ttl") + masked_ack.set_do_not_care_scapy(scapy.IP, "proto") masked_ack.set_do_not_care_scapy(scapy.IP, "chksum") - masked_ack.set_do_not_care_scapy(scapy.IP, "src") - masked_ack.set_do_not_care_scapy(scapy.IP, "dst") + masked_ack.set_do_not_care_scapy(scapy.IP, "options") + masked_ack.set_do_not_care_scapy(scapy.UDP, "len") masked_ack.set_do_not_care_scapy(scapy.UDP, "chksum") - masked_ack.set_do_not_care_scapy(scapy.UDP, "dport") - # Also mask out lease time (see comment in verify_offer_received() above) - masked_ack.set_do_not_care((self.DHCP_LEASE_TIME_OFFSET * 8), (self.DHCP_LEASE_TIME_LEN * 8)) + masked_ack.set_do_not_care_scapy(scapy.BOOTP, "sname") + masked_ack.set_do_not_care_scapy(scapy.BOOTP, "file") + + masked_ack.set_do_not_care_scapy(scapy.DHCP, "lease_time") # NOTE: verify_packet() will fail for us via an assert, so no need to check a return value here testutils.verify_packet(self, masked_ack, self.client_port_index) diff --git a/ansible/roles/test/files/ptftests/fdb_mac_expire_test.py b/ansible/roles/test/files/ptftests/fdb_mac_expire_test.py new file mode 100644 index 00000000000..7bdbf42eb69 --- /dev/null +++ b/ansible/roles/test/files/ptftests/fdb_mac_expire_test.py @@ -0,0 +1,37 @@ +import fdb + +import ptf +import ptf.packet as scapy +import ptf.dataplane as dataplane + +from ptf import config +from ptf.base_tests import BaseTest +from ptf.testutils import * + +class FdbMacExpireTest(BaseTest): + def __init__(self): + BaseTest.__init__(self) + self.test_params = test_params_get() + #-------------------------------------------------------------------------- + def setUp(self): + self.dataplane = ptf.dataplane_instance + self.router_mac = self.test_params['router_mac'] + self.dummy_mac_prefix = self.test_params['dummy_mac_prefix'] + self.fdb_info = self.test_params['fdb_info'] + #-------------------------------------------------------------------------- + def populateFdb(self): + self.fdb = fdb.Fdb(self.fdb_info) + vlan_table = self.fdb.get_vlan_table() + for vlan in vlan_table: + for member in vlan_table[vlan]: + mac = self.dummy_mac_prefix + ":" + "{:02X}".format(member) + # Send a packet to switch to populate the layer 2 table + pkt = simple_eth_packet(eth_dst=self.router_mac, + eth_src=mac, + eth_type=0x1234) + send(self, member, pkt) + #-------------------------------------------------------------------------- + def runTest(self): + self.populateFdb() + return + #-------------------------------------------------------------------------- diff --git a/ansible/roles/test/files/ptftests/fib_test.py b/ansible/roles/test/files/ptftests/fib_test.py index 5f11b96a9ff..a53e572772a 100644 --- a/ansible/roles/test/files/ptftests/fib_test.py +++ b/ansible/roles/test/files/ptftests/fib_test.py @@ -87,6 +87,7 @@ def setUp(self): self.dataplane = ptf.dataplane_instance self.fib = fib.Fib(self.test_params['fib_info']) self.router_mac = self.test_params['router_mac'] + self.pktlen = self.test_params['testbed_mtu'] self.test_ipv4 = self.test_params.get('ipv4', True) self.test_ipv6 = self.test_params.get('ipv6', True) @@ -100,6 +101,8 @@ def setUp(self): self.src_ports = [0, 1, 4, 5, 16, 17, 20, 21, 34, 36, 37, 38, 39, 42, 44, 45, 46, 47, 50, 52, 53, 54, 55, 58, 60, 61, 62, 63] if self.test_params['testbed_type'] == 't0': self.src_ports = range(1, 25) + range(28, 32) + if self.test_params['testbed_type'] == 't0-56': + self.src_ports = [0, 1, 4, 5, 8, 9] + range(12, 18) + [20, 21, 24, 25, 28, 29, 32, 33, 36, 37] + range(40, 46) + [48, 49, 52, 53] if self.test_params['testbed_type'] == 't0-64': self.src_ports = range(0, 2) + range(4, 18) + range(20, 33) + range(36, 43) + range(48, 49) + range(52, 59) if self.test_params['testbed_type'] == 't0-116': @@ -170,6 +173,7 @@ def check_ipv4_route(self, src_port, dst_ip_addr, dst_port_list): src_mac = self.dataplane.get_mac(0, 0) pkt = simple_tcp_packet( + pktlen=self.pktlen, eth_dst=self.router_mac, eth_src=src_mac, ip_src=ip_src, @@ -178,6 +182,7 @@ def check_ipv4_route(self, src_port, dst_ip_addr, dst_port_list): tcp_dport=dport, ip_ttl=64) exp_pkt = simple_tcp_packet( + self.pktlen, eth_src=self.router_mac, ip_src=ip_src, ip_dst=ip_dst, @@ -208,6 +213,7 @@ def check_ipv6_route(self, src_port, dst_ip_addr, dst_port_list): src_mac = self.dataplane.get_mac(0, 0) pkt = simple_tcpv6_packet( + pktlen=self.pktlen, eth_dst=self.router_mac, eth_src=src_mac, ipv6_dst=ip_dst, @@ -216,6 +222,7 @@ def check_ipv6_route(self, src_port, dst_ip_addr, dst_port_list): tcp_dport=dport, ipv6_hlim=64) exp_pkt = simple_tcpv6_packet( + pktlen=self.pktlen, eth_src=self.router_mac, ipv6_dst=ip_dst, ipv6_src=ip_src, diff --git a/ansible/roles/test/files/ptftests/sad_path.py b/ansible/roles/test/files/ptftests/sad_path.py new file mode 100644 index 00000000000..fec1f64fd41 --- /dev/null +++ b/ansible/roles/test/files/ptftests/sad_path.py @@ -0,0 +1,276 @@ +import datetime +import re +import subprocess +import time + +from arista import Arista + + +class PrebootTest(object): + def __init__(self, oper_type, vm_list, portchannel_ports, vm_dut_map, test_args, dut_ssh): + self.oper_type = oper_type + self.vm_list = vm_list + self.portchannel_ports = portchannel_ports + self.vm_dut_map = vm_dut_map + self.test_args = test_args + self.dut_ssh = dut_ssh + self.fails_vm = set() + self.fails_dut = set() + self.log = [] + self.shandle = SadOper(self.oper_type, self.vm_list, self.portchannel_ports, self.vm_dut_map, self.test_args, self.dut_ssh) + + def setup(self): + self.shandle.sad_setup(is_up=False) + return self.shandle.retreive_test_info(), self.shandle.retreive_logs() + + def verify(self, pre_check=True): + self.shandle.sad_bgp_verify() + if 'lag' in self.oper_type: + self.shandle.sad_lag_verify(pre_check=pre_check) + return self.shandle.retreive_logs() + + def revert(self): + self.shandle.sad_setup() + return self.shandle.retreive_logs() + + +class SadPath(object): + def __init__(self, oper_type, vm_list, portchannel_ports, vm_dut_map, test_args): + self.oper_type = oper_type + self.vm_list = vm_list + self.portchannel_ports = portchannel_ports + self.vm_dut_map = vm_dut_map + self.test_args = test_args + self.neigh_vm = None + self.neigh_name = None + self.vm_handle = None + self.neigh_bgp = None + self.dut_bgp = None + self.log = [] + self.fails = dict() + self.fails['dut'] = set() + + def cmd(self, cmds): + process = subprocess.Popen(cmds, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + return_code = process.returncode + + return stdout, stderr, return_code + + def select_vm(self): + self.vm_list.sort() + # use the day of the month to select a VM from the list for the sad pass operation + vm_index = datetime.datetime.now().day % len(self.vm_list) + self.neigh_vm = self.vm_list.pop(vm_index) + + def get_neigh_name(self): + for key in self.vm_dut_map.keys(): + if self.vm_dut_map[key]['mgmt_addr'] == self.neigh_vm: + self.neigh_name = key + break + + def down_neigh_port(self): + # extract ptf ports for the selected VM and mark them down + for port in self.vm_dut_map[self.neigh_name]['ptf_ports']: + self.portchannel_ports.remove(port) + + def vm_connect(self): + self.vm_handle = Arista(self.neigh_vm, None, self.test_args) + self.vm_handle.connect() + + def __del__(self): + self.vm_disconnect() + + def vm_disconnect(self): + self.vm_handle.disconnect() + + def setup(self): + self.select_vm() + self.get_neigh_name() + self.down_neigh_port() + self.vm_connect() + self.neigh_bgp, self.dut_bgp = self.vm_handle.get_bgp_info() + self.fails[self.neigh_vm] = set() + self.log.append('Neighbor AS: %s' % self.neigh_bgp['asn']) + self.log.append('BGP v4 neighbor: %s' % self.neigh_bgp['v4']) + self.log.append('BGP v6 neighbor: %s' % self.neigh_bgp['v6']) + self.log.append('DUT BGP v4: %s' % self.dut_bgp['v4']) + self.log.append('DUT BGP v6: %s' % self.dut_bgp['v6']) + + def retreive_test_info(self): + return self.vm_list, self.portchannel_ports, self.neigh_vm + + def retreive_logs(self): + return self.log, self.fails['dut'], self.fails[self.neigh_vm] + + +class SadOper(SadPath): + def __init__(self, oper_type, vm_list, portchannel_ports, vm_dut_map, test_args, dut_ssh): + super(SadOper, self).__init__(oper_type, vm_list, portchannel_ports, vm_dut_map, test_args) + self.dut_ssh = dut_ssh + self.dut_needed = None + self.lag_members_down = None + self.neigh_lag_state = None + self.msg_prefix = ['Postboot', 'Preboot'] + + def populate_bgp_state(self): + if self.oper_type == 'neigh_bgp_down': + self.neigh_bgp['changed_state'] = 'down' + self.dut_bgp['changed_state'] = 'Active' + self.dut_needed = None + elif self.oper_type == 'dut_bgp_down': + self.neigh_bgp['changed_state'] = 'Active' + self.dut_bgp['changed_state'] = 'Idle' + self.dut_needed = self.dut_bgp + elif self.oper_type == 'neigh_lag_down': + # on the DUT side, bgp states are different pre and post boot. hence passing multiple values + self.neigh_bgp['changed_state'] = 'Idle' + self.dut_bgp['changed_state'] = 'Connect,Active,Idle' + self.dut_needed = self.dut_bgp + elif self.oper_type == 'dut_lag_down': + self.neigh_bgp['changed_state'] = 'Idle' + self.dut_bgp['changed_state'] = 'Active,Connect,Idle' + self.dut_needed = self.dut_bgp + + def sad_setup(self, is_up=True): + self.log = [] + + if not is_up: + self.setup() + self.populate_bgp_state() + if 'lag' in self.oper_type: + self.populate_lag_state() + + if 'bgp' in self.oper_type: + self.log.append('BGP state change will be for %s' % self.neigh_vm) + if self.oper_type == 'neigh_bgp_down': + self.log.append('Changing state of AS %s to shut' % self.neigh_bgp['asn']) + self.vm_handle.change_bgp_neigh_state(self.neigh_bgp['asn'], is_up=is_up) + elif self.oper_type == 'dut_bgp_down': + self.change_bgp_dut_state(is_up=is_up) + time.sleep(30) + elif 'lag' in self.oper_type: + self.log.append('LAG state change will be for %s' % self.neigh_vm) + if self.oper_type == 'neigh_lag_down': + self.log.append('Changing state of LAG %s to shut' % self.vm_dut_map[self.neigh_name]['neigh_portchannel']) + self.vm_handle.change_neigh_lag_state(self.vm_dut_map[self.neigh_name]['neigh_portchannel'], is_up=is_up) + elif self.oper_type == 'dut_lag_down': + self.change_dut_lag_state(is_up=is_up) + # wait for sometime for lag members state to sync + time.sleep(120) + + def change_bgp_dut_state(self, is_up=True): + state = ['shutdown', 'startup'] + for key in self.neigh_bgp.keys(): + if key not in ['v4', 'v6']: + continue + + self.log.append('Changing state of BGP peer %s from DUT side to %s' % (self.neigh_bgp[key], state[is_up])) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'sudo config bgp %s neighbor %s' % (state[is_up], self.neigh_bgp[key])]) + if return_code != 0: + self.fails['dut'].add('State change not successful from DUT side for peer %s' % self.neigh_bgp[key]) + self.fails['dut'].add('Return code: %d' % return_code) + self.fails['dut'].add('Stderr: %s' % stderr) + + def verify_bgp_dut_state(self, state='Idle'): + states = state.split(',') + bgp_state = {} + bgp_state['v4'] = bgp_state['v6'] = False + for key in self.neigh_bgp.keys(): + if key not in ['v4', 'v6']: + continue + self.log.append('Verifying if the DUT side BGP peer %s is %s' % (self.neigh_bgp[key], states)) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'show ip bgp neighbor %s' % self.neigh_bgp[key]]) + if return_code == 0: + for line in stdout.split('\n'): + if 'BGP state' in line: + curr_state = re.findall('BGP state = (\w+)', line)[0] + bgp_state[key] = (curr_state in states) + break + else: + self.fails['dut'].add('Retreiving BGP info for peer %s from DUT side failed' % self.neigh_bgp[key]) + self.fails['dut'].add('Return code: %d' % return_code) + self.fails['dut'].add('Stderr: %s' % stderr) + return bgp_state + + def sad_bgp_verify(self): + self.log = [] + fails_vm, bgp_state = self.vm_handle.verify_bgp_neigh_state(dut=self.dut_needed, state=self.neigh_bgp['changed_state']) + self.fails[self.neigh_vm] |= fails_vm + if bgp_state['v4'] and bgp_state['v6']: + self.log.append('BGP state down as expected for %s' % self.neigh_vm) + else: + self.fails[self.neigh_vm].add('BGP state not down for %s' % self.neigh_vm) + bgp_state = self.verify_bgp_dut_state(state=self.dut_bgp['changed_state']) + if bgp_state['v4'] and bgp_state['v6']: + self.log.append('BGP state down as expected on DUT') + else: + self.fails['dut'].add('BGP state not down on DUT') + + def populate_lag_state(self): + if self.oper_type == 'neigh_lag_down': + self.neigh_lag_state = 'disabled' + self.lag_members_down = self.vm_dut_map[self.neigh_name]['dut_ports'] + elif self.oper_type == 'dut_lag_down': + self.lag_members_down = self.vm_dut_map[self.neigh_name]['dut_ports'] + self.neigh_lag_state = 'notconnect' + + def change_dut_lag_state(self, is_up=True): + state = ['shutdown', 'startup'] + dut_portchannel = self.vm_dut_map[self.neigh_name]['dut_portchannel'] + if not re.match('(PortChannel|Ethernet)\d+', dut_portchannel): return + self.log.append('Changing state of %s from DUT side to %s' % (dut_portchannel, state[is_up])) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'sudo config interface %s %s' % (state[is_up], dut_portchannel)]) + if return_code != 0: + self.fails['dut'].add('%s: State change not successful from DUT side for %s' % (self.msg_prefix[1 - is_up], dut_portchannel)) + self.fails['dut'].add('%s: Return code: %d' % (self.msg_prefix[1 - is_up], return_code)) + self.fails['dut'].add('%s: Stderr: %s' % (self.msg_prefix[1 - is_up], stderr)) + else: + self.log.append('State change successful on DUT') + + def verify_dut_lag_member_state(self, lag_memb_output, pre_check=True): + success = True + for member in self.vm_dut_map[self.neigh_name]['dut_ports']: + if self.lag_members_down is not None and member in self.lag_members_down: + search_str = '%s(D)' % member + else: + search_str = '%s(S)' % member + + if lag_memb_output.find(search_str) != -1: + self.log.append('Lag member %s state as expected' % member) + else: + success = False + self.fails['dut'].add('%s: Lag member %s state not as expected' % (self.msg_prefix[pre_check], member)) + return success + + def verify_dut_lag_state(self, pre_check=True): + pat = re.compile(".*%s\s+LACP\(A\)\(Dw\)\s+(.*)" % self.vm_dut_map[self.neigh_name]['dut_portchannel']) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'show interfaces portchannel']) + if return_code == 0: + for line in stdout.split('\n'): + if self.vm_dut_map[self.neigh_name]['dut_portchannel'] in line: + is_match = pat.match(line) + if is_match and self.verify_dut_lag_member_state(is_match.group(1), pre_check=pre_check): + self.log.append('Lag state is down as expected on the DUT') + self.log.append('Pattern check: %s' % line) + else: + self.fails['dut'].add('%s: Lag state is not down on the DUT' % self.msg_prefix[pre_check]) + self.fails['dut'].add('%s: Obtained: %s' % (self.msg_prefix[pre_check], line)) + break + else: + self.fails['dut'].add('%s: Retreiving LAG info from DUT side failed' % self.msg_prefix[pre_check]) + self.fails['dut'].add('%s: Return code: %d' % (self.msg_prefix[pre_check], return_code)) + self.fails['dut'].add('%s: Stderr: %s' % (self.msg_prefix[pre_check], stderr)) + + def sad_lag_verify(self, pre_check=True): + fails_vm, lag_state = self.vm_handle.verify_neigh_lag_state(self.vm_dut_map[self.neigh_name]['neigh_portchannel'], state=self.neigh_lag_state, pre_check=pre_check) + self.fails[self.neigh_vm] |= fails_vm + if lag_state: + self.log.append('LAG state down as expected for %s' % self.neigh_vm) + else: + self.fails[self.neigh_vm].add('%s: LAG state not down for %s' % (self.msg_prefix[pre_check], self.neigh_vm)) + self.log.append('Verifying LAG state on the dut end') + self.verify_dut_lag_state(pre_check=pre_check) diff --git a/ansible/roles/test/files/ptftests/vxlan-decap.py b/ansible/roles/test/files/ptftests/vxlan-decap.py index 1f4471dadd0..005e3132d47 100644 --- a/ansible/roles/test/files/ptftests/vxlan-decap.py +++ b/ansible/roles/test/files/ptftests/vxlan-decap.py @@ -15,6 +15,7 @@ import os.path import json import ptf +import ptf.packet as scapy from ptf.base_tests import BaseTest from ptf import config import ptf.testutils as testutils diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py index 9b631f612d2..8875e874b47 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py @@ -616,9 +616,9 @@ def main(argv): analyzer.place_marker(log_file_list, analyzer.create_start_marker()) return 0 elif (action == "analyze"): - match_file_list = match_files_in.split(tokenizer); - ignore_file_list = ignore_files_in.split(tokenizer); - expect_file_list = expect_files_in.split(tokenizer); + match_file_list = match_files_in.split(tokenizer) + ignore_file_list = ignore_files_in.split(tokenizer) + expect_file_list = expect_files_in.split(tokenizer) analyzer.place_marker(log_file_list, analyzer.create_end_marker()) diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt index e69de29bb2d..03bada03d8b 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt @@ -0,0 +1 @@ +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml index 2dd4d5826ba..d3ab2301feb 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml @@ -21,10 +21,14 @@ register: expected_missing_matches - set_fact: - fail_in_logs: "{{ errors_found.stdout != \"0\" or expected_missing_matches.stdout != \"0\" }}" + fail_in_logs: "{{ errors_found.stdout != \"0\" or expected_missing_matches.stdout != \"0\" }}" + +- set_fact: + dump_since: '1 hour ago' + when: dump_since is not defined - name: Generate system dump - command: generate_dump + command: "generate_dump -s '{{ dump_since }}'" become: true register: generate_dump when: fail_in_logs diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml index ea6ee30b0d2..a90106869aa 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml @@ -18,7 +18,7 @@ when: expect_file is not defined - set_fact: - testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ ansible_date_time.time }}" + testname_unique: "{{ testname }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}" when: testname_unique is not defined - set_fact: diff --git a/ansible/roles/test/tasks/acl/acl_traffic_test/run_ping_test.yml b/ansible/roles/test/tasks/acl/acl_traffic_test/run_ping_test.yml index 0eabbfed108..01ea8dba5ee 100644 --- a/ansible/roles/test/tasks/acl/acl_traffic_test/run_ping_test.yml +++ b/ansible/roles/test/tasks/acl/acl_traffic_test/run_ping_test.yml @@ -2,7 +2,13 @@ # Execute ping and check the log. #----------------------------------------- -- include_vars: "vars/run_ping_test_vars.yml" +- name: Initialize some variables for loganalyzer + set_fact: + testname_unique: "{{ testname }}.{{ lookup('pipe','date +%Y-%m-%d-%H:%M:%S') }}" +- set_fact: + test_out_dir: "{{ out_dir }}/{{ testname_unique }}" + summary_file: "summary.loganalysis.{{ testname_unique }}.log" + result_file: "result.loganalysis.{{ testname_unique }}.log" - name: Start log analyser include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml diff --git a/ansible/roles/test/tasks/acl/acl_traffic_test/run_ptf_test.yml b/ansible/roles/test/tasks/acl/acl_traffic_test/run_ptf_test.yml index 059652ddaec..dc8436a49dd 100644 --- a/ansible/roles/test/tasks/acl/acl_traffic_test/run_ptf_test.yml +++ b/ansible/roles/test/tasks/acl/acl_traffic_test/run_ptf_test.yml @@ -2,7 +2,13 @@ # Send some TCP packets. #----------------------------------------- -- include_vars: "vars/run_ptf_test_vars.yml" +- name: Initialize some variables for loganalyzer + set_fact: + testname_unique: "{{ testname }}.{{ lookup('pipe','date +%Y-%m-%d-%H:%M:%S') }}" +- set_fact: + test_out_dir: "{{ out_dir }}/{{ testname_unique }}" + summary_file: "summary.loganalysis.{{ testname_unique }}.log" + result_file: "result.loganalysis.{{ testname_unique }}.log" - block: - name: Execute ping from host to switch to validate link diff --git a/ansible/roles/test/tasks/acl/acltb_ignore_messages.txt b/ansible/roles/test/tasks/acl/acltb_ignore_messages.txt index e69de29bb2d..03bada03d8b 100644 --- a/ansible/roles/test/tasks/acl/acltb_ignore_messages.txt +++ b/ansible/roles/test/tasks/acl/acltb_ignore_messages.txt @@ -0,0 +1 @@ +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/tasks/acl/acltb_test_rules.json b/ansible/roles/test/tasks/acl/acltb_test_rules.json index 488770c0ade..4106b702e88 100644 --- a/ansible/roles/test/tasks/acl/acltb_test_rules.json +++ b/ansible/roles/test/tasks/acl/acltb_test_rules.json @@ -201,6 +201,36 @@ "source-ip-address": "10.0.0.2/32" } } + }, + "14": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 27 + }, + "transport": { + "config": { + "source-port": "179" + } + } + }, + "15": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 28 + }, + "transport": { + "config": { + "destination-port": "179" + } + } } } } diff --git a/ansible/roles/test/tasks/acl/acltb_test_rules_part_1.json b/ansible/roles/test/tasks/acl/acltb_test_rules_part_1.json index 29c6b02d034..1687fbda9cf 100644 --- a/ansible/roles/test/tasks/acl/acltb_test_rules_part_1.json +++ b/ansible/roles/test/tasks/acl/acltb_test_rules_part_1.json @@ -96,6 +96,36 @@ "source-ip-address": "10.0.0.2/32" } } + }, + "15": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 27 + }, + "transport": { + "config": { + "source-port": "179" + } + } + }, + "16": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 28 + }, + "transport": { + "config": { + "destination-port": "179" + } + } } } } diff --git a/ansible/roles/test/tasks/acl/acltb_test_rules_part_2.json b/ansible/roles/test/tasks/acl/acltb_test_rules_part_2.json index 488770c0ade..4106b702e88 100644 --- a/ansible/roles/test/tasks/acl/acltb_test_rules_part_2.json +++ b/ansible/roles/test/tasks/acl/acltb_test_rules_part_2.json @@ -201,6 +201,36 @@ "source-ip-address": "10.0.0.2/32" } } + }, + "14": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 27 + }, + "transport": { + "config": { + "source-port": "179" + } + } + }, + "15": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 28 + }, + "transport": { + "config": { + "destination-port": "179" + } + } } } } diff --git a/ansible/roles/test/tasks/acltb_ranges_test.yml b/ansible/roles/test/tasks/acltb_ranges_test.yml index e9325c247b4..9c7d37afece 100644 --- a/ansible/roles/test/tasks/acltb_ranges_test.yml +++ b/ansible/roles/test/tasks/acltb_ranges_test.yml @@ -27,7 +27,7 @@ # Separate set_fact is required to be able to use 'testname' fact. - set_fact: - testname_unique: "{{ testname }}.{{ ansible_date_time.date}}.{{ ansible_date_time.hour}}-{{ ansible_date_time.minute}}-{{ ansible_date_time.second}}" + testname_unique: "{{ testname }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}" # Separate set_fact is required to be able to use 'testname_unique' fact. - set_fact: diff --git a/ansible/roles/test/tasks/advanced-reboot.yml b/ansible/roles/test/tasks/advanced-reboot.yml index 40626e1f238..e9531d3793f 100644 --- a/ansible/roles/test/tasks/advanced-reboot.yml +++ b/ansible/roles/test/tasks/advanced-reboot.yml @@ -20,6 +20,21 @@ - fail: msg="Please set vm_hosts variable with a list of VMs" when: vm_hosts is not defined + - name: Preboot-list initialization + set_fact: preboot_list={% if preboot_list is not defined %}[None]{% else %}{{ preboot_list }}{% endif %} + + - name: Preboot files initialization + set_fact: preboot_files={% if preboot_files is not defined %}None{% else %}{{ preboot_files }}{% endif %} + + - debug: + msg: "Preboot-list: {{ preboot_list }} Preboot-files: {{ preboot_files }}" + + - name: Set PTF test params + set_fact: + dut_mac: "{{ ansible_Ethernet0['macaddress'] }}" + vlan_ip_range: "{{ minigraph_vlan_interfaces[0]['subnet'] }}" + lo_v6_prefix: "{{ minigraph_lo_interfaces | map(attribute='addr') | ipv6 | first | ipsubnet(64) }}" + - name: Remove existing ip from ptf host script: roles/test/files/helpers/remove_ip.sh delegate_to: "{{ ptf_host }}" @@ -108,110 +123,64 @@ - block: - - name: Save image version - shell: 'sonic_installer list | grep Current | cut -f2 -d " "' - register: current_sonic_image - become: true + - name: Copy peer device info to ptf host + copy: + content: "{{ minigraph_devices | to_nice_json }}" + dest: /tmp/peer_dev_info.json + delegate_to: "{{ ptf_host }}" - - set_fact: - new_image_location: '/tmp/new_sonic_image.bin' + - name: Copy neighbor port info to ptf host + copy: + content: "{{ minigraph_neighbors | to_nice_json }}" + dest: /tmp/neigh_port_info.json + delegate_to: "{{ ptf_host }}" - - name: Download SONiC image. - local_action: get_url url={{ new_sonic_image }} dest={{ new_image_location }} + when: preboot_list|length > 1 - - name: Upload SONiC image to device. - copy: - src: "{{ new_image_location }}" - dest: "{{ new_image_location }}" + - debug: msg="Defined new sonic image url is {{ new_sonic_image }}" + when: new_sonic_image is defined - - name: Install a new SONiC image if requested - shell: sonic_installer install -y {{ new_image_location }} - become: true - notify: - - restore current image - - reboot sonic + - set_fact: + stay_in_target_image: "{{ stay_in_target_image | default('false') | bool }}" + cleanup_old_sonic_images: "{{ cleanup_old_sonic_images | default('false') | bool }}" + allow_vlan_flooding: "{{ allow_vlan_flooding | default('false') | bool }}" + - include: advanced_reboot/reboot-image-handle.yml when: new_sonic_image is defined - - include: ptf_runner.yml - vars: - ptf_test_name: Advanced-reboot test - ptf_test_dir: ptftests - ptf_test_path: advanced-reboot.ReloadTest - ptf_platform: remote - ptf_platform_dir: ptftests - ptf_qlen: 1000 - ptf_test_params: - - verbose=False - - dut_username=\"{{ ansible_ssh_user }}\" - - dut_hostname=\"{{ ansible_host }}\" - - reboot_limit_in_seconds={{ reboot_limit }} - - reboot_type=\"{{ reboot_type }}\" - - portchannel_ports_file=\"/tmp/portchannel_interfaces.json\" - - vlan_ports_file=\"/tmp/vlan_interfaces.json\" - - ports_file=\"/tmp/ports.json\" - - dut_mac='{{ ansible_Ethernet0['macaddress'] }}' - - dut_vlan_ip='192.168.0.1' - - default_ip_range='192.168.0.0/16' - - vlan_ip_range=\"{{ minigraph_vlan_interfaces[0]['subnet'] }}\" - - lo_v6_prefix=\"{{ minigraph_lo_interfaces | map(attribute='addr') | ipv6 | first | ipsubnet(64) }}\" - - arista_vms=\"['{{ vm_hosts | list | join("','") }}']\" + - include: advanced_reboot/upgrade_mlnx_fw.yml + when: + - new_sonic_image is defined + - reboot_type == "fast-reboot" + - minigraph_hwsku is defined and minigraph_hwsku in mellanox_hwskus + + - include: ptf_runner_reboot.yml + with_items: "{{ preboot_list }}" + + # When new image is defined, test removed /host/config_db.json + # before warm rebooting. So after the device boots up, it will + # miss /etc/sonic/config_db.json. It is not an issue for the + # device to stay up. But it will be an issue when device reboot + # again (cold or fast). + - name: Save configuration after warm rebooting into new image + shell: config save -y + become: yes + when: + - new_sonic_image is defined + - reboot_type == "warm-reboot" - always: - - name: Copy test results from ptf to the local box /tmp/*-reboot.log - fetch: src='/tmp/{{reboot_type}}.log' dest='/tmp/' flat=true fail_on_missing=false - delegate_to: "{{ ptf_host }}" + - name: check /etc/sonic/config_db.json existence + stat: + path: /etc/sonic/config_db.json + register: stat_result - - name: Copy pcap files from ptf to the local box /tmp/ - fetch: src={{ item }} dest='/tmp/' flat=true fail_on_missing=false - delegate_to: "{{ ptf_host }}" - with_items: - - "/tmp/capture.pcap" - - "/tmp/capture_filtered.pcap" + - fail: msg="/etc/sonic/config_db.json is missing" + when: not stat_result.stat.exists + always: - name: Remove existing ip from ptf host script: roles/test/files/helpers/remove_ip.sh delegate_to: "{{ ptf_host }}" - - name: Extract all syslog entries since the last reboot - extract_log: - directory: '/var/log' - file_prefix: 'syslog' - start_string: 'Linux version' - target_filename: '/tmp/syslog' - become: yes - - - name: Copy the exctracted syslog entries to the local machine - fetch: - src: '/tmp/syslog' - dest: '/tmp/' - flat: yes - - - name: Extract all sairedis.rec entries since the last reboot - extract_log: - directory: '/var/log/swss' - file_prefix: 'sairedis.rec' - start_string: 'recording on:' - target_filename: '/tmp/sairedis.rec' - - - name: Copy the exctracted sairedis.rec entries to the local machine - fetch: - src: '/tmp/sairedis.rec' - dest: '/tmp/' - flat: yes - - - name: Extract all swss.rec entries since the last reboot - extract_log: - directory: '/var/log/swss' - file_prefix: 'swss.rec' - start_string: 'recording started' - target_filename: '/tmp/swss.rec' - - - name: Copy the exctracted swss.rec entries to the local machine - fetch: - src: '/tmp/swss.rec' - dest: '/tmp/' - flat: yes - - name: make sure all handlers run meta: flush_handlers diff --git a/ansible/roles/test/tasks/advanced_reboot/reboot-image-handle.yml b/ansible/roles/test/tasks/advanced_reboot/reboot-image-handle.yml new file mode 100755 index 00000000000..62fd8334c3d --- /dev/null +++ b/ansible/roles/test/tasks/advanced_reboot/reboot-image-handle.yml @@ -0,0 +1,53 @@ +- block: + - fail: msg="Please set new_sonic_image variable" + when: new_sonic_image is not defined + + - fail: msg="Please set cleanup_old_sonic_images variable" + when: cleanup_old_sonic_images is not defined + + - fail: msg="Please set stay_in_target_image variable" + when: stay_in_target_image is not defined + + - name: Save image version + shell: 'sonic_installer list | grep Current | cut -f2 -d " "' + register: current_sonic_image + become: true + when: not stay_in_target_image + + - name: Generate temp file name on target device + shell: mktemp + register: tempfile + + - set_fact: + new_image_location: '{{ tempfile.stdout }}' + + - debug: msg='Setting image file name to {{ new_image_location }}' + + - name: Download SONiC image. + local_action: get_url url={{ new_sonic_image }} dest={{ new_image_location }} + + - name: Upload SONiC image to device. + copy: + src: "{{ new_image_location }}" + dest: "{{ new_image_location }}" + + - name: Cleanup sonic images that is not current and/or next + shell: sonic_installer cleanup -y + become: true + when: cleanup_old_sonic_images + + - name: 'Setup restoring initial image {{ current_sonic_image }}' + shell: /bin/true + connection: local + notify: + - restore current image + - reboot sonic + when: not stay_in_target_image + + - name: Installing new SONiC image + shell: sonic_installer install -y {{ new_image_location }} + become: true + + - name: Remove config_db.json so the new image will reload minigraph + file: path=/host/old_config/config_db.json state=absent + become: true diff --git a/ansible/roles/test/tasks/advanced_reboot/upgrade_mlnx_fw.yml b/ansible/roles/test/tasks/advanced_reboot/upgrade_mlnx_fw.yml new file mode 100644 index 00000000000..3bbb0282aa5 --- /dev/null +++ b/ansible/roles/test/tasks/advanced_reboot/upgrade_mlnx_fw.yml @@ -0,0 +1,25 @@ +- name: Get current image version + shell: sonic_installer list | grep Current | cut -f2 -d " " + register: cur_image + become: true + +- name: Get next image version + shell: sonic_installer list | grep Next | cut -f2 -d " " + register: next_image + become: true + +- set_fact: + current_sonic_image={{cur_image.stdout}} + next_sonic_image={{next_image.stdout}} + +- debug: + msg: "current: {{current_sonic_image}} next: {{next_sonic_image}} reboot: {{reboot_type}}" + +- block: + - name: Run the FW upgrade script + script: roles/test/files/mlnx/upgrade_mlnx_fw.sh + become: true + + when: + - current_sonic_image | search('SONiC-OS-201803') + - next_sonic_image | search('SONiC-OS-201811') diff --git a/ansible/roles/test/tasks/arpall.yml b/ansible/roles/test/tasks/arpall.yml index 137392071dd..14e6cad97fb 100644 --- a/ansible/roles/test/tasks/arpall.yml +++ b/ansible/roles/test/tasks/arpall.yml @@ -35,7 +35,7 @@ when: po1 is defined - name: bring {{ intf1 }} up - shell: config interface {{ intf1 }} startup + shell: config interface startup {{ intf1 }} become: yes when: po1 is defined @@ -51,16 +51,16 @@ when: po2 is defined - name: bring {{ intf2 }} up - shell: config interface {{ intf2 }} startup + shell: config interface startup {{ intf2 }} become: yes when: po2 is defined - name: change SONiC DUT interface IP to test IP address - command: config interface {{ intf1 }} ip add 10.10.1.2/28 + command: config interface ip add {{ intf1 }} 10.10.1.2/28 become: yes - name: change SONiC DUT interface IP to test IP address - command: config interface {{ intf2 }} ip add 10.10.1.20/28 + command: config interface ip add {{ intf2 }} 10.10.1.20/28 become: yes - name: wait for interfaces to be up after removed from portchannel @@ -73,7 +73,8 @@ delegate_to: "{{ ptf_host }}" - name: Clear DUT arp cache - command: ip nei flush all + command: ip -stats neigh flush all + ignore_errors: yes become: yes - name: Start PTF runner and Send correct unicast arp packets (10.10.1.3 to 10.10.1.2 with src_mac=00:06:07:08:09:00) @@ -99,7 +100,8 @@ - "{{ arptable['v4']['10.10.1.3']['interface'] == intf1 }}" - name: Clear DUT arp cache - command: ip nei flush all + command: ip -stats neigh flush all + ignore_errors: yes become: yes # Send correct ARP request from correct interface, expecting normal behavior @@ -127,7 +129,8 @@ ## check DUT won't reply ARP and install ARP entry when ARP request coming from other interfaces - name: Clear DUT arp cache - command: ip nei flush all + command: ip -stats neigh flush all + ignore_errors: yes become: yes - name: Send correct arp packets from other interface expect no reply(10.10.1.4 to 10.10.1.2 with src_mac=00:02:07:08:09:0a) @@ -154,7 +157,8 @@ ## check DUT won't reply ARP and install ARP entry when src address is not in interface subnet range - name: Clear DUT arp cache - command: ip nei flush all + command: ip -stats neigh flush all + ignore_errors: yes become: yes - name: Send Src IP out of interface subnet range arp packets, expect no reply and no arp table entry (10.10.1.22 to 10.10.1.2 with src_mac=00:03:07:08:09:0a) @@ -181,7 +185,8 @@ ## Test Gratuitous ARP behavior, no Gratuitous ARP installed when arp was not resolved before - name: Clear DUT arp cache - command: ip nei flush all + command: ip -stats neigh flush all + ignore_errors: yes become: yes - name: Send garp packets (10.10.1.7 to 10.10.1.7) diff --git a/ansible/roles/test/tasks/bgp_speaker.yml b/ansible/roles/test/tasks/bgp_speaker.yml index adfc5046227..86cf8581740 100644 --- a/ansible/roles/test/tasks/bgp_speaker.yml +++ b/ansible/roles/test/tasks/bgp_speaker.yml @@ -7,7 +7,7 @@ when: (testbed_type is not defined or ptf_host is not defined) - fail: msg="Invalid testbed_type value '{{testbed_type}}'" - when: testbed_type not in ['t0', 't0-64', 't0-116'] + when: testbed_type not in testcases['bgp_speaker']['topologies'] - name: Gather minigraph facts about the device minigraph_facts: host={{inventory_hostname}} @@ -198,7 +198,8 @@ - fib_info='/root/bgp_speaker_route.txt' - ipv4={{ ipv4|default(false) }} - ipv6={{ ipv6|default(false) }} - ptf_extra_options: "--relax --debug info --log-file /tmp/bgp_speaker_test.FibTest.log" + - testbed_mtu={{ mtu|default(9114) }} + ptf_extra_options: "--relax --debug info --log-file /tmp/bgp_speaker_test.FibTest.log --socket-recv-size 16384" always: - name: Kill exabgp instances shell: pkill exabgp diff --git a/ansible/roles/test/tasks/check_fanout_interfaces.yml b/ansible/roles/test/tasks/check_fanout_interfaces.yml new file mode 100644 index 00000000000..4a4d4e2f43f --- /dev/null +++ b/ansible/roles/test/tasks/check_fanout_interfaces.yml @@ -0,0 +1,18 @@ +- block: + - name: Gathering lab graph facts about the device + conn_graph_facts: host={{ inventory_hostname }} + connection: local + + - name: Fanout hostname + set_fact: fanout_switch={{ device_conn['Ethernet0']['peerdevice'] }} + + - name: Check Fanout interfaces + local_action: shell ansible-playbook -i lab fanout.yml -l {{ fanout_switch }} --tags check_interfaces_status + ignore_errors: yes + register: fanout_interfaces_status + + - name: Debug Fanout interfaces + debug: msg={{ fanout_interfaces_status }} + when: fanout_interfaces_status is defined + + when: check_fanout is defined diff --git a/ansible/roles/test/tasks/check_sw_vm_interfaces.yml b/ansible/roles/test/tasks/check_sw_vm_interfaces.yml new file mode 100644 index 00000000000..9a09d4968e4 --- /dev/null +++ b/ansible/roles/test/tasks/check_sw_vm_interfaces.yml @@ -0,0 +1,57 @@ +- block: + - name: Get Portchannel status + shell: show interfaces portchannel + register: portchannel_status + ignore_errors: yes + + - name: Get teamd dump + shell: teamdctl '{{ item }}' state dump + with_items: "{{ minigraph_portchannels }}" + ignore_errors: yes + register: teamd_dump + when: + - minigraph_portchannels is defined + + - name: Debug teamd dump + debug: msg={{ teamd_dump }} + when: teamd_dump is defined + + - name: Define testbed_name when not obtained + set_fact: + testbed_name: "{{ inventory_hostname + '-' + topo }}" + when: testbed_name is not defined + + - name: Gathering testbed information + test_facts: testbed_name="{{ testbed_name }}" + connection: local + ignore_errors: yes + + - name: Gather vm list from Testbed server + local_action: shell ansible-playbook testbed_vm_status.yml -i veos -l "{{ testbed_facts['server'] }}" + ignore_errors: yes + register: testbed_vm_list + + - name: Debug VM list on Testbed + debug: msg={{ testbed_vm_list }} + when: testbed_vm_list is defined + + - set_fact: + vms: "{{ minigraph_devices }}" + peer_hwsku: 'Arista-VM' + + - name: Gather Port-Channel status from VMs + action: apswitch template=roles/vm_set/templates/show_int_portchannel_status.j2 + args: + host: "{{ vms[item]['mgmt_addr'] }}" + login: "{{ switch_login[hwsku_map[peer_hwsku]] }}" + connection: switch + ignore_errors: yes + when: vms["{{ item }}"]['hwsku'] == 'Arista-VM' + with_items: vms + register: vm_portchannel_status + + - name: Debug Port-Channel on VMs + debug: msg={{ vm_portchannel_status }} + when: vm_portchannel_status is defined + + when: check_vms is defined diff --git a/ansible/roles/test/tasks/common_tasks/reboot_sonic.yml b/ansible/roles/test/tasks/common_tasks/reboot_sonic.yml index 6440da71ad6..5c844a68723 100644 --- a/ansible/roles/test/tasks/common_tasks/reboot_sonic.yml +++ b/ansible/roles/test/tasks/common_tasks/reboot_sonic.yml @@ -59,3 +59,13 @@ - name: wait for 2 minute for prcesses and interfaces to be stable pause: seconds=120 + +- name: Wait for warmboot-finalizer service to finish + become: true + ignore_errors: true + shell: systemctl is-active warmboot-finalizer.service + register: status + until: status.stdout.find('inactive') != -1 + delay: 10 + retries: 30 + when: reboot_type == 'warm-reboot' diff --git a/ansible/roles/test/tasks/config.yml b/ansible/roles/test/tasks/config.yml index 83060a5b38b..001a82d475d 100644 --- a/ansible/roles/test/tasks/config.yml +++ b/ansible/roles/test/tasks/config.yml @@ -1,8 +1,5 @@ - debug: msg="Configuration Test" -- fail: msg="testbed_type {{testbed_type}} is not supported" - when: testbed_type not in ['t1-lag', 't1-64-lag'] - - name: Gather minigraph facts minigraph_facts: host={{inventory_hostname}} @@ -45,7 +42,7 @@ remove_portchannel_members: true - name: Step 2 Remove {{ portchannel_ip }} from {{ portchannel }} - shell: config interface {{ portchannel }} ip remove {{ portchannel_ip }}/31 + shell: config interface ip remove {{ portchannel }} {{ portchannel_ip }}/31 become: yes - set_fact: remove_portchannel_ip: true @@ -76,7 +73,7 @@ add_tmp_portchannel_members: true - name: Step 5 Add {{ portchannel_ip }} to {{ tmp_portchannel }} - shell: config interface {{ tmp_portchannel }} ip add {{ portchannel_ip }}/31 + shell: config interface ip add {{ tmp_portchannel }} {{ portchannel_ip }}/31 become: yes - set_fact: add_tmp_portchannel_ip: true @@ -100,7 +97,7 @@ always: - name: Remove {{ portchannel_ip }} from {{ tmp_portchannel }} - shell: config interface {{ tmp_portchannel }} ip remove {{ portchannel_ip }}/31 + shell: config interface ip remove {{ tmp_portchannel }} {{ portchannel_ip }}/31 become: yes when: add_tmp_portchannel_ip @@ -116,7 +113,7 @@ when: create_tmp_portchannel - name: Add {{ portchannel_ip }} to {{ portchannel }} - shell: config interface {{ portchannel }} ip add {{ portchannel_ip }}/31 + shell: config interface ip add {{ portchannel }} {{ portchannel_ip }}/31 become: yes when: remove_portchannel_ip diff --git a/ansible/roles/test/tasks/decap.yml b/ansible/roles/test/tasks/decap.yml index ac5ee5d715e..ce08e508335 100644 --- a/ansible/roles/test/tasks/decap.yml +++ b/ansible/roles/test/tasks/decap.yml @@ -103,7 +103,7 @@ # Separate set_fact is required to be able to use 'testname' fact. - set_fact: - testname_unique: "{{ testname }}.{{ ansible_date_time.date}}.{{ ansible_date_time.hour}}-{{ ansible_date_time.minute}}-{{ ansible_date_time.second}}" + testname_unique: "{{ testname }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}" - debug: msg="generated run id:{{testname_unique}}" diff --git a/ansible/roles/test/tasks/everflow_testbed/apply_config.yml b/ansible/roles/test/tasks/everflow_testbed/apply_config.yml index dde5b04e9d7..c75e1b0337d 100644 --- a/ansible/roles/test/tasks/everflow_testbed/apply_config.yml +++ b/ansible/roles/test/tasks/everflow_testbed/apply_config.yml @@ -8,13 +8,11 @@ - name: Get session info. include: roles/test/tasks/everflow_testbed/get_session_info.yml -- name: Copy python script for session configuration. - copy: src=roles/test/files/helpers/mirror_session.py dest={{ run_dir }}/ - - name: Copy ACL rules configuration file. copy: src={{ tests_location }}/{{ testname}}/acl_rule_persistent.json dest={{ run_dir }}/ -- command: "python {{ run_dir }}/mirror_session.py create {{ session_name }} {{ session_src_ip }} {{ session_dst_ip }} {{ session_gre }} {{ session_dscp }} {{ session_ttl }} {{ session_queue }}" +- command: "config mirror_session add {{session_name}} {{session_src_ip}} {{session_dst_ip}} {{session_dscp}} {{session_ttl}} {{session_gre}} {{session_queue}}" + become: yes - command: "acl-loader update full {{ run_dir }}/acl_rule_persistent.json --session_name={{ session_name }}" become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/del_config.yml b/ansible/roles/test/tasks/everflow_testbed/del_config.yml index 07b688e1d3a..4bcff2f2d11 100644 --- a/ansible/roles/test/tasks/everflow_testbed/del_config.yml +++ b/ansible/roles/test/tasks/everflow_testbed/del_config.yml @@ -8,13 +8,11 @@ - name: Get session info. include: roles/test/tasks/everflow_testbed/get_session_info.yml -- name: Copy python script for session configuration. - copy: src=roles/test/files/helpers/mirror_session.py dest={{ run_dir }}/ - - name: Copy ACL rules configuration file. copy: src={{ tests_location }}/{{ testname}}/acl_rule_persistent-del.json dest={{ run_dir }}/ - command: "acl-loader update full {{ run_dir }}/acl_rule_persistent-del.json" become: yes -- command: "python {{ run_dir }}/mirror_session.py delete {{ session_name }}" +- command: "config mirror_session remove {{session_name}}" + become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/get_port_info.yml b/ansible/roles/test/tasks/everflow_testbed/get_port_info.yml index a6d46508f80..01851ded32e 100644 --- a/ansible/roles/test/tasks/everflow_testbed/get_port_info.yml +++ b/ansible/roles/test/tasks/everflow_testbed/get_port_info.yml @@ -5,6 +5,7 @@ set_fact: tor_ports: [] spine_ports: [] + spine_ptf_ports: [] dst_port_1_is_lag_member: "" dst_port_1_ptf_id: "" dst_port_2: "" @@ -20,12 +21,8 @@ with_dict: "{{ minigraph_neighbors }}" when: "'T0' in item.value.name" -- name: Sort tor ports by name. - set_fact: - tor_port: "{{ tor_ports|sort }}" - - name: Print tor ports - debug: msg="{{ tor_ports }}" + debug: msg={{ tor_ports }} - name: Get spine ports set_fact: @@ -33,12 +30,13 @@ with_dict: "{{ minigraph_neighbors }}" when: "'T2' in item.value.name" -- name: Sort tor ports by name. - set_fact: - tor_port: "{{ spine_ports|sort }}" - - name: Print spine ports - debug: msg="{{ spine_ports }}" + debug: msg={{ spine_ports }} + +- name: Define spine PTF ports + set_fact: + spine_ptf_ports: "{{ spine_ptf_ports + [minigraph_port_indices[item] | string] }}" + with_items: "{{ spine_ports }}" - name: Define SRC port variables. set_fact: diff --git a/ansible/roles/test/tasks/everflow_testbed/get_session_info.yml b/ansible/roles/test/tasks/everflow_testbed/get_session_info.yml index 6642472747e..a1fcd77c058 100644 --- a/ansible/roles/test/tasks/everflow_testbed/get_session_info.yml +++ b/ansible/roles/test/tasks/everflow_testbed/get_session_info.yml @@ -13,7 +13,7 @@ session_gre: "0x8949" when: sonic_hwsku in mellanox_hwskus -- debug: msg="session name {{ session_name }}" +- debug: msg=session name {{ session_name }} - set_fact: addr_1: "{{ session_dst_ip }}/24" diff --git a/ansible/roles/test/tasks/everflow_testbed/run_test.yml b/ansible/roles/test/tasks/everflow_testbed/run_test.yml index bc72ced85fd..847a675333d 100644 --- a/ansible/roles/test/tasks/everflow_testbed/run_test.yml +++ b/ansible/roles/test/tasks/everflow_testbed/run_test.yml @@ -12,7 +12,7 @@ minigraph_facts: host={{ inventory_hostname }} - name: Print neighbors in minigraph - debug: msg="{{ minigraph_neighbors }}" + debug: msg={{ minigraph_neighbors }} - set_fact: testname: everflow_testbed @@ -44,7 +44,7 @@ delegate_to: "{{ ptf_host }}" - name: Add route to unresolved next hop. - shell: ip route add {{ unresolved_nexthop_prefix }} dev {{ dst_port_2 }} + shell: vtysh -e "conf t" -e "ip route {{ unresolved_nexthop_prefix }} {{ dst_port_2 }}" become: yes - name: Run testcase 1 - Resolved route @@ -69,12 +69,12 @@ - name: Run testcase 7 - ECMP route change (remove next hop used by session). include: roles/test/tasks/everflow_testbed/testcase_7.yml - - name: Run testcase 8 - ECMP route change (remove next hop not used by session). + - name: Run testcase 8 - Policer enforced with DSCP value/mask include: roles/test/tasks/everflow_testbed/testcase_8.yml always: - name: Remove route to unresolved next hop. - shell: ip route del {{ unresolved_nexthop_prefix }} + shell: vtysh -e "conf t" -e "no ip route {{ unresolved_nexthop_prefix }} {{ dst_port_2 }}" become: yes - include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_1.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_1.yml index 1da85f49817..74c4142eef9 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_1.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_1.yml @@ -2,9 +2,12 @@ # Verify that session with resolved route has active state. - name: Create route with next hop {{ dst_port_1 }}. - shell: ip route add {{ session_prefix_1 }} via {{ neighbor_info_1['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" become: yes +- pause: + seconds: 3 + - block: - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -15,5 +18,5 @@ always: - name: Remove route - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_2.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_2.yml index ae69cf97db8..84f414e1363 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_2.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_2.yml @@ -3,7 +3,10 @@ - block: - name: Create route with next hop on {{ dst_port_1 }}. - shell: ip route add {{ session_prefix_1 }} via {{ neighbor_info_1['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -13,7 +16,10 @@ register: out - name: Create route with best match and unresolved next hop. - shell: ip route add {{ session_prefix_2 }} via {{ unresolved_nexthop }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_2 }} {{ unresolved_nexthop }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -23,7 +29,10 @@ register: out - name: Create route with best match prefix and resolved next hop on destination port {{ dst_port_2 }}. - shell: ip route change {{ session_prefix_2 }} via {{ neighbor_info_2['addr'] }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_2 }} {{ unresolved_nexthop }}" -e "ip route {{ session_prefix_2 }} {{ neighbor_info_2['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_2 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -35,10 +44,14 @@ always: - name: Remove route. - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + ignore_errors: yes + + - name: Remove best match route. + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_2 }} {{ unresolved_nexthop }}" ignore_errors: yes - name: Remove best match route. - shell: ip route del {{ session_prefix_2 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_2 }} {{ neighbor_info_2['addr'] }}" ignore_errors: yes become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_3.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_3.yml index b5894488cb4..0eefbcc3cf2 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_3.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_3.yml @@ -3,7 +3,10 @@ - block: - name: Create route with next hop on {{ dst_port_1 }}. - shell: ip route add {{ session_prefix_1 }} via {{ neighbor_info_1['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -13,7 +16,10 @@ register: out - name: Create route with best match prefix and resolved next hop {{ dst_port_2 }}. - shell: ip route add {{ session_prefix_2 }} via {{ neighbor_info_2['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_2 }} {{ neighbor_info_2['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_2}}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -23,8 +29,10 @@ register: out - name: Remove best match route. - shell: ip route del {{ session_prefix_2 }} - ignore_errors: yes + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_2 }} {{ neighbor_info_2['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -36,10 +44,10 @@ always: - name: Remove route. - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" ignore_errors: yes - name: Remove best match route. - shell: ip route del {{ session_prefix_2 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_2 }} {{ neighbor_info_2['addr'] }}" ignore_errors: yes become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_4.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_4.yml index e67912cc5de..b388ffdd7d6 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_4.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_4.yml @@ -3,7 +3,10 @@ - block: - name: Create route with next hop on {{ dst_port_1 }}. - shell: ip route add {{ session_prefix_1 }} via {{ neighbor_info_1['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";expected_dst_mac="{{ neighbor_mac_1 }}";verbose=True' @@ -15,6 +18,9 @@ - name: Change neighbor MAC address. shell: ip neigh replace {{ neighbor_info_1['addr'] }} lladdr "00:11:22:33:44:55" nud permanent dev {{ dst_port_1 }} + - pause: + seconds: 3 + - name: Send traffic and verify that packets with correct Everflow header are received on destination port {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";expected_dst_mac="00:11:22:33:44:55";verbose=True' args: @@ -31,6 +37,6 @@ shell: ping {{ neighbor_info_1['addr'] }} -c3 - name: Remove route. - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" ignore_errors: yes become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_5.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_5.yml index aa128dc3417..fe953eacfc8 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_5.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_5.yml @@ -1,9 +1,12 @@ # Test case 5 - Resolved ECMP route. - name: Create ECMP route with next hops on {{ dst_port_1 }} and {{ dst_port_2 }}. - shell: ip route add {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" become: yes +- pause: + seconds: 3 + - block: - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -14,5 +17,5 @@ always: - name: Remove route - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_6.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_6.yml index 4fafe2b658a..9445d648179 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_6.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_6.yml @@ -1,9 +1,25 @@ -# Test case 6 - ECMP route change (add next hop). -# Verify that insertion of additional next hop to ECMP group doesn't affects session DST MAC and port. +# Test case 8 - ECMP route change (remove next hop not used by session). +# Verify that after removal of next hop that was used by session from ECMP route session state is active. - block: - name: Create ECMP route with next hops on {{ dst_port_1 }} and {{ dst_port_2 }}. - shell: ip route add {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" + + - pause: + seconds: 3 + + - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. + shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' + args: + chdir: /root + delegate_to: "{{ ptf_host }}" + register: out + + - name: Add next hop to ECMP route. + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_3['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -12,8 +28,19 @@ delegate_to: "{{ ptf_host }}" register: out - - name: Add next hop on {{ dst_port_3 }} to ECMP route. - shell: ip route change {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} via {{ neighbor_info_3['addr'] }} + - name: Send traffic and verify that packets are not received on {{ dst_port_3 }}. + shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_3_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' + args: + chdir: /root + delegate_to: "{{ ptf_host }}" + register: out + failed_when: out.rc == 0 + + - name: Delete next hop from ECMP route. + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_3['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -33,5 +60,6 @@ always: - name: Remove route - shell: ip route del {{ session_prefix_1 }} - become: yes + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_3['addr'] }}" + ignore_errors: yes + become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_7.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_7.yml index 4c628e11292..92ebb6d1f1e 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_7.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_7.yml @@ -3,7 +3,10 @@ - block: - name: Create route with next hop on {{ dst_port_1 }}. - shell: ip route add {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -13,7 +16,10 @@ register: out - name: Add next hops on {{ dst_port_2 }} and {{ dst_port_3 }} to route. - shell: ip route change {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} via {{ neighbor_info_3['addr'] }} + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_3['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -30,8 +36,11 @@ register: out failed_when: out.rc == 0 - - name: Delete next hop from ECMP route. - shell: ip route change {{ session_prefix_1 }} nexthop via {{ neighbor_info_2['addr'] }} via {{ neighbor_info_3['addr'] }} + - name: Delete one next hop from ECMP route. + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + + - pause: + seconds: 3 - name: Send traffic and verify that packets are not received {{ dst_port_1 }}. shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' @@ -50,6 +59,7 @@ become: yes always: - - name: Remove route - shell: ip route del {{ session_prefix_1 }} - become: yes + - name: Remove route {{session_prefix_1}} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_2['addr'] }}" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_3['addr'] }}" + ignore_errors: yes + become: yes diff --git a/ansible/roles/test/tasks/everflow_testbed/testcase_8.yml b/ansible/roles/test/tasks/everflow_testbed/testcase_8.yml index ee99a10c219..de87836dc3b 100644 --- a/ansible/roles/test/tasks/everflow_testbed/testcase_8.yml +++ b/ansible/roles/test/tasks/everflow_testbed/testcase_8.yml @@ -1,55 +1,77 @@ -# Test case 8 - ECMP route change (remove next hop not used by session). -# Verify that after removal of next hop that was used by session from ECMP route session state is active. +# Test case 8 - Policer enforced DSCP value/mask test -- block: - - name: Create ECMP route with next hops on {{ dst_port_1 }} and {{ dst_port_2 }}. - shell: ip route add {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} - - - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. - shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' - args: - chdir: /root - delegate_to: "{{ ptf_host }}" - register: out - - - name: Add next hop to ECMP route. - shell: ip route change {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} via {{ neighbor_info_3['addr'] }} - - - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. - shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' - args: - chdir: /root - delegate_to: "{{ ptf_host }}" - register: out - - - name: Send traffic and verify that packets are not received on {{ dst_port_3 }}. - shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_3_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' - args: - chdir: /root - delegate_to: "{{ ptf_host }}" - register: out - failed_when: out.rc == 0 - - - name: Delete next hop from ECMP route. - shell: ip route change {{ session_prefix_1 }} nexthop via {{ neighbor_info_1['addr'] }} via {{ neighbor_info_2['addr'] }} - - - name: Send traffic and verify that packets with correct Everflow header are received on {{ dst_port_1 }} or {{ dst_port_2 }}. - shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_1_ptf_id }}, {{ dst_port_2_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' - args: - chdir: /root - delegate_to: "{{ ptf_host }}" - register: out - - - name: Send traffic and verify that packets are not received on {{ dst_port_3 }}. - shell: ptf --test-dir acstests everflow_tb_test.EverflowTest --platform-dir ptftests --platform remote -t 'asic_type="{{ sonic_asic_type }}";hwsku="{{ sonic_hwsku }}";router_mac="{{ ansible_Ethernet0['macaddress'] }}";src_port="{{ src_port_ptf_id }}";dst_ports="{{ dst_port_3_ptf_id }}";session_src_ip="{{ session_src_ip }}";session_dst_ip="{{ session_dst_ip }}";session_ttl="{{ session_ttl }}";session_dscp="{{ session_dscp }}";verbose=True' - args: - chdir: /root - delegate_to: "{{ ptf_host }}" - register: out - failed_when: out.rc == 0 +- set_fact: + policer_name: TEST_POLICER + policer_session_name: TEST_POLICER_SESSION + dscp_table_name: EVERFLOW_DSCP + +- name: Create route with next hop {{ dst_port_1 }}. + shell: vtysh -e "conf t" -e "ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" become: yes +- block: + - name: Create a policer + shell: | + redis-cli -n 4 hmset "POLICER|{{policer_name}}" "meter_type" "packets" "mode" "sr_tcm" "cir" "100" "cbs" "100" "red_packet_action" "drop" + become: yes + + - name: Create a policer enforced mirror session + shell: | + config mirror_session add {{policer_session_name}} {{session_src_ip}} {{session_dst_ip}} {{session_dscp}} {{session_ttl}} --policer {{policer_name}} + become: yes + + - name: Create an ACL table with MIRROR_DSCP type + shell: config acl add table {{dscp_table_name}} "MIRROR_DSCP" --description "EVERFLOW_TEST" + become: yes + + - name: Create a rule with DSCP value and mask + shell: | + redis-cli -n 4 hmset "ACL_RULE|{{dscp_table_name}}|RULE_1" "PRIORITY" "9999" "MIRROR_ACTION" "{{policer_session_name}}" "DSCP" "8/56" + become: yes + + - name: "Start PTF runner" + include: roles/test/tasks/ptf_runner.yml + vars: + ptf_test_name: EVERFLOW Policer Test + ptf_test_dir: acstests + ptf_test_path: everflow_policer_test.EverflowPolicerTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - asic_type='{{sonic_asic_type}}' + - hwsku='{{sonic_hwsku}}' + - router_mac='{{ansible_Ethernet0['macaddress']}}' + - src_port='{{src_port_ptf_id}}' + - dst_ports='{{",".join((spine_ptf_ports))}}' + - dst_mirror_ports='{{dst_port_1_ptf_id}}' + ptf_extra_options: "--relax --debug info" + always: - name: Remove route - shell: ip route del {{ session_prefix_1 }} + shell: vtysh -e "conf t" -e "no ip route {{ session_prefix_1 }} {{ neighbor_info_1['addr'] }}" + ignore_errors: yes become: yes + + - name: Remove the rule with DSCP value and mask + shell: | + redis-cli -n 4 del "ACL_RULE|{{dscp_table_name}}|RULE_1" + ignore_errors: yes + become: yes + + - name: Remove the ACL table with MIRROR_DSCP type + shell: config acl remove table {{dscp_table_name}} + ignore_errors: yes + become: yes + + - name: Remove the policer enforced mirror session + shell: | + config mirror_session remove {{policer_session_name}} + ignore_errors: yes + become: yes + + - name: Remove policer + shell: | + redis-cli -n 4 del "POLICER|{{policer_name}}" + ignore_errors: yes + become: yes + diff --git a/ansible/roles/test/tasks/fdb_mac_expire.yml b/ansible/roles/test/tasks/fdb_mac_expire.yml new file mode 100644 index 00000000000..56d3a070e67 --- /dev/null +++ b/ansible/roles/test/tasks/fdb_mac_expire.yml @@ -0,0 +1,115 @@ +- fail: msg="testbed_type is not defined" + when: testbed_type is not defined + +- fail: msg="testbed_type {{test_type}} is invalid" + when: testbed_type not in ['t0', 't0-64', 't0-64-32', 't0-116'] + +- name: set fdb_aging_time to default if no user input + set_fact: + fdb_aging_time: 60 + when: fdb_aging_time is not defined + +- include_vars: "vars/topo_{{testbed_type}}.yml" + +- name: Gather minigraph facts about the device + minigraph_facts: host={{inventory_hostname}} + +- name: Copy tests to PTF + copy: src=roles/test/files/ptftests dest=/root + delegate_to: "{{ptf_host}}" + +- name: Copy FDB information file to PTF + template: src=roles/test/templates/fdb.j2 dest=/root/fdb_info.txt + delegate_to: "{{ ptf_host }}" + +- name: Clear FDB table + shell: sonic-clear fdb all + + # Change the config, populate fdb and observe expire time +- block: + - name: copy current switch.json from docker to host + shell: docker cp swss:/etc/swss/config.d/switch.json . + + - name: set fdb value to "{{fdb_aging_time}}" + replace: + dest: switch.json + regexp: '"fdb_aging_time": ".*"' + replace: '"fdb_aging_time": "{{fdb_aging_time}}"' + become: true + + - name: copy current switch.json from host to docker + shell: docker cp switch.json swss:/etc/swss/config.d/switch.json + + - name: run swssconfig switch.json command in container swss + shell: docker exec swss bash -c "swssconfig /etc/swss/config.d/switch.json" + + - name: set dummy mac prefix to look for in mac table + set_fact: dummy_mac_prefix="00:11:22:33:44" + + - name: check entries in mac table before adding dummy mac + shell: show mac | grep {{dummy_mac_prefix}} | wc -l + register: show_mac_output + failed_when: "show_mac_output.stdout|int > 0" + + - debug: msg="{{show_mac_output.stdout}}" + + - name: "Start PTF runner" + include: ptf_runner.yml + vars: + ptf_test_name: FDB Mac Expire test + ptf_test_dir: ptftests + ptf_test_path: fdb_mac_expire_test.FdbMacExpireTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - testbed_type='{{testbed_type}}' + - router_mac='{{ansible_Ethernet0['macaddress']}}' + - fdb_info='/root/fdb_info.txt' + - dummy_mac_prefix='{{dummy_mac_prefix}}' + ptf_extra_options: "--relax --debug info --log-file /tmp/fdb_mac_expire_test.FdbMacExpireTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + + - name: check entries in mac table after adding dummy mac + shell: show mac | grep {{dummy_mac_prefix}} | wc -l + register: show_mac_output + failed_when: "show_mac_output.stdout|int == 0" + + - debug: msg="{{show_mac_output}}" + + - name: wait for "{{fdb_aging_time}}" secs + pause: seconds="{{fdb_aging_time}}" + + - name: check entries in mac table after wait + shell: show mac | grep {{dummy_mac_prefix}} | wc -l + register: show_mac_after_wait + + - debug: msg="{{show_mac_after_wait}}" + + - name: set extra wait time period + set_fact: + extra_retries: "{{fdb_aging_time|int / 15 + 1}}" + + - debug: msg="{{extra_retries}}" + + # wait in slot of 15 secs to find when MAC expires + - block: + - name: check in mac table after "{{fdb_aging_time}}" secs to find exact time + shell: show mac | grep {{dummy_mac_prefix}} | wc -l + register: show_mac_after_more_wait + until: "show_mac_after_more_wait.stdout|int == 0" + retries: "{{extra_retries|int}}" + delay: 15 + + - fail: + msg: "MAC Entires are not cleaned even after {{2*fdb_aging_time|int}} secs" + when: "show_mac_after_more_wait.stdout|int > 0" + + - debug: msg="MAC Entires are Cleared within {{2*fdb_aging_time|int}} secs." + when: "show_mac_after_wait|int > 0" + + - debug: msg="MAC Entires are Cleared {{fdb_aging_time}} secs." + when: "show_mac_after_wait|int == 0" + + always: + - name: Clear FDB table + shell: sonic-clear fdb all + diff --git a/ansible/roles/test/tasks/fib/fib_ignore_messages.txt b/ansible/roles/test/tasks/fib/fib_ignore_messages.txt index e69de29bb2d..03bada03d8b 100644 --- a/ansible/roles/test/tasks/fib/fib_ignore_messages.txt +++ b/ansible/roles/test/tasks/fib/fib_ignore_messages.txt @@ -0,0 +1 @@ +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/tasks/interface.yml b/ansible/roles/test/tasks/interface.yml index bd6f8dc639f..90fb9134a77 100644 --- a/ansible/roles/test/tasks/interface.yml +++ b/ansible/roles/test/tasks/interface.yml @@ -32,12 +32,26 @@ - debug: msg="Found link down ports {{ansible_interface_link_down_ports}}" when: ansible_interface_link_down_ports | length > 0 -- name: Verify interfaces are up correctly - assert: { that: "{{ ansible_interface_link_down_ports | length }} == 0" } +- block: + - name: Verify interfaces are up correctly + assert: { that: "{{ ansible_interface_link_down_ports | length }} == 0" } + rescue: + - include: check_fanout_interfaces.yml + vars: + check_fanout: true + - debug: msg="Not all Interfaces are up" -- name: Verify port channel interfaces are up correctly - assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" } - with_items: "{{ minigraph_portchannels.keys() }}" +- block: + - name: Verify port channel interfaces are up correctly + assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" } + with_items: "{{ minigraph_portchannels.keys() }}" + + rescue: + - include: check_sw_vm_interfaces.yml + vars: + check_vms: true + - debug: msg="Not all PortChannels are up '{{ portchannel_status['stdout_lines'] }}' " + when: portchannel_status is defined - name: Verify VLAN interfaces are up correctly assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" } diff --git a/ansible/roles/test/tasks/lag.yml b/ansible/roles/test/tasks/lag.yml index 25197d86157..3be043c1a69 100644 --- a/ansible/roles/test/tasks/lag.yml +++ b/ansible/roles/test/tasks/lag.yml @@ -33,7 +33,7 @@ # Separate set_fact is required to be able to use 'testname' fact. - set_fact: - testname_unique: "{{ testname }}.{{ ansible_date_time.date}}.{{ ansible_date_time.hour}}-{{ ansible_date_time.minute}}-{{ ansible_date_time.second}}" + testname_unique: "{{ testname }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}" # Separate set_fact is required to be able to use 'testname_unique' fact. - set_fact: diff --git a/ansible/roles/test/tasks/lag/lag_ignore_messages.txt b/ansible/roles/test/tasks/lag/lag_ignore_messages.txt index e69de29bb2d..03bada03d8b 100644 --- a/ansible/roles/test/tasks/lag/lag_ignore_messages.txt +++ b/ansible/roles/test/tasks/lag/lag_ignore_messages.txt @@ -0,0 +1 @@ +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/tasks/lag_minlink.yml b/ansible/roles/test/tasks/lag_minlink.yml index f2c9028a0bd..7b5af1f72c0 100644 --- a/ansible/roles/test/tasks/lag_minlink.yml +++ b/ansible/roles/test/tasks/lag_minlink.yml @@ -45,7 +45,7 @@ connection: switch - pause: - seconds: 20 + seconds: 35 - lag_facts: host={{ inventory_hostname }} diff --git a/ansible/roles/test/tasks/link_flap.yml b/ansible/roles/test/tasks/link_flap.yml index 69ad611761b..2a1ae0256a3 100644 --- a/ansible/roles/test/tasks/link_flap.yml +++ b/ansible/roles/test/tasks/link_flap.yml @@ -7,8 +7,9 @@ connection: local tags: always -- set_fact: - neighbors: "{{device_conn}}" +- name: Set neighbor facts + set_fact: + neighbors: "{{ device_conn }}" - include: link_flap/link_flap_helper.yml with_items: "{{ device_conn.keys() }}" diff --git a/ansible/roles/test/tasks/link_flap/link_flap_helper.yml b/ansible/roles/test/tasks/link_flap/link_flap_helper.yml index 9ef6ba54d91..ef3ae6dd81f 100644 --- a/ansible/roles/test/tasks/link_flap/link_flap_helper.yml +++ b/ansible/roles/test/tasks/link_flap/link_flap_helper.yml @@ -3,10 +3,14 @@ # flapped. - block: - - set_fact: - interface: "{{item}}" + - name: Set interface name + set_fact: + interface: "{{ item }}" - - debug: msg={{interface}} + - name: Set default link timeout + set_fact: + link_timeout: 20 + link_delay: 5 - set_fact: peer_device: "{{neighbors[interface]['peerdevice']}}" @@ -42,8 +46,13 @@ delegate_to: "{{peer_host}}" when: peer_type == "FanoutLeafSonic" - - pause: - seconds: 20 + - name: Wait until interface {{ interface }} on {{ inventory_hostname }} is down + interface_facts: up_ports="[ '{{ interface }}' ]" + register: out + until: out.ansible_facts.ansible_interface_link_down_ports | length > 0 + retries: "{{ (link_timeout / link_delay) | round(0, 'ceil') | int }}" + delay: "{{ link_delay }}" + when: "interface in minigraph_ports.keys()" - interface_facts: up_ports={{minigraph_ports | difference(intfs_to_exclude)}} @@ -71,8 +80,13 @@ delegate_to: "{{peer_host}}" when: peer_type == "FanoutLeafSonic" - - pause: - seconds: 20 + - name: Wait until interface {{ interface }} on {{ inventory_hostname }} is up + interface_facts: up_ports="[ '{{ interface }}' ]" + register: out + until: out.ansible_facts.ansible_interface_link_down_ports | length == 0 + retries: "{{ (link_timeout / link_delay) | round(0, 'ceil') | int }}" + delay: "{{ link_delay }}" + when: "interface in minigraph_ports.keys()" - interface_facts: up_ports={{minigraph_ports}} diff --git a/ansible/roles/test/tasks/neighbour-mac.yml b/ansible/roles/test/tasks/neighbour-mac.yml index 7f3b11f1cec..fbf9d3e189c 100644 --- a/ansible/roles/test/tasks/neighbour-mac.yml +++ b/ansible/roles/test/tasks/neighbour-mac.yml @@ -12,7 +12,7 @@ - name: Change DUT interface IP to test IP address become: yes - command: /sbin/ifconfig {{ dut_if }} {{ dut_ip }} netmask 255.255.255.0 + command: config interface ip add {{ dut_if }} {{ dut_ip }}/24 - name: Change host interface IP to test IP address become: yes diff --git a/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages b/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages index ffa8a48f365..b93ffa1530b 100644 --- a/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages +++ b/ansible/roles/test/tasks/pfc_wd/config_test/config_test_ignore_messages @@ -5,3 +5,4 @@ r, ".* Unknown.*" r, ".* SAI_STATUS_ATTR_NOT_SUPPORT.*" r, ".* snmp.*" r, ".* Trying to remove nonexisting queue from flex counter .*" +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml index dc592a91a4a..3ef4c0abf7c 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml @@ -1,11 +1,22 @@ -- name: Create pfc generater file in case it doesn't exist. - file: path=/mnt/flash/{{pfc_gen_file}} state=touch - delegate_to: "{{peer_mgmt}}" - become: true - when: peer_hwsku | search("Arista") or peer_hwsku | search("arista") +- block: + - name: Ensure destination directory exists on fanout + file: + path: "/mnt/flash/" + state: directory + delegate_to: "{{peer_mgmt}}" + become: true + + - name: Create pfc generator file in case it doesn't exist. + file: + path: "/mnt/flash/{{pfc_gen_file}}" + state: touch + delegate_to: "{{peer_mgmt}}" + become: true -- name: Deploy PFC generator to the fanout switch - copy: src=roles/test/files/helpers/{{pfc_gen_file}} dest=/mnt/flash - delegate_to: "{{peer_mgmt}}" - become: true + - name: Deploy PFC generator to the fanout switch + copy: + src: "roles/test/files/helpers/{{pfc_gen_file}}" + dest: "/mnt/flash" + delegate_to: "{{peer_mgmt}}" + become: true when: peer_hwsku | search("Arista") or peer_hwsku | search("arista") diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages index 4b211076a6b..391d16ed990 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages @@ -6,3 +6,4 @@ r, ".* SAI_STATUS_ATTR_NOT_SUPPORT.*" r, ".* snmp.*" r, ".* Trying to remove nonexisting queue from flex counter .*" r, ".* SAI_STATUS_BUFFER_OVERFLOW" +r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" diff --git a/ansible/roles/test/tasks/port_toggle.yml b/ansible/roles/test/tasks/port_toggle.yml index e77fc5da1f7..4e6f02b38fd 100644 --- a/ansible/roles/test/tasks/port_toggle.yml +++ b/ansible/roles/test/tasks/port_toggle.yml @@ -1,8 +1,8 @@ - name: build shell command string - debug: msg="PORTS={{minigraph_ports.keys() | join(' ')}}; for port in $PORTS; do config interface $port shutdown; done" + debug: msg="PORTS={{minigraph_ports.keys() | join(' ')}}; for port in $PORTS; do config interface shutdown $port; done" - name: turn off all ports on device - shell: PORTS="{{minigraph_ports.keys() | join(' ')}}"; for port in $PORTS; do config interface $port shutdown; done + shell: PORTS="{{minigraph_ports.keys() | join(' ')}}"; for port in $PORTS; do config interface shutdown $port; done become: yes - name: Get interface facts @@ -13,10 +13,10 @@ - always: - name: build shell command string - debug: msg="PORTS={{minigraph_ports.keys() | join(' ')}}; for port in $PORTS; do config interface $port startup; done" + debug: msg="PORTS={{minigraph_ports.keys() | join(' ')}}; for port in $PORTS; do config interface startup $port; done" - name: turn on all ports on device - shell: PORTS="{{minigraph_ports.keys() | join(' ')}}"; for port in $PORTS; do config interface $port startup; done + shell: PORTS="{{minigraph_ports.keys() | join(' ')}}"; for port in $PORTS; do config interface startup $port; done become: yes - name: wait 1 minute for ports to come up diff --git a/ansible/roles/test/tasks/ptf_runner.yml b/ansible/roles/test/tasks/ptf_runner.yml index 467d946d617..67a5f15d45b 100644 --- a/ansible/roles/test/tasks/ptf_runner.yml +++ b/ansible/roles/test/tasks/ptf_runner.yml @@ -51,5 +51,36 @@ - debug: var=out.stdout_lines +- name: Set default PTF log filename + set_fact: + ptf_log_file: "/root/ptf.log" + ptf_log_file_param_index: "{{ out.cmd.find('--log-file') }}" + +- name: Parse custom log filename specified in PTF command + set_fact: + ptf_log_file: "{{ out.cmd[ptf_log_file_param_index|int:].split(' ')[1] }}" + when: ptf_log_file_param_index|int >= 0 + +- name: Set PTF pcap filename + set_fact: + ptf_pcap_file: "{{ ptf_log_file | replace('.log', '.pcap') }}" + +- name : Fetch result files from switch to ansible machine + fetch: + src: "{{ item }}" + dest: "test/{{ inventory_hostname }}/ptf/{{ item | basename }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}" + flat: yes + with_items: + - "{{ ptf_log_file }}" + - "{{ ptf_pcap_file }}" + delegate_to: "{{ ptf_host }}" + when: out.rc != 0 and save_ptf_log is defined and save_ptf_log|bool == true + +- debug: msg="File {{ item }} saved to test/{{ inventory_hostname }}/ptf/" + with_items: + - "{{ptf_log_file}}" + - "{{ptf_pcap_file}}" + when: out.rc != 0 and save_ptf_log is defined and save_ptf_log|bool == true + - fail: msg="Failed test '{{ ptf_test_name }}'" when: out.rc != 0 diff --git a/ansible/roles/test/tasks/ptf_runner_reboot.yml b/ansible/roles/test/tasks/ptf_runner_reboot.yml new file mode 100644 index 00000000000..09ac5a61c94 --- /dev/null +++ b/ansible/roles/test/tasks/ptf_runner_reboot.yml @@ -0,0 +1,105 @@ +- block: + - include: ptf_runner.yml + vars: + ptf_test_name: Advanced-reboot test + ptf_test_dir: ptftests + ptf_test_path: advanced-reboot.ReloadTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_qlen: 1000 + ptf_test_params: + - verbose=False + - dut_username=\"{{ ansible_ssh_user }}\" + - dut_hostname=\"{{ ansible_host }}\" + - reboot_limit_in_seconds={{ reboot_limit }} + - reboot_type=\"{{ reboot_type }}\" + - portchannel_ports_file=\"/tmp/portchannel_interfaces.json\" + - vlan_ports_file=\"/tmp/vlan_interfaces.json\" + - ports_file=\"/tmp/ports.json\" + - dut_mac='{{ dut_mac }}' + - dut_vlan_ip='192.168.0.1' + - default_ip_range='192.168.0.0/16' + - vlan_ip_range='{{ vlan_ip_range }}' + - lo_v6_prefix='{{ lo_v6_prefix }}' + - arista_vms=\"['{{ vm_hosts | list | join("','") }}']\" + - preboot_files='{{ preboot_files }}' + - preboot_oper='{{ item }}' + - allow_vlan_flooding='{{ allow_vlan_flooding }}' + + always: + + - name: Set all the filename vars when there is no preboot type + set_fact: + reboot_log: '/tmp/{{reboot_type}}.log' + capture_pcap: '/tmp/capture.pcap' + filter_pcap: '/tmp/capture_filtered.pcap' + syslog_file: '/tmp/syslog' + sairedis_rec: '/tmp/sairedis.rec' + swss_rec: '/tmp/swss.rec' + when: not item or item == 'None' + + - name: Set all the filename vars when there is a preboot type + set_fact: + reboot_log: '/tmp/{{reboot_type}}-{{item}}.log' + capture_pcap: '/tmp/capture_{{item}}.pcap' + filter_pcap: '/tmp/capture_filtered_{{item}}.pcap' + syslog_file: '/tmp/syslog_{{item}}' + sairedis_rec: '/tmp/sairedis.rec.{{item}}' + swss_rec: '/tmp/swss.rec.{{item}}' + when: item and item != 'None' + + - name: Copy test results from ptf to the local box /tmp/*-reboot*.log + fetch: src="{{ reboot_log }}" dest='/tmp/' flat=true fail_on_missing=false + delegate_to: "{{ ptf_host }}" + + - name: Copy pcap files from ptf to the local box /tmp/ + fetch: src={{ item }} dest='/tmp/' flat=true fail_on_missing=false + delegate_to: "{{ ptf_host }}" + with_items: + - "{{ capture_pcap }}" + - "{{ filter_pcap }}" + + - name: Extract all syslog entries since the last reboot + extract_log: + directory: '/var/log' + file_prefix: 'syslog' + start_string: 'Linux version' + target_filename: "{{ syslog_file }}" + become: yes + + - name: Copy the exctracted syslog entries to the local machine + fetch: + src: "{{ syslog_file }}" + dest: '/tmp/' + flat: yes + + - name: Extract all sairedis.rec entries since the last reboot + extract_log: + directory: '/var/log/swss' + file_prefix: 'sairedis.rec' + start_string: 'recording on:' + target_filename: "{{ sairedis_rec }}" + + - name: Copy the exctracted sairedis.rec entries to the local machine + fetch: + src: "{{ sairedis_rec }}" + dest: '/tmp/' + flat: yes + + - name: Extract all swss.rec entries since the last reboot + extract_log: + directory: '/var/log/swss' + file_prefix: 'swss.rec' + start_string: 'recording started' + target_filename: "{{ swss_rec }}" + + - name: Copy the exctracted swss.rec entries to the local machine + fetch: + src: "{{ swss_rec }}" + dest: '/tmp/' + flat: yes + + +- name: Wait for the DUT to be ready for the next test + pause: seconds=420 + when: preboot_list|length > 1 diff --git a/ansible/roles/test/tasks/run_command_with_log_analyzer.yml b/ansible/roles/test/tasks/run_command_with_log_analyzer.yml index f163ef27832..6f04d1e16d5 100644 --- a/ansible/roles/test/tasks/run_command_with_log_analyzer.yml +++ b/ansible/roles/test/tasks/run_command_with_log_analyzer.yml @@ -5,7 +5,17 @@ # 3) Run cleanup if needed. #----------------------------------------- -- include_vars: "vars/run_config_test_vars.yml" +- name: Initialize some variables for loganalyzer + set_fact: + testname_unique: "{{ testname }}.{{ lookup('pipe','date +%Y-%m-%d-%H:%M:%S') }}" +- set_fact: + test_out_dir: "{{ out_dir }}/{{ testname_unique }}" + loganalyzer_init: roles/test/files/tools/loganalyzer/loganalyzer_init.yml + loganalyzer_analyze: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml + match_file: loganalyzer_common_match.txt + ignore_file: loganalyzer_common_ignore.txt + summary_file: summary.loganalysis.{{ testname_unique }}.log + result_file: result.loganalysis.{{ testname_unique }}.log - block: - name: Initialize loganalizer. Put start marker to log file. @@ -31,21 +41,24 @@ register: expects_found when: errors_expected == true - - name: Check that expected error messages are found (negative tests only). - fail: msg="Expected error messages are not found while running {{ testname }} / {{ command_to_run }}" - when: errors_expected == true and expects_found.stdout == "0" - - name: Get the total number of error messages. shell: grep "TOTAL MATCHES" "{{ test_out_dir }}/{{ summary_file }}" | sed -n "s/TOTAL MATCHES:[[:space:]]*//p" register: errors_found - - name: Check the number of error messages (positive tests only). - fail: msg="{{ errors_found.stdout }} errors found while running {{ testname }} / {{ command_to_run }}." - when: errors_expected == false and errors_found.stdout != "0" - - name: Copy test data to host. - fetch: src={{ test_out_dir }}/{{ item }} dest=failed-test-data/{{ testname_unique }}/{{ item }} + fetch: + src: "{{ test_out_dir }}/{{ item }}" + dest: "test/{{ inventory_hostname }}/{{ item | basename }}" + flat: yes with_items: - "{{ summary_file }}" - "{{ result_file }}" when: (errors_expected == true and expects_found.stdout == "0") or (errors_expected == false and errors_found.stdout != "0") + + - name: Check that expected error messages are found (negative tests only). + fail: msg="Expected error messages are not found while running {{ testname }} / {{ command_to_run }}" + when: errors_expected == true and expects_found.stdout == "0" + + - name: Check the number of error messages (positive tests only). + fail: msg="{{ errors_found.stdout }} errors found while running {{ testname }} / {{ command_to_run }}." + when: errors_expected == false and errors_found.stdout != "0" diff --git a/ansible/roles/test/tasks/run_loganalyzer.yml b/ansible/roles/test/tasks/run_loganalyzer.yml index 73f36c582dc..125c058b9a7 100644 --- a/ansible/roles/test/tasks/run_loganalyzer.yml +++ b/ansible/roles/test/tasks/run_loganalyzer.yml @@ -3,7 +3,17 @@ # or analyze-phase of loganalyzer. #----------------------------------------- -- include_vars: "vars/run_loganalyzer_vars.yml" +- name: Initialize some variables for loganalyzer + set_fact: + testname_unique: "{{ testname }}.{{ lookup('pipe','date +%Y-%m-%d-%H:%M:%S') }}" +- set_fact: + test_out_dir: "{{ out_dir }}/{{ testname_unique }}" + loganalyzer_init: roles/test/files/tools/loganalyzer/loganalyzer_init.yml + loganalyzer_analyze: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml + match_file: loganalyzer_common_match.txt + ignore_file: loganalyzer_common_ignore.txt + summary_file: summary.loganalysis.{{ testname_unique }}.log + result_file: result.loganalysis.{{ testname_unique }}.log - name: Initialize loganalizer. Put start marker to log file. include: "{{ loganalyzer_init }}" diff --git a/ansible/roles/test/tasks/shared-fib.yml b/ansible/roles/test/tasks/shared-fib.yml index 73aee0d19bb..87557248ea5 100644 --- a/ansible/roles/test/tasks/shared-fib.yml +++ b/ansible/roles/test/tasks/shared-fib.yml @@ -8,7 +8,7 @@ when: testbed_type is not defined - fail: msg="testbed_type {{testbed_type}} is invalid." - when: testbed_type not in ['t1-lag', 't1', 't1-64-lag', 't0', 't0-64', 't0-116'] + when: testbed_type not in testcases['fib']['topologies'] - include_vars: "vars/topo_{{testbed_type}}.yml" @@ -18,7 +18,7 @@ - name: Expand properties into props set_fact: props="{{configuration_properties['common']}}" - when: testbed_type in ['t0', 't0-64', 't0-116'] + when: testbed_type in ['t0', 't0-56', 't0-64', 't0-116'] - name: Expand ToR properties into props set_fact: props_tor="{{configuration_properties['tor']}}" diff --git a/ansible/roles/test/tasks/simple-fib.yml b/ansible/roles/test/tasks/simple-fib.yml index e21b16a71fc..889538da3c3 100644 --- a/ansible/roles/test/tasks/simple-fib.yml +++ b/ansible/roles/test/tasks/simple-fib.yml @@ -6,6 +6,9 @@ - debug : msg="Start FIB Test" +- set_fact: mtu=9114 + when: mtu is not defined + - name: "Start PTF runner" include: ptf_runner.yml vars: @@ -20,4 +23,5 @@ - fib_info='/root/fib_info.txt' - ipv4={{ipv4}} - ipv6={{ipv6}} - ptf_extra_options: "--relax --debug info --log-file /tmp/fib_test.FibTest.ipv4.{{ipv4}}.ipv6.{{ipv6}}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + - testbed_mtu={{mtu}} + ptf_extra_options: "--relax --debug info --log-file /tmp/fib_test.FibTest.ipv4.{{ipv4}}.ipv6.{{ipv6}}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log --socket-recv-size 16384" diff --git a/ansible/roles/test/tasks/single_lag_lacp_rate_test.yml b/ansible/roles/test/tasks/single_lag_lacp_rate_test.yml index f6c27937d81..7785437dc67 100644 --- a/ansible/roles/test/tasks/single_lag_lacp_rate_test.yml +++ b/ansible/roles/test/tasks/single_lag_lacp_rate_test.yml @@ -47,36 +47,53 @@ neighbor_lag_intfs: "{{ neighbor_lag_intfs }} + [ '{{ vm_neighbors[item]['port'] }}' ]" with_items: "{{ po_interfaces }}" -# make sure portchannel peer rate is set to fast -- name: make sure all lag members on VM are set to fast - action: apswitch template=neighbor_lag_rate_fast.j2 - args: - host: "{{peer_host}}" - login: "{{switch_login[hwsku_map[peer_hwsku]]}}" - connection: switch - -- pause: - seconds: 5 - -- name: test lacp packet sending rate is 1 seconds - include: lag_lacp_timing_test.yml - vars: - vm_name: "{{ peer_device }}" - lacp_timer: 1 - -# make sure portchannel peer rate is set to slow -- name: make sure all lag members on VM are set to slow - action: apswitch template=neighbor_lag_rate_slow.j2 - args: - host: "{{peer_host}}" - login: "{{switch_login[hwsku_map[peer_hwsku]]}}" - connection: switch - -- pause: - seconds: 5 - -- name: test lacp packet sending rate is 30 seconds - include: lag_lacp_timing_test.yml - vars: - vm_name: "{{ peer_device }}" - lacp_timer: 30 +- block: + # make sure portchannel peer rate is set to fast + - name: make sure all lag members on VM are set to fast + action: apswitch template=neighbor_lag_rate_fast.j2 + args: + host: "{{peer_host}}" + login: "{{switch_login[hwsku_map[peer_hwsku]]}}" + connection: switch + + - set_fact: + lag_rate_current_setting: "fast" + + - pause: + seconds: 5 + + - name: test lacp packet sending rate is 1 seconds + include: lag_lacp_timing_test.yml + vars: + vm_name: "{{ peer_device }}" + lacp_timer: 1 + + # make sure portchannel peer rate is set to slow + - name: make sure all lag members on VM are set to slow + action: apswitch template=neighbor_lag_rate_slow.j2 + args: + host: "{{peer_host}}" + login: "{{switch_login[hwsku_map[peer_hwsku]]}}" + connection: switch + + - set_fact: + lag_rate_current_setting: "slow" + + - pause: + seconds: 5 + + - name: test lacp packet sending rate is 30 seconds + include: lag_lacp_timing_test.yml + vars: + vm_name: "{{ peer_device }}" + lacp_timer: 30 + + always: + - name: Restore lag rate setting on VM in case of failure + action: apswitch template=neighbor_lag_rate_slow.j2 + args: + host: "{{peer_host}}" + login: "{{switch_login[hwsku_map[peer_hwsku]]}}" + timeout: 300 + connection: switch + when: "lag_rate_current_setting is defined and lag_rate_current_setting == 'fast'" diff --git a/ansible/roles/test/tasks/single_lag_test.yml b/ansible/roles/test/tasks/single_lag_test.yml index de8e558abee..852eb5734cb 100644 --- a/ansible/roles/test/tasks/single_lag_test.yml +++ b/ansible/roles/test/tasks/single_lag_test.yml @@ -31,9 +31,9 @@ - name: test fanout interface (physical) flap and lacp keep correct po status follow minimum links requirement include: lag_minlink.yml vars: - wait_down_time: 20 + wait_down_time: 35 -### Now figure out remote VM and interface info for the falpping lag member and run minlink test +### Now figure out remote VM and interface info for the flapping lag member and run minlink test - set_fact: peer_device: "{{vm_neighbors[flap_intf]['name']}}" neighbor_interface: "{{vm_neighbors[flap_intf]['port']}}" diff --git a/ansible/roles/test/tasks/snmp/pfc_counters.yml b/ansible/roles/test/tasks/snmp/pfc_counters.yml index 1a2cb0a47ef..1170474a822 100644 --- a/ansible/roles/test/tasks/snmp/pfc_counters.yml +++ b/ansible/roles/test/tasks/snmp/pfc_counters.yml @@ -7,5 +7,9 @@ # Ignore management ports, assuming the names starting with 'eth', eg. eth0 - fail: msg: "Port {{ item.key }} does not have PFC counters" - when: (not item.value.name.startswith("eth")) and (not item.value.cpfcIfRequests or not item.value.cpfcIfIndications or not item.value.requestsPerPriority or not item.value.indicationsPerPriority) + when: "{{ (not item.value.name.startswith('eth')) and + ('cpfcIfRequests' not in item.value.keys() or + 'cpfcIfIndications' not in item.value.keys() or + 'requestsPerPriority' not in item.value.keys() or + 'indicationsPerPriority' not in item.value.keys()) }}" with_dict: "{{ snmp_interfaces }}" diff --git a/ansible/roles/test/tasks/warm-reboot-fib.yml b/ansible/roles/test/tasks/warm-reboot-fib.yml index 3f1a2c5ac15..8bb93dec474 100644 --- a/ansible/roles/test/tasks/warm-reboot-fib.yml +++ b/ansible/roles/test/tasks/warm-reboot-fib.yml @@ -19,7 +19,8 @@ - fib_info='/root/fib_info.txt' - ipv4={{ipv4}} - ipv6={{ipv6}} - ptf_extra_options: "--relax --debug info --log-file /tmp/fib_test.FibTest.ipv4.{{ipv4}}.ipv6.{{ipv6}}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + - testbed_mtu={{ mtu|default(9114) }} + ptf_extra_options: "--relax --debug info --log-file /tmp/fib_test.FibTest.ipv4.{{ipv4}}.ipv6.{{ipv6}}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log --socket-recv-size 16384" - name: "Show ptf params PTF Test - {{ ptf_test_name }}" debug: msg="ptf --test-dir {{ ptf_test_dir }} {{ ptf_test_path }} {% if ptf_qlen is defined %} --qlen={{ ptf_qlen }} {% endif %} {% if ptf_platform_dir is defined %} --platform-dir {{ ptf_platform_dir }} {% endif %} --platform {{ ptf_platform }} {% if ptf_test_params is defined %} -t \"{{ ptf_test_params | default([]) | join(';') }}\" {% endif %} {{ ptf_extra_options | default(\"\")}} --disable-vxlan --disable-geneve --disable-erspan --disable-mpls --disable-nvgre 2>&1" diff --git a/ansible/roles/test/tasks/warm-reboot-sad.yml b/ansible/roles/test/tasks/warm-reboot-sad.yml new file mode 100644 index 00000000000..6ace17b74e0 --- /dev/null +++ b/ansible/roles/test/tasks/warm-reboot-sad.yml @@ -0,0 +1,11 @@ +- name: set default reboot_limit in seconds + set_fact: + reboot_limit: 1 + when: reboot_limit is not defined + +- name: Warm-reboot test + include: advanced-reboot.yml + vars: + reboot_type: warm-reboot + preboot_list: ['neigh_bgp_down', 'dut_bgp_down', 'dut_lag_down', 'neigh_lag_down'] + preboot_files: "peer_dev_info,neigh_port_info" diff --git a/ansible/roles/test/tasks/warm-reboot.yml b/ansible/roles/test/tasks/warm-reboot.yml index 0f457cd260b..7959e2fa18a 100644 --- a/ansible/roles/test/tasks/warm-reboot.yml +++ b/ansible/roles/test/tasks/warm-reboot.yml @@ -1,6 +1,6 @@ - name: set default reboot_limit in seconds set_fact: - reboot_limit: 0 + reboot_limit: 1 when: reboot_limit is not defined - name: Warm-reboot test diff --git a/ansible/roles/test/templates/pfc_wd_config.j2 b/ansible/roles/test/templates/pfc_wd_config.j2 index 3dc440256de..fb6ccd713f6 100644 --- a/ansible/roles/test/templates/pfc_wd_config.j2 +++ b/ansible/roles/test/templates/pfc_wd_config.j2 @@ -1,5 +1,5 @@ { - "PFC_WD_TABLE": { + "PFC_WD": { "{{ pfc_wd_interface_list }}": { "action": "{{ pfc_wd_action }}", "detection_time": "{{ pfc_wd_detection_time }}", diff --git a/ansible/roles/test/vars/testcases.yml b/ansible/roles/test/vars/testcases.yml index e5d705ceccf..4e470062860 100644 --- a/ansible/roles/test/vars/testcases.yml +++ b/ansible/roles/test/vars/testcases.yml @@ -26,18 +26,18 @@ testcases: bgp_speaker: filename: bgp_speaker.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116] required_vars: ptf_host: testbed_type: config: filename: config.yml - topologies: [t1-lag, t1-64-lag, t0, t0-64] + topologies: [t1-lag, t1-64-lag, t0, t0-64, t0-116] continuous_reboot: filename: continuous_reboot.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag] copp: filename: copp.yml @@ -47,7 +47,7 @@ testcases: decap: filename: decap.yml - topologies: [t1, t1-lag, t1-64-lag, t0, t0-64, t0-116] + topologies: [t1, t1-lag, t1-64-lag, t0, t0-56, t0-64, t0-116] required_vars: ptf_host: testbed_type: @@ -55,7 +55,7 @@ testcases: dhcp_relay: filename: dhcp_relay.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116] required_vars: ptf_host: @@ -72,7 +72,7 @@ testcases: fast-reboot: filename: fast-reboot.yml - topologies: [t0, t0-64, t0-64-32, t0-116] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116] required_vars: ptf_host: vm_hosts: @@ -84,9 +84,16 @@ testcases: ptf_host: vm_hosts: + warm-reboot-sad: + filename: warm-reboot-sad.yml + topologies: [t0, t0-64, t0-64-32, t0-116] + required_vars: + ptf_host: + vm_hosts: + fib: filename: simple-fib.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag] required_vars: ptf_host: testbed_type: @@ -100,36 +107,44 @@ testcases: fdb: filename: fdb.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116] + required_vars: + ptf_host: + testbed_type: + + fdb_mac_expire: + filename: fdb_mac_expire.yml + topologies: [t0, t0-64, t0-64-32, t0-116] required_vars: + fdb_aging_time: ptf_host: testbed_type: dir_bcast: filename: dir_bcast.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116] required_vars: ptf_host: testbed_type: lag_2: filename: lag_2.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1-lag, t1-64-lag] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1-lag, t1-64-lag] required_vars: ptf_host: testbed_type: lldp: filename: lldp.yml - topologies: [t0, t0-16, t0-64, t0-116, t0-64-32, t1, t1-lag, t1-64-lag] + topologies: [t0, t0-16, t0-56, t0-64, t0-116, t0-64-32, t1, t1-lag, t1-64-lag] link_flap: filename: link_flap.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] mem_check: filename: mem_check.yml - topologies: [t0, t0-16, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-16, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] mtu: filename: mtu.yml @@ -150,11 +165,11 @@ testcases: ntp: filename: ntp.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] pfc_wd: filename: pfc_wd.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] port_toggle: filename: port_toggle.yml @@ -166,7 +181,7 @@ testcases: reboot: filename: reboot.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] repeat_harness: filename: repeat_harness.yml @@ -186,19 +201,19 @@ testcases: sensors: filename: sensors_check.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] service_acl: filename: service_acl.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] snmp: filename: snmp.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] syslog: filename: syslog.yml - topologies: [t0, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] + topologies: [t0, t0-56, t0-64, t0-64-32, t0-116, t1, t1-lag, t1-64-lag, ptf32, ptf64] vlan: filename: vlantb.yml diff --git a/ansible/roles/vm_set/tasks/add_topo.yml b/ansible/roles/vm_set/tasks/add_topo.yml index 292ca5cff4b..d80577c8999 100644 --- a/ansible/roles/vm_set/tasks/add_topo.yml +++ b/ansible/roles/vm_set/tasks/add_topo.yml @@ -43,3 +43,7 @@ fp_mtu: "{{ fp_mtu_size }}" max_fp_num: "{{ max_fp_num }}" become: yes + +- name: Send arp ping packet to gw for flusing the ARP table + command: docker exec -i ptf_{{ vm_set_name }} python -c "from scapy.all import *; arping('{{ mgmt_gw }}')" + become: yes diff --git a/ansible/roles/vm_set/tasks/docker.yml b/ansible/roles/vm_set/tasks/docker.yml index bec6ba2bd6d..c6936bf2882 100644 --- a/ansible/roles/vm_set/tasks/docker.yml +++ b/ansible/roles/vm_set/tasks/docker.yml @@ -20,9 +20,7 @@ when: host_distribution_version.stdout == "18.04" - name: Add docker official GPG key - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present + shell: "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -" become: yes environment: "{{ proxy_env | default({}) }}" diff --git a/ansible/roles/vm_set/templates/show_int_portchannel_status.j2 b/ansible/roles/vm_set/templates/show_int_portchannel_status.j2 new file mode 100644 index 00000000000..4a8f3b3bae3 --- /dev/null +++ b/ansible/roles/vm_set/templates/show_int_portchannel_status.j2 @@ -0,0 +1 @@ +show interfaces Port-Channel 1-$ status \ No newline at end of file diff --git a/ansible/swap_syncd.yml b/ansible/swap_syncd.yml index f7e1f3c215c..d460204c952 100644 --- a/ansible/swap_syncd.yml +++ b/ansible/swap_syncd.yml @@ -58,8 +58,10 @@ set_fact: sonic_image_version: "{{ result.stdout }}" + # Note: no_log requires passlib python library - name: Pull syncd-rpc docker from registry shell: docker login -u {{docker_registry_username}} -p {{docker_registry_password}} {{docker_registry_host}}; docker pull {{docker_registry_host}}/{{docker_rpc_image_name}}:{{sonic_image_version}} + no_log: true - name: Tag pulled images as syncd shell: docker tag {{docker_registry_host}}/{{docker_rpc_image_name}}:{{sonic_image_version}} {{docker_syncd_name}} diff --git a/ansible/testbed_vm_status.yml b/ansible/testbed_vm_status.yml new file mode 100755 index 00000000000..d809c6ac3ae --- /dev/null +++ b/ansible/testbed_vm_status.yml @@ -0,0 +1,7 @@ +- hosts: servers:&vm_host + tasks: + - name: Get VM statuses from Testbed server + shell: virsh list + register: virsh_list + - name: Show VM statuses + debug: msg="{{ virsh_list['stdout_lines'] }}" \ No newline at end of file diff --git a/ansible/vars/run_config_test_vars.yml b/ansible/vars/run_config_test_vars.yml deleted file mode 100644 index 0284d436790..00000000000 --- a/ansible/vars/run_config_test_vars.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ ansible_date_time.time }}" - -test_out_dir: "{{ out_dir }}/{{ testname_unique }}" -loganalyzer_init: roles/test/files/tools/loganalyzer/loganalyzer_init.yml -loganalyzer_analyze: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - -match_file: loganalyzer_common_match.txt -ignore_file: loganalyzer_common_ignore.txt - -summary_file: summary.loganalysis.{{ testname_unique }}.log -result_file: result.loganalysis.{{ testname_unique }}.log diff --git a/ansible/vars/run_loganalyzer_vars.yml b/ansible/vars/run_loganalyzer_vars.yml deleted file mode 100644 index c7d2bf095b2..00000000000 --- a/ansible/vars/run_loganalyzer_vars.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ ansible_date_time.time }}" - -test_out_dir: "{{ out_dir }}/{{ testname_unique }}" -loganalyzer_init: roles/test/files/tools/loganalyzer/loganalyzer_init.yml -loganalyzer_analyze: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - -match_file: loganalyzer_common_match.txt -ignore_file: loganalyzer_common_ignore.txt - -summary_file: summary.loganalysis.{{ testname_unique }}.log -result_file: result.loganalysis.{{ testname_unique }}.log - -run_analyze_and_check: "roles/test/tasks/run_analyze_and_check.yml" diff --git a/ansible/vars/run_ping_test_vars.yml b/ansible/vars/run_ping_test_vars.yml deleted file mode 100644 index 40dafcb8a7b..00000000000 --- a/ansible/vars/run_ping_test_vars.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ ansible_date_time.time }}" - -test_out_dir: "{{ out_dir }}/{{ testname_unique }}" -summary_file: "summary.loganalysis.{{ testname_unique }}.log" -result_file: "result.loganalysis.{{ testname_unique }}.log" diff --git a/ansible/vars/run_ptf_test_vars.yml b/ansible/vars/run_ptf_test_vars.yml deleted file mode 100644 index 40dafcb8a7b..00000000000 --- a/ansible/vars/run_ptf_test_vars.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ ansible_date_time.time }}" - -test_out_dir: "{{ out_dir }}/{{ testname_unique }}" -summary_file: "summary.loganalysis.{{ testname_unique }}.log" -result_file: "result.loganalysis.{{ testname_unique }}.log"