Test case 1 of PFC watchdog against warm-reboot: happy path

ansible/roles/test/tasks/pfc_wd.yml

@@ -105,10 +105,15 @@
 - set_fact:
     select_test_ports: "{{select_test_ports | default({}) | combine({item.key: item.value})}}"
   with_dict: "{{test_ports}}"
-  when: item.value.test_port_id | int % 15 == seed | int % 15
+  when: (item.value.test_port_id | int % 15) == (seed | int % 15)
 
 - debug: msg="{{select_test_ports}}"
 
+- name: Run default test if user has not specified warm reboot test
+  set_fact:
+    warm_reboot_test: false
+  when: warm_reboot_test is not defined
+
  #****************************************#
  #           Start tests                  #
  #****************************************#
@@ -124,41 +129,48 @@
         minigraph_vlan_interfaces: []
       when: minigraph_vlan_interfaces is undefined
 
-    - name: Test PFC WD configuration validation.
-      vars:
-        pfc_wd_template: roles/test/templates/pfc_wd_config.j2
-      include: roles/test/tasks/pfc_wd/config_test/config_test.yml
-
-    - name: Test PFC WD Functional tests.
-      include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
-      with_dict: "{{select_test_ports}}"
-
-    - name: Test PFC WD Timer accuracy.
-      include: roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml
-
-    - name: Test PFC WD extreme case when all ports have storm
-      include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml
-
-    - name: Set vlan members
-      set_fact:
-        vlan_members: "{{ minigraph_vlans[minigraph_vlan_interfaces[0]['attachto']]['members']}}"
-      when:
-        - pfc_asym is defined
-
-    - name: Enable asymmetric PFC on all server interfaces
-      command: config interface pfc asymmetric on {{ item }}
-      become: yes
-      with_items: "{{ vlan_members }}"
-      when:
-        - pfc_asym is defined
-        - testbed_type in ['t0']
-
-    - name: Test PFC WD Functional tests.
-      include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
-      with_dict: "{{select_test_ports}}"
-      when:
-        - pfc_asym is defined
-        - testbed_type in ['t0']
+    - block:
+        - name: Test PFC WD configuration validation.
+          vars:
+            pfc_wd_template: roles/test/templates/pfc_wd_config.j2
+          include: roles/test/tasks/pfc_wd/config_test/config_test.yml
+
+        - name: Test PFC WD Functional tests.
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
+          with_dict: "{{select_test_ports}}"
+
+        - name: Test PFC WD Timer accuracy.
+          include: roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml
+
+        - name: Test PFC WD extreme case when all ports have storm
+          include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml
+
+        - name: Set vlan members
+          set_fact:
+            vlan_members: "{{ minigraph_vlans[minigraph_vlan_interfaces[0]['attachto']]['members']}}"
+          when:
+            - pfc_asym is defined
+
+        - name: Enable asymmetric PFC on all server interfaces
+          command: config interface pfc asymmetric on {{ item }}
+          become: yes
+          with_items: "{{ vlan_members }}"
+          when:
+            - pfc_asym is defined
+            - testbed_type in ['t0']
+
+        - name: Test PFC WD Functional tests.
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
+          with_dict: "{{select_test_ports}}"
+          when:
+            - pfc_asym is defined
+            - testbed_type in ['t0']
+      when: warm_reboot_test | bool == false
+
+    - block:
+        - name: Test PFC WD function against warm reboot
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml
+      when: warm_reboot_test | bool == true
 
   always:
     - name: General cleanup.

ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml

@@ -0,0 +1,142 @@
+#------------------------------------
+# Test the PFC restore action
+# 1. Verify restore is detected via syslog entry analysis on the dut
+# 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+#------------------------------------
+
+- name: Set fact for item values
+  set_fact:
+   peer_device: "{{item.value.peer_device}}"
+   pfc_wd_test_port: "{{item.key}}"
+   pfc_wd_rx_port: "{{item.value.rx_port}}"
+   pfc_wd_test_neighbor_addr: "{{item.value.test_neighbor_addr}}"
+   pfc_wd_rx_neighbor_addr: "{{item.value.rx_neighbor_addr}}"
+   pfc_wd_test_port_id: "{{item.value.test_port_id}}"
+   pfc_wd_rx_port_id: "{{item.value.rx_port_id}}"
+   port_type: "{{item.value.test_port_type}}"
+
+- name: Set fact for test port ids and include portchannel other members if exists
+  set_fact:
+    pfc_wd_test_port_ids: "[{{item.value.test_portchannel_members | join(' ')}}]"
+  when: port_type == "portchannel"
+
+- set_fact:
+    pfc_wd_test_port_ids: "[{{pfc_wd_test_port_id}}]"
+  when: port_type in ["vlan", "interface"]
+
+- name: Remove existing IPs from PTF host
+  script: roles/test/files/helpers/remove_ip.sh
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Set {{pfc_wd_test_neighbor_addr}} to eth{{pfc_wd_test_port_id}} on PTF host"
+  shell: ifconfig eth{{pfc_wd_test_port_id}} {{pfc_wd_test_neighbor_addr}}
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Update ARP entry on DUT"
+  shell: ping {{minigraph_vlan_interfaces[0]['addr']}} -c 10
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Update ARP entry on DUT"
+  shell: docker exec -i swss arping {{pfc_wd_test_neighbor_addr}} -c 5
+  when: port_type == "vlan"
+
+- conn_graph_facts: host={{ peer_device }}
+  connection: local
+  become: no
+
+- name: Prepare variables required for PFC test
+  set_fact:
+    pfc_queue_index: 4
+    pfc_frames_number: 100000000
+    pfc_wd_test_pkt_count: 100
+    pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}"
+    peer_hwsku: "{{device_info['HwSku']}}"
+    peer_mgmt: "{{device_info['mgmtip']}}"
+    testname: functional_test
+
+- set_fact:
+    class_enable: "{{(1).__lshift__(pfc_queue_index)}}"
+
+- set_fact:
+      peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+
+- name: set pfc storm templates based on fanout platform sku
+  include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml
+
+
+- block:
+    # 1. Verify restore is detected via syslog entry analysis on the dut
+    - set_fact:
+        test_expect_file: "expect_pfc_wd_restore"
+
+    - name: Initialize loganalyzer
+      include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml
+
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch
+
+    - name: Wait for queue to recover from PFC storm
+      pause:
+        seconds: 1
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml
+
+    # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+    - name: "Send packets via {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_rx_port_id[0]}}'
+          - port_dst='{{pfc_wd_test_port_ids}}'
+          - ip_dst='{{pfc_wd_test_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+    - name: "Send packets to {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_test_port_id}}'
+          - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]'
+          - ip_dst='{{pfc_wd_rx_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+  rescue:
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch