Test case 2 of PFC watchdog against warm-reboot: sad path

ansible/roles/test/files/tools/loganalyzer/loganalyzer.py

@@ -79,9 +79,10 @@ def init_sys_logger(self):
         return logger
     #---------------------------------------------------------------------
 
-    def __init__(self, run_id, verbose):
+    def __init__(self, run_id, verbose, start_marker = None):
         self.run_id = run_id
         self.verbose = verbose
+        self.start_marker = start_marker
     #---------------------------------------------------------------------
 
     def print_diagnostic_message(self, message):
@@ -92,7 +93,10 @@ def print_diagnostic_message(self, message):
     #---------------------------------------------------------------------
 
     def create_start_marker(self):
-        return self.start_marker_prefix + "-" + self.run_id
+        if (self.start_marker is None) or (len(self.start_marker) == 0):
+            return self.start_marker_prefix + "-" + self.run_id
+        else:
+            return self.start_marker
 
     #---------------------------------------------------------------------
 
@@ -576,6 +580,7 @@ def main(argv):
 
     action = None
     run_id = None
+    start_marker = None
     log_files_in = ""
     out_dir = None
     match_files_in = None
@@ -584,7 +589,7 @@ def main(argv):
     verbose = False
 
     try:
-        opts, args = getopt.getopt(argv, "a:r:l:o:m:i:e:vh", ["action=", "run_id=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
+        opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
 
     except getopt.GetoptError:
         print "Invalid option specified"
@@ -602,6 +607,9 @@ def main(argv):
         elif (opt in ("-r", "--run_id")):
             run_id = arg
 
+        elif (opt in ("-s", "--start_marker")):
+            start_marker = arg
+
         elif (opt in ("-l", "--logs")):
             log_files_in = arg
 
@@ -624,7 +632,7 @@ def main(argv):
         usage()
         sys.exit(err_invalid_input)
 
-    analyzer = AnsibleLogAnalyzer(run_id, verbose)
+    analyzer = AnsibleLogAnalyzer(run_id, verbose, start_marker)
 
     log_file_list = filter(None, log_files_in.split(tokenizer))
 

ansible/roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml

@@ -113,7 +113,7 @@
       extract_log:
         directory: '/var/log'
         file_prefix: 'syslog'
-        start_string: 'start-LogAnalyzer-{{ testname_unique }}'
+        start_string: "{% if start_marker is defined %}{{ start_marker }}{% else %}start-LogAnalyzer-{{ testname_unique }}{% endif %}"
         target_filename: "/tmp/syslog"
       become: yes
 
@@ -122,7 +122,7 @@
       shell: sed -i 's/^#//g' /etc/cron.d/logrotate
       become: yes
 
-- set_fact: cmd="python {{ run_dir }}/loganalyzer.py --action analyze --logs {{ tmp_log_file }} --run_id {{ testname_unique }} --out_dir {{ test_out_dir }} {{ match_file_option }} {{ ignore_file_option }} {{ expect_file_option }} -v"
+- set_fact: cmd="python {{ run_dir }}/loganalyzer.py --action analyze --logs {{ tmp_log_file }} --run_id {{ testname_unique }} {% if start_marker is defined %}--start_marker '{{ start_marker }}'{% endif %} --out_dir {{ test_out_dir }} {{ match_file_option }} {{ ignore_file_option }} {{ expect_file_option }} -v"
 
 - debug: msg={{cmd}}
 

ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml

@@ -8,7 +8,7 @@
     test_fetch_dir: test/{{ inventory_hostname }}
 
 # Output content of result files to ansible console
-- shell: cat {{ test_out_dir }}/*
+- shell: cat {{ test_out_dir }}/{{ result_file }} {{ test_out_dir }}/{{ summary_file }}
   register: out
 - debug: var=out.stdout_lines
 
@@ -20,8 +20,23 @@
   shell: grep "TOTAL EXPECTED MISSING MATCHES" "{{ test_out_dir }}/{{ summary_file }}" | sed -n "s/TOTAL EXPECTED MISSING MATCHES:[[:space:]]*//p"
   register: expected_missing_matches
 
+- debug: msg={{expected_missing_matches}}
+
+- name: Check if loganalyzer gets the exact number of expected messages
+  shell: grep "TOTAL EXPECTED MATCHES" "{{ test_out_dir }}/{{ summary_file }}" | sed -n "s/TOTAL EXPECTED MATCHES:[[:space:]]*//p"
+  register: expected_matches
+  when: expected_matches_target is defined and expected_matches_target != ""
+
+- debug: msg={{expected_matches}}
+  when: expected_matches_target is defined and expected_matches_target != ""
+
 - set_fact:
     fail_in_logs: "{{ errors_found.stdout  != \"0\" or expected_missing_matches.stdout != \"0\" }}"
+  when: expected_matches_target is not defined or expected_matches_target == ""
+
+- set_fact:
+    fail_in_logs: "{{ errors_found.stdout  != \"0\" or expected_missing_matches.stdout != \"0\" or expected_matches.stdout != expected_matches_target|string }}"
+  when: expected_matches_target is defined and expected_matches_target != ""
 
 - set_fact:
     dump_since: '1 hour ago'

ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml

@@ -8,8 +8,8 @@
   copy: src="{{ loganalyzer_location }}/loganalyzer.py" dest="{{ run_dir }}"
 
 - set_fact:
-    testname_unique: "{{ testname }}.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}"
-  when: testname_unique is not defined
+    testname_unique: "{{ testname }}.{{ ansible_date_time.date }}.{{ lookup('pipe', 'date +%H:%M:%S') }}"
+  when: testname_unique is not defined or (testname_unique_gen is defined and testname_unique_gen == true)
 
 - debug: msg="starting loganalyzer_init.py"
 - debug: msg="python {{ run_dir }}/loganalyzer.py --action init --run_id {{ testname_unique }}"

ansible/roles/test/tasks/pfc_wd.yml

@@ -105,10 +105,15 @@
 - set_fact:
     select_test_ports: "{{select_test_ports | default({}) | combine({item.key: item.value})}}"
   with_dict: "{{test_ports}}"
-  when: item.value.test_port_id | int % 15 == seed | int % 15
+  when: (item.value.test_port_id | int % 15) == (seed | int % 15)
 
 - debug: msg="{{select_test_ports}}"
 
+- name: Run default test if user has not specified warm reboot test
+  set_fact:
+    warm_reboot_test: false
+  when: warm_reboot_test is not defined
+
  #****************************************#
  #           Start tests                  #
  #****************************************#
@@ -124,41 +129,48 @@
         minigraph_vlan_interfaces: []
       when: minigraph_vlan_interfaces is undefined
 
-    - name: Test PFC WD configuration validation.
-      vars:
-        pfc_wd_template: roles/test/templates/pfc_wd_config.j2
-      include: roles/test/tasks/pfc_wd/config_test/config_test.yml
-
-    - name: Test PFC WD Functional tests.
-      include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
-      with_dict: "{{select_test_ports}}"
-
-    - name: Test PFC WD Timer accuracy.
-      include: roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml
-
-    - name: Test PFC WD extreme case when all ports have storm
-      include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml
-
-    - name: Set vlan members
-      set_fact:
-        vlan_members: "{{ minigraph_vlans[minigraph_vlan_interfaces[0]['attachto']]['members']}}"
-      when:
-        - pfc_asym is defined
-
-    - name: Enable asymmetric PFC on all server interfaces
-      command: config interface pfc asymmetric on {{ item }}
-      become: yes
-      with_items: "{{ vlan_members }}"
-      when:
-        - pfc_asym is defined
-        - testbed_type in ['t0']
-
-    - name: Test PFC WD Functional tests.
-      include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
-      with_dict: "{{select_test_ports}}"
-      when:
-        - pfc_asym is defined
-        - testbed_type in ['t0']
+    - block:
+        - name: Test PFC WD configuration validation.
+          vars:
+            pfc_wd_template: roles/test/templates/pfc_wd_config.j2
+          include: roles/test/tasks/pfc_wd/config_test/config_test.yml
+
+        - name: Test PFC WD Functional tests.
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
+          with_dict: "{{select_test_ports}}"
+
+        - name: Test PFC WD Timer accuracy.
+          include: roles/test/tasks/pfc_wd/functional_test/check_timer_accuracy_test.yml
+
+        - name: Test PFC WD extreme case when all ports have storm
+          include: roles/test/tasks/pfc_wd/functional_test/storm_all_test.yml
+
+        - name: Set vlan members
+          set_fact:
+            vlan_members: "{{ minigraph_vlans[minigraph_vlan_interfaces[0]['attachto']]['members']}}"
+          when:
+            - pfc_asym is defined
+
+        - name: Enable asymmetric PFC on all server interfaces
+          command: config interface pfc asymmetric on {{ item }}
+          become: yes
+          with_items: "{{ vlan_members }}"
+          when:
+            - pfc_asym is defined
+            - testbed_type in ['t0']
+
+        - name: Test PFC WD Functional tests.
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test.yml
+          with_dict: "{{select_test_ports}}"
+          when:
+            - pfc_asym is defined
+            - testbed_type in ['t0']
+      when: warm_reboot_test | bool == false
+
+    - block:
+        - name: Test PFC WD function against warm reboot
+          include: roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml
+      when: warm_reboot_test | bool == true
 
   always:
     - name: General cleanup.

ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml

@@ -0,0 +1,78 @@
+#------------------------------------
+# Test the PFC restore action
+# Iterate the test over a list of lossless queues
+# 1. Verify restore is detected via syslog entry analysis on the dut
+# 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+#------------------------------------
+
+- name: Set fact for item values
+  set_fact:
+   peer_device: "{{item.value.peer_device}}"
+   pfc_wd_test_port: "{{item.key}}"
+   pfc_wd_rx_port: "{{item.value.rx_port}}"
+   pfc_wd_test_neighbor_addr: "{{item.value.test_neighbor_addr}}"
+   pfc_wd_rx_neighbor_addr: "{{item.value.rx_neighbor_addr}}"
+   pfc_wd_test_port_id: "{{item.value.test_port_id}}"
+   pfc_wd_rx_port_id: "{{item.value.rx_port_id}}"
+   port_type: "{{item.value.test_port_type}}"
+
+- name: Set fact for test port ids and include portchannel other members if exists
+  set_fact:
+    pfc_wd_test_port_ids: "[{{item.value.test_portchannel_members | join(' ')}}]"
+  when: port_type == "portchannel"
+
+- set_fact:
+    pfc_wd_test_port_ids: "[{{pfc_wd_test_port_id}}]"
+  when: port_type in ["vlan", "interface"]
+
+- name: Remove existing IPs from PTF host
+  script: roles/test/files/helpers/remove_ip.sh
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Set {{pfc_wd_test_neighbor_addr}} to eth{{pfc_wd_test_port_id}} on PTF host"
+  shell: ifconfig eth{{pfc_wd_test_port_id}} {{pfc_wd_test_neighbor_addr}}
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Update ARP entry on DUT"
+  shell: ping {{minigraph_vlan_interfaces[0]['addr']}} -c 10
+  delegate_to: "{{ptf_host}}"
+  when: port_type == "vlan"
+
+- name: "Update ARP entry on DUT"
+  shell: docker exec -i swss arping {{pfc_wd_test_neighbor_addr}} -c 5
+  when: port_type == "vlan"
+
+- conn_graph_facts: host={{ peer_device }}
+  connection: local
+  become: no
+
+- name: Prepare variables required for PFC test
+  set_fact:
+    pfc_queue_indices: [4]
+    pfc_frames_number: 100000000
+    pfc_wd_test_pkt_count: 100
+    pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}"
+    peer_hwsku: "{{device_info['HwSku']}}"
+    peer_mgmt: "{{device_info['mgmtip']}}"
+    testname: functional_test
+
+- name: Add queue index 3 to pfc_queue_indices when seed is an odd number
+  set_fact:
+    pfc_queue_indices: "{{pfc_queue_indices + [3]}}"
+  when: seed | int is odd
+
+- set_fact:
+      peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+
+- name: set pfc storm templates based on fanout platform sku
+  include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml
+
+
+- name: Test PFC restore function per queue
+  include: roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml
+  vars:
+    pfc_queue_index: "{{item}}"
+    class_enable: "{{(1).__lshift__(item)}}"
+  with_items: "{{pfc_queue_indices}}"

ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml

@@ -0,0 +1,81 @@
+#------------------------------------
+# Test the PFC restore action
+# 1. Verify restore is detected via syslog entry analysis on the dut
+# 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+#------------------------------------
+
+- block:
+    # 1. Verify restore is detected via syslog entry analysis on the dut
+    - set_fact:
+        test_expect_file: "expect_pfc_wd_restore"
+
+    - name: Initialize loganalyzer
+      include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml
+      vars:
+        testname_unique_gen: true
+
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch
+
+    - name: Allow enough time for the PFC storm restoration to flow into the syslog
+      pause:
+        seconds: 1
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml
+
+    # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+    - name: "Send packets via {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_rx_port_id[0]}}'
+          - port_dst='{{pfc_wd_test_port_ids}}'
+          - ip_dst='{{pfc_wd_test_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+    - name: "Send packets to {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_test_port_id}}'
+          - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]'
+          - ip_dst='{{pfc_wd_rx_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+  rescue:
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch