Improve "Verify all interfaces are up" error handling

ansible/roles/test/tasks/check_testbed_interfaces.yml

@@ -0,0 +1,57 @@
+- block:
+  - name: Gathering lab graph facts about the device
+    conn_graph_facts: host={{ inventory_hostname }}
+    connection: local
+
+  - name: Fanout hostname
+    set_fact: fanout_switch={{ device_conn['Ethernet0']['peerdevice'] }}
+
+  - name: Check Fanout interfaces
+    local_action: shell ansible-playbook -i lab fanout.yml -l {{ fanout_switch }} --tags check_interfaces_status
+    ignore_errors: yes
+
+  when: 
+    - check_fanout is defined
+
+- block:
+  - name: Get Portchannel status 
+    shell: show interfaces portchannel
+    register: portchannel_status
+    ignore_errors: yes
+
+  - name: Get teamd dump
+    shell: teamdctl '{{ item }}' state dump
+    with_items: "{{ minigraph_portchannels }}"
+    ignore_errors: yes
+    when:
+      - minigraph_portchannels is defined
+
+  - name: Define testbed_name when not obtained
+    set_fact:
+      testbed_name: "{{ inventory_hostname + '-' + topo }}"
+    when: testbed_name is not defined
+
+  - name: Gathering testbed information
+    test_facts: testbed_name="{{ testbed_name }}"
+    connection: local
+    ignore_errors: yes
+
+  - name: Gather vm list from Testbed server
+    local_action: shell ansible-playbook testbed_vm_status.yml -i veos -l "{{ testbed_facts['server'] }}"
+    ignore_errors: yes
+
+  - set_fact:
+      vms: "{{ minigraph_devices }}"
+      peer_hwsku: 'Arista-VM'
+
+  - name: Gather Port-Channel status from VMs
+    action: apswitch template=roles/vm_set/templates/show_int_portchannel_status.j2
+    args:
+      host: "{{ vms[item]['mgmt_addr'] }}"
+      login: "{{ switch_login[hwsku_map[peer_hwsku]] }}"
+    connection: switch
+    ignore_errors: yes
+    when: vms["{{ item }}"]['hwsku'] == 'Arista-VM'
+    with_items: vms
+
+  when: check_vms is defined

ansible/roles/test/tasks/interface.yml

@@ -32,12 +32,26 @@
 - debug: msg="Found link down ports {{ansible_interface_link_down_ports}}"
   when: ansible_interface_link_down_ports | length > 0
 
-- name: Verify interfaces are up correctly
-  assert: { that: "{{ ansible_interface_link_down_ports | length }} == 0" }
+- block:
+  - name: Verify interfaces are up correctly
+    assert: { that: "{{ ansible_interface_link_down_ports | length }} == 0" }
+  rescue:
+  - include: check_testbed_interfaces.yml
+    vars:
+      check_fanout: true
+  - fail: msg="Not all Interfaces are up"
 
-- name: Verify port channel interfaces are up correctly
-  assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" }
-  with_items: "{{ minigraph_portchannels.keys() }}"
+- block:
+  - name: Verify port channel interfaces are up correctly
+    assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" }
+    with_items: "{{ minigraph_portchannels.keys() }}"
+
+  rescue:
+  - include: check_testbed_interfaces.yml
+    vars:
+      check_vms: true
+  - fail: msg="Not all PortChannels are up '{{ portchannel_status['stdout_lines'] }}' "
+    when: portchannel_status is defined
 
 - name: Verify VLAN interfaces are up correctly
   assert: { that: "'{{ ansible_interface_facts[item]['active'] }}' == 'True'" }

ansible/roles/vm_set/templates/show_int_portchannel_status.j2

@@ -0,0 +1 @@
+show interfaces Port-Channel 1-$ status

ansible/testbed_vm_status.yml

@@ -0,0 +1,9 @@
+# Example:
+# ansible-playbook testbed_vm_status.yml -i veos -l server_1
+- hosts: servers:&vm_host
+  tasks:
+  - name: Get VM statuses from Testbed server
+    shell: virsh list
+    register: virsh_list
+  - name: Show VM statuses
+    debug: msg="{{ virsh_list['stdout_lines'] }}"