Fix flakiness in pfcwd/test_pfcwd_cli.py (sonic-net#19969)

What is the motivation for this PR?
Recent fix: sonic-net#17411

The test was flaky before this fix (and continues to be so). When the test picks up an egress interface which happens to be a member of a LAG consisting of multiple members, only this member is stormed and some of the traffic successfully egresses out of the other LAG members leading to lesser drops than expected when PFCWD is triggered with DROP action. The proposed fix was to shut down all but one LAG members by reducing the number of min_links. But the same config on cEOS was missing therefore LAG doesn't come up after shutting down other LAG members.

This is being rectified in this change for cEOS neighbors.

How did you do it?
The proposed fix is to change the min_link setting for the involved port channel on the cEOS side as well.

How did you verify/test it?
Stressed this test 10 times on dualtor-120 and t0-116 with Arista 7260CX3 platform.

Author: vivekverma-arista

tests/pfcwd/test_pfcwd_cli.py

@@ -16,6 +16,7 @@
 from tests.common.utilities import wait_until
 from tests.common.cisco_data import is_cisco_device
 from tests.common import config_reload
+from tests.common.devices.eos import EosHost
 
 pytestmark = [
     pytest.mark.topology("t0", "t1")
@@ -306,10 +307,10 @@ def send_rx_ingress(self, action, verify):
 
 class TestPfcwdFunc(SetupPfcwdFunc):
     """ Test PFC function and supporting methods """
-    def __shutdown_lag_members(self, duthost, selected_port):
+    def __shutdown_lag_members(self, duthost, selected_port, tbinfo, nbrhosts):
 
         if self.ports[selected_port]['test_port_type'] != 'portchannel':
-            return
+            return None, None, None
 
         config_facts = duthost.config_facts(host=duthost.hostname, source="persistent")['ansible_facts']
         portChannels = config_facts['PORTCHANNEL_MEMBER']
@@ -321,6 +322,25 @@ def __shutdown_lag_members(self, duthost, selected_port):
                 portChannelMembers = portChannels[intf]
                 break
 
+        dst_mgfacts = duthost.get_extended_minigraph_facts(tbinfo)
+        vm_neighbors = dst_mgfacts['minigraph_neighbors']
+        peer_device = vm_neighbors[list(portChannelMembers.keys())[0]]['name']
+        peer_port = vm_neighbors[list(portChannelMembers.keys())[0]]['port']
+        vm_host = nbrhosts[peer_device]['host']
+        neigh_port_channel = None
+        min_links = None
+        if isinstance(vm_host, EosHost):
+            neigh_port_channels = vm_host.eos_command(
+                commands=['show port-channel | json'])['stdout'][0]["portChannels"]
+            for po_name, po_config in neigh_port_channels.items():
+                for member in po_config['activePorts']:
+                    if member == peer_port:
+                        neigh_port_channel = po_name
+                        min_links = len(po_config['activePorts'])
+                        break
+
+            vm_host.eos_config(lines=['port-channel min-links 1'], parents=[f'int {neigh_port_channel}'])
+
         cmd_data = f'.PORTCHANNEL.{portChannel}.min_links = "1"'
 
         for port in portChannelMembers:
@@ -334,13 +354,17 @@ def __shutdown_lag_members(self, duthost, selected_port):
         duthost.command(cmd, _uses_shell=True)
         duthost.command("sudo cp /tmp/config_db.json /etc/sonic/config_db.json", _uses_shell=True)
         config_reload(duthost, config_source='config_db', safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True)
+        return vm_host, neigh_port_channel, min_links
 
-    def __restore_original_config(self, duthost, selected_port):
+    def __restore_original_config(self, duthost, selected_port, vm_host, neigh_port_channel, min_links):
 
         if self.ports[selected_port]['test_port_type'] != 'portchannel':
             return
 
-        duthost.command("sudo mv /tmp/config_db_backup.json /etc/sonic/config.json", _uses_shell=True)
+        if isinstance(vm_host, EosHost):
+            vm_host.eos_config(lines=[f'port-channel min-links {min_links}'], parents=[f'int {neigh_port_channel}'])
+
+        duthost.command("sudo mv /tmp/config_db_backup.json /etc/sonic/config_db.json", _uses_shell=True)
         config_reload(duthost, config_source='config_db', safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True)
 
     def storm_detect_path(self, dut, port, action):
@@ -479,7 +503,7 @@ def set_traffic_action(self, duthost, action):
             self.rx_action = action
         self.tx_action = action
 
-    def test_pfcwd_show_stat(self, request, setup_pfc_test,
+    def test_pfcwd_show_stat(self, request, setup_pfc_test, tbinfo, nbrhosts,
                              setup_dut_test_params, enum_fanout_graph_facts, ptfhost,  # noqa: F811
                              duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts,
                              setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally,
@@ -525,7 +549,7 @@ def test_pfcwd_show_stat(self, request, setup_pfc_test,
         # for idx, port in enumerate(self.ports):
         port = list(self.ports.keys())[0]
 
-        self.__shutdown_lag_members(duthost, port)
+        vm_host, neigh_port_channel, min_links = self.__shutdown_lag_members(duthost, port, tbinfo, nbrhosts)
 
         logger.info("--- Testing various Pfcwd actions on {} ---".format(port))
         self.setup_test_params(port, setup_info['vlan'], init=True, ip_version=ip_version)
@@ -558,4 +582,4 @@ def test_pfcwd_show_stat(self, request, setup_pfc_test,
                     self.storm_hndle.stop_storm()
                 logger.info("--- Stop PFC WD ---")
                 self.dut.command("pfcwd stop")
-        self.__restore_original_config(duthost, port)
+                self.__restore_original_config(duthost, port, vm_host, neigh_port_channel, min_links)