Improve the pytest port_toggle library

tests/common/port_toggle.py

@@ -2,14 +2,12 @@
 Tool used for shutdown/startup port on the DUT.
 """
 
+import datetime
 import time
 import logging
 import pprint
 
 from tests.common.helpers.assertions import pytest_assert
-from tests.platform_tests.link_flap.link_flap_utils import watch_system_status
-from tests.common.utilities import wait_until
-
 
 logger = logging.getLogger(__name__)
 
@@ -26,19 +24,16 @@ def port_toggle(duthost, tbinfo, ports=None, wait=60, wait_after_ports_up=60, wa
         watch: Logging system state
     """
 
-    def __check_interface_state(state='up'):
-        """
-        Check interfaces status
-
-        Args:
-            state: state of DUT's interface
+    def __get_down_ports():
+        """Check interface status and return the down ports in a set
         """
+        total_down_ports = set()
         ports_down = duthost.interface_facts(up_ports=ports)['ansible_facts']['ansible_interface_link_down_ports']
-
-        if 'down' in state:
-            return len(ports_down) == len(ports)
-        else:
-            return len(ports_down) == 0
+        db_ports_down = duthost.show_interface(command='status', up_ports=ports)['ansible_facts']\
+            ['ansible_interface_link_down_ports']
+        total_down_ports.update(ports_down)
+        total_down_ports.update(db_ports_down)
+        return total_down_ports
 
     if ports is None:
         logger.debug('ports is None, toggling all minigraph ports')
@@ -47,24 +42,71 @@ def __check_interface_state(state='up'):
 
     logger.info('toggling ports:\n%s', pprint.pformat(ports))
 
+    cmds_down = []
+    cmds_up = []
     for port in ports:
-        duthost.command('config interface shutdown {}'.format(port))
+        cmds_down.append('config interface shutdown {}'.format(port))
+        cmds_up.append('config interface startup {}'.format(port))
+
+    shutdown_ok = False
+    shutdown_err_msg = ''
+    try:
+        duthost.shell_cmds(cmds=cmds_down)
         if watch:
             time.sleep(1)
-            watch_system_status(duthost)
-
-    # verify all interfaces are down
-    pytest_assert(wait_until(3, 1, __check_interface_state, 'down'),
-                  "dut ports {} didn't go down as expected"
-                  .format(list(set(ports).difference(set(duthost.interface_facts(up_ports=ports)['ansible_facts']['ansible_interface_link_down_ports'])))))
-
-    for port in ports:
-        duthost.command('config interface startup {}'.format(port))
-
-    logger.info('waiting for ports to become up')
 
-    pytest_assert(wait_until(wait, 1, __check_interface_state),
-                  "dut ports {} didn't go up as expected".format(duthost.interface_facts(up_ports=ports)['ansible_facts']['ansible_interface_link_down_ports']))
+            # Watch memory status
+            memory_output = duthost.shell("show system-memory")["stdout"]
+            logger.info("Memory Status: %s", memory_output)
+
+            # Watch orchagent CPU utilization
+            orch_cpu = duthost.shell("show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
+            logger.info("Orchagent CPU Util: %s", orch_cpu)
+
+            # Watch Redis Memory
+            redis_memory = duthost.shell("redis-cli info memory | grep used_memory_human")["stdout"]
+            logger.info("Redis Memory: %s", redis_memory)
+
+        logger.info('Wait for ports to become down.')
+        start_time = datetime.datetime.now()
+        while True:
+            down_ports = __get_down_ports()
+            if len(down_ports) == len(ports):
+                shutdown_ok = True
+                break
+            time.sleep(5)
+            if (datetime.datetime.now() - start_time).seconds > 20:
+                break
+
+        if not shutdown_ok:
+            shutdown_err_msg = 'Some ports did not go down as expected: {}'.format(str(set(ports) - set(down_ports)))
+    except Exception as e:
+        shutdown_err_msg = 'Shutdown ports failed with exception: {}'.format(repr(e))
+
+    startup_ok = False
+    startup_err_msg = ''
+    try:
+        duthost.shell_cmds(cmds=cmds_up)
+
+        logger.info('Wait for ports to become up.')
+        start_time = datetime.datetime.now()
+        while True:
+            down_ports = __get_down_ports()
+            if len(down_ports) == 0:
+                startup_ok = True
+                break
+            time.sleep(5)
+            if (datetime.datetime.now() - start_time).seconds > wait:
+                break
+
+        if not startup_ok:
+            startup_err_msg = 'Some ports did not go up as expected: {}'.format(str(down_ports))
+
+    except Exception as e:
+        startup_err_msg = 'Startup interfaces failed with exception: {}'.format(repr(e))
+
+    pytest_assert(shutdown_ok, shutdown_err_msg)
+    pytest_assert(startup_ok, startup_err_msg)
 
     logger.info('wait %d seconds for system to startup', wait_after_ports_up)
     time.sleep(wait_after_ports_up)