Workaround for "dead worker" after docker-sonic-mgmt upgrade (#21407)

After the docker-sonic-mgmt image is upgraded, some tests failed while creating `nbrhosts` fixture with below error:
```
self = <ansible.plugins.strategy.linear.StrategyModule object at 0x7596c07986e0>
iterator = <ansible.executor.play_iterator.PlayIterator object at 0x7596c09b2a80>

    def _wait_on_pending_results(self, iterator):
        '''
        Wait for the shared counter to drop to zero, using a short sleep
        between checks to ensure we don't spin lock
        '''

        ret_results = []

        display.debug("waiting for pending results...")
        while self._pending_results > 0 and not self._tqm._terminated:

            if self._tqm.has_dead_workers():
>               raise AnsibleError("A worker was found in a dead state")
E               ansible.errors.AnsibleError: A worker was found in a dead state
```

The new docker-sonic-mgmt image has ansible upgraded from 2.13 to 2.18.
While creating the `nbrhosts` fixture, thread pool is used to improve the performance for initializing large number of neighbors. For the t0-sonic topology, sonic VM is used as neighbor. The `nbrhosts` fixture needs to initialize multiple `SonicHost` objects. In __init__ of SonicHost, parallel thread is again being used to boost the execution of multiple commands on the device to gather various facts.
For new ansible 2.18, it is not able to handle the complicated scenario properly. Possibly the task queue manager is checking state of workers of other task queue manager created by other threads.

Because of this issue, PR testing easily fail. To stop bleeding and unblock PR testing, this change added a threading lock for initializing neighbor hosts.

Signed-off-by: Xin Wang <[email protected]>

Author: wangxin

tests/conftest.py

@@ -89,6 +89,7 @@
 
 logger = logging.getLogger(__name__)
 cache = FactsCache()
+_ansible_tqm_lock = threading.Lock()
 
 DUTHOSTS_FIXTURE_FAILED_RC = 15
 CUSTOM_MSG_PREFIX = "sonic_custom_msg"
@@ -858,49 +859,50 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request):
         return devices
 
     def initial_neighbor(neighbor_name, vm_name):
-        logger.info(f"nbrhosts started: {neighbor_name}_{vm_name}")
-        if neighbor_type == "eos":
-            device = NeighborDevice(
-                {
-                    'host': EosHost(
-                        ansible_adhoc,
-                        vm_name,
-                        creds['eos_login'],
-                        creds['eos_password'],
-                        shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None,
-                        shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None
-                    ),
-                    'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
-                }
-            )
-        elif neighbor_type == "sonic":
-            device = NeighborDevice(
-                {
-                    'host': SonicHost(
-                        ansible_adhoc,
-                        vm_name,
-                        ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None,
-                        ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None
-                    ),
-                    'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
-                }
-            )
-        elif neighbor_type == "cisco":
-            device = NeighborDevice(
-                {
-                    'host': CiscoHost(
-                        ansible_adhoc,
-                        vm_name,
-                        creds['cisco_login'],
-                        creds['cisco_password'],
-                    ),
-                    'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
-                }
-            )
-        else:
-            raise ValueError("Unknown neighbor type %s" % (neighbor_type,))
-        devices[neighbor_name] = device
-        logger.info(f"nbrhosts finished: {neighbor_name}_{vm_name}")
+        with _ansible_tqm_lock:
+            logger.info(f"nbrhosts started: {neighbor_name}_{vm_name}")
+            if neighbor_type == "eos":
+                device = NeighborDevice(
+                    {
+                        'host': EosHost(
+                            ansible_adhoc,
+                            vm_name,
+                            creds['eos_login'],
+                            creds['eos_password'],
+                            shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None,
+                            shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None
+                        ),
+                        'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
+                    }
+                )
+            elif neighbor_type == "sonic":
+                device = NeighborDevice(
+                    {
+                        'host': SonicHost(
+                            ansible_adhoc,
+                            vm_name,
+                            ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None,
+                            ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None
+                        ),
+                        'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
+                    }
+                )
+            elif neighbor_type == "cisco":
+                device = NeighborDevice(
+                    {
+                        'host': CiscoHost(
+                            ansible_adhoc,
+                            vm_name,
+                            creds['cisco_login'],
+                            creds['cisco_password'],
+                        ),
+                        'conf': tbinfo['topo']['properties']['configuration'][neighbor_name]
+                    }
+                )
+            else:
+                raise ValueError("Unknown neighbor type %s" % (neighbor_type,))
+            devices[neighbor_name] = device
+            logger.info(f"nbrhosts finished: {neighbor_name}_{vm_name}")
 
     executor = concurrent.futures.ThreadPoolExecutor(max_workers=8)
     futures = []