[202411] Support Ubuntu 24 server in KVM (sonic-net#18929)

What is the motivation for this PR?
When running with Ubuntu 24.04 host, many tasks from a sonic-mgmt container may fail.
One reason is that a newer version of Python being used in Ubuntu 24.04 that doesn't support some of the packages that ansible modules attempt to import. Such as imp module is removed in Python 3.11, need to use importlib, and
PR sonic-net#16039 and sonic-net#17883 did the fix and need to cherry-pick to 202411

How did you do it?
Merge the change of sonic-net#16039 and sonic-net#17883 to 202411

Author: xwjiang-ms

ansible/ansible.cfg

@@ -170,7 +170,7 @@ fact_caching_timeout = 86400
 
 [privilege_escalation]
 #become=True
-become_method='sudo'
+become_method=sudo
 #become_user='root'
 #become_ask_pass=False
 

ansible/devutil/conn_graph_helper.py

@@ -1,11 +1,32 @@
 import os
 import inspect
 import sys
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 
 CONN_GRAPH_LOG = "/tmp/conn_graph_debug.txt"
 
 
+def load_source(modname, filename):
+    if use_importlib:
+        loader = importlib.machinery.SourceFileLoader(modname, filename)
+        spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+        module = importlib.util.module_from_spec(spec)
+        # The module is always executed and not cached in sys.modules.
+        # Uncomment the following line to cache the module.
+        # sys.modules[module.__name__] = module
+        loader.exec_module(module)
+    else:
+        # For Python 2.x compatibility
+        module = imp.load_source(modname, filename)
+    return module
+
+
 def get_conn_graph_facts(hostnames):
     """
     @summary: Load conn_graph_facts from conn_graph_facts.xml
@@ -18,7 +39,7 @@ def get_conn_graph_facts(hostnames):
     if ansible_path not in sys.path:
         sys.path.append(ansible_path)
 
-    utils = imp.load_source('conn_graph_utils', os.path.join(
+    utils = load_source('conn_graph_utils', os.path.join(
         ansible_path, 'library/conn_graph_facts.py'))
     utils.LAB_GRAPHFILE_PATH = os.path.join(
         ansible_path, utils.LAB_GRAPHFILE_PATH)

ansible/health_checker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import logging
 import sys
@@ -13,6 +12,7 @@
 except ImportError:
     # ToDo: Support running without Ansible
     has_ansible = False
+from devutil.conn_graph_helper import load_source
 
 ANSIBLE_DIR = os.path.abspath(os.path.dirname(__file__))
 SONIC_MGMT_DIR = os.path.dirname(ANSIBLE_DIR)
@@ -35,7 +35,7 @@
 
 def get_testbeds_dict():
     """Return a dictionary containing mapping from dut hostname to testbed name."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds_dict = testbed.TestbedInfo(TESTBED_FILE).testbed_topo
     return testbeds_dict

ansible/plugins/connection/multi_passwd_ssh.py

@@ -1,5 +1,11 @@
 import hashlib
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 import logging
 import os
 
@@ -10,9 +16,24 @@
 
 logger = logging.getLogger(__name__)
 
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+
 # HACK: workaround to import the SSH connection plugin
 _ssh_mod = os.path.join(os.path.dirname(connection.__file__), "ssh.py")
-_ssh = imp.load_source("_ssh", _ssh_mod)
+if use_importlib:
+    _ssh = load_source("_ssh", _ssh_mod)
+else:
+    _ssh = imp.load_source("_ssh", _ssh_mod)
 
 # Use same options as the builtin Ansible SSH plugin
 DOCUMENTATION = _ssh.DOCUMENTATION

ansible/recover_server.py

@@ -10,15 +10,14 @@
 import argparse
 import collections
 import datetime
-import imp
 import logging
 import os
 import subprocess
 import sys
 import tempfile
 import threading
 import time
-
+from devutil.conn_graph_helper import load_source
 from tabulate import tabulate
 # Add tests path to syspath
 sys.path.append('../')
@@ -40,7 +39,7 @@
 
 def parse_testbed(testbedfile, testbed_servers):
     """Return a dictionary containing mapping from server name to testbeds."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds = {server_name: list() for server_name in testbed_servers}
     for tbname, tb in testbed.TestbedInfo(testbedfile).testbed_topo.items():
@@ -258,7 +257,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/restart_nightly_ptf.py

@@ -2,15 +2,14 @@
 
 import argparse
 import logging
-import imp
 import os
 import recover_server
 import sys
 import collections
 import datetime
 import time
 import tempfile
-
+from devutil.conn_graph_helper import load_source
 # Add tests path to syspath
 sys.path.append('../')
 
@@ -69,7 +68,7 @@ def __call__(self):
 
 def parse_testbed(testbedfile, servers):
     """Return a dictionary containing mapping from server name to nightly testbeds that need restart-ptf."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     all_testbeds = testbed.TestbedInfo(testbedfile).testbed_topo
     nightly_dir = os.path.join(SONIC_MGMT_DIR, ".azure-pipelines", "nightly")
@@ -117,7 +116,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/roles/vm_set/tasks/control_mux_simulator.yml

@@ -17,27 +17,43 @@
       werkzeug_version: "1.0.1"
       python_command: "python"
 
-  - name: Use newer Flask version for pip3
+  - name: Use newer Flask and Werkzeug version for pip3
     set_fact:
       flask_version: "2.3.3"
-      python_command: "python3"
-    when: pip_executable == "pip3"
-
-  - name: Use newer Werkzeug version for pip3
-    set_fact:
       werkzeug_version: "2.3.7"
       python_command: "python3"
     when: pip_executable == "pip3"
 
-  - name: Install flask
-    pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
-
-  - name: Install werkzeug
-    pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
+  - name: Run python3 in a virtualenv
+    set_fact:
+      python_command: "/tmp/sonic-mgmt-virtualenv/bin/python3"
+    when: host_distribution_version.stdout >= "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout < "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout >= "24.04"
 
   - name: Copy the mux simulator to test server
     copy:

ansible/roles/vm_set/tasks/docker.yml

@@ -1,8 +1,3 @@
-- name: Add docker official GPG key
-  apt_key: url=https://download.docker.com/linux/ubuntu/gpg state=present
-  become: yes
-  environment: "{{ proxy_env | default({}) }}"
-
 - name: Check docker repository
   find:
     paths: /etc/apt/sources.list.d/
@@ -54,6 +49,13 @@
   become: yes
   when: host_distribution_version.stdout == "22.04" and docker_repo.matched == 0
 
+- name: Add docker repository for 24.04
+  apt_repository:
+    repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable
+    state: present
+  become: yes
+  when: host_distribution_version.stdout == "24.04" and docker_repo.matched == 0
+
 # In ansible 2.8, there isn't update_cache_retries option in apt module, we can manually run update as a seperate and retryable step
 - name: Run the "apt-get update" as a separate and retryable step
   apt:
@@ -94,4 +96,23 @@
     pip: name=docker version=7.1.0 state=forcereinstall executable={{ pip_executable }}
     become: yes
     environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Update python3 packages
+  block:
+  - name: Ensure python3-venv is installed
+    apt:
+      name: python3-venv
+      state: present
+      update_cache: yes
+    become: yes
+  - name: Install python packages
+    pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+    become: yes
+    environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"