[202411] Support Ubuntu 24 server in KVM

ansible/ansible.cfg

@@ -170,7 +170,7 @@ fact_caching_timeout = 86400
 
 [privilege_escalation]
 #become=True
-become_method='sudo'
+become_method=sudo
 #become_user='root'
 #become_ask_pass=False
 

ansible/devutil/conn_graph_helper.py

@@ -1,11 +1,32 @@
 import os
 import inspect
 import sys
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 
 CONN_GRAPH_LOG = "/tmp/conn_graph_debug.txt"
 
 
+def load_source(modname, filename):
+    if use_importlib:
+        loader = importlib.machinery.SourceFileLoader(modname, filename)
+        spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+        module = importlib.util.module_from_spec(spec)
+        # The module is always executed and not cached in sys.modules.
+        # Uncomment the following line to cache the module.
+        # sys.modules[module.__name__] = module
+        loader.exec_module(module)
+    else:
+        # For Python 2.x compatibility
+        module = imp.load_source(modname, filename)
+    return module
+
+
 def get_conn_graph_facts(hostnames):
     """
     @summary: Load conn_graph_facts from conn_graph_facts.xml
@@ -18,7 +39,7 @@ def get_conn_graph_facts(hostnames):
     if ansible_path not in sys.path:
         sys.path.append(ansible_path)
 
-    utils = imp.load_source('conn_graph_utils', os.path.join(
+    utils = load_source('conn_graph_utils', os.path.join(
         ansible_path, 'library/conn_graph_facts.py'))
     utils.LAB_GRAPHFILE_PATH = os.path.join(
         ansible_path, utils.LAB_GRAPHFILE_PATH)

ansible/health_checker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import logging
 import sys
@@ -13,6 +12,7 @@
 except ImportError:
     # ToDo: Support running without Ansible
     has_ansible = False
+from devutil.conn_graph_helper import load_source
 
 ANSIBLE_DIR = os.path.abspath(os.path.dirname(__file__))
 SONIC_MGMT_DIR = os.path.dirname(ANSIBLE_DIR)
@@ -35,7 +35,7 @@
 
 def get_testbeds_dict():
     """Return a dictionary containing mapping from dut hostname to testbed name."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds_dict = testbed.TestbedInfo(TESTBED_FILE).testbed_topo
     return testbeds_dict

ansible/plugins/connection/multi_passwd_ssh.py

@@ -1,5 +1,11 @@
 import hashlib
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 import logging
 import os
 
@@ -10,9 +16,24 @@
 
 logger = logging.getLogger(__name__)
 
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+
 # HACK: workaround to import the SSH connection plugin
 _ssh_mod = os.path.join(os.path.dirname(connection.__file__), "ssh.py")
-_ssh = imp.load_source("_ssh", _ssh_mod)
+if use_importlib:
+    _ssh = load_source("_ssh", _ssh_mod)
+else:
+    _ssh = imp.load_source("_ssh", _ssh_mod)
 
 # Use same options as the builtin Ansible SSH plugin
 DOCUMENTATION = _ssh.DOCUMENTATION

ansible/recover_server.py

@@ -10,15 +10,14 @@
 import argparse
 import collections
 import datetime
-import imp
 import logging
 import os
 import subprocess
 import sys
 import tempfile
 import threading
 import time
-
+from devutil.conn_graph_helper import load_source
 from tabulate import tabulate
 # Add tests path to syspath
 sys.path.append('../')
@@ -40,7 +39,7 @@
 
 def parse_testbed(testbedfile, testbed_servers):
     """Return a dictionary containing mapping from server name to testbeds."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds = {server_name: list() for server_name in testbed_servers}
     for tbname, tb in testbed.TestbedInfo(testbedfile).testbed_topo.items():
@@ -258,7 +257,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/restart_nightly_ptf.py

@@ -2,15 +2,14 @@
 
 import argparse
 import logging
-import imp
 import os
 import recover_server
 import sys
 import collections
 import datetime
 import time
 import tempfile
-
+from devutil.conn_graph_helper import load_source
 # Add tests path to syspath
 sys.path.append('../')
 
@@ -69,7 +68,7 @@ def __call__(self):
 
 def parse_testbed(testbedfile, servers):
     """Return a dictionary containing mapping from server name to nightly testbeds that need restart-ptf."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     all_testbeds = testbed.TestbedInfo(testbedfile).testbed_topo
     nightly_dir = os.path.join(SONIC_MGMT_DIR, ".azure-pipelines", "nightly")
@@ -117,7 +116,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/roles/vm_set/tasks/control_mux_simulator.yml

@@ -17,27 +17,43 @@
       werkzeug_version: "1.0.1"
       python_command: "python"
 
-  - name: Use newer Flask version for pip3
+  - name: Use newer Flask and Werkzeug version for pip3
     set_fact:
       flask_version: "2.3.3"
-      python_command: "python3"
-    when: pip_executable == "pip3"
-
-  - name: Use newer Werkzeug version for pip3
-    set_fact:
       werkzeug_version: "2.3.7"
       python_command: "python3"
     when: pip_executable == "pip3"
 
-  - name: Install flask
-    pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
-
-  - name: Install werkzeug
-    pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
+  - name: Run python3 in a virtualenv
+    set_fact:
+      python_command: "/tmp/sonic-mgmt-virtualenv/bin/python3"
+    when: host_distribution_version.stdout >= "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout < "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout >= "24.04"
 
   - name: Copy the mux simulator to test server
     copy:

ansible/roles/vm_set/tasks/docker.yml

@@ -1,8 +1,3 @@
-- name: Add docker official GPG key
-  apt_key: url=https://download.docker.com/linux/ubuntu/gpg state=present
-  become: yes
-  environment: "{{ proxy_env | default({}) }}"
-
 - name: Check docker repository
   find:
     paths: /etc/apt/sources.list.d/
@@ -54,6 +49,13 @@
   become: yes
   when: host_distribution_version.stdout == "22.04" and docker_repo.matched == 0
 
+- name: Add docker repository for 24.04
+  apt_repository:
+    repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable
+    state: present
+  become: yes
+  when: host_distribution_version.stdout == "24.04" and docker_repo.matched == 0
+
 # In ansible 2.8, there isn't update_cache_retries option in apt module, we can manually run update as a seperate and retryable step
 - name: Run the "apt-get update" as a separate and retryable step
   apt:
@@ -94,4 +96,23 @@
     pip: name=docker version=7.1.0 state=forcereinstall executable={{ pip_executable }}
     become: yes
     environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Update python3 packages
+  block:
+  - name: Ensure python3-venv is installed
+    apt:
+      name: python3-venv
+      state: present
+      update_cache: yes
+    become: yes
+  - name: Install python packages
+    pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+    become: yes
+    environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"