Add support for Ubuntu 24.04 as the host and sonic-mgmt container

ansible/ansible.cfg

@@ -170,7 +170,7 @@ fact_caching_timeout = 86400
 
 [privilege_escalation]
 #become=True
-become_method='sudo'
+become_method=sudo
 #become_user='root'
 #become_ask_pass=False
 

ansible/plugins/connection/multi_passwd_ssh.py

@@ -1,5 +1,11 @@
 import hashlib
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 import logging
 import os
 
@@ -10,9 +16,24 @@
 
 logger = logging.getLogger(__name__)
 
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+
 # HACK: workaround to import the SSH connection plugin
 _ssh_mod = os.path.join(os.path.dirname(connection.__file__), "ssh.py")
-_ssh = imp.load_source("_ssh", _ssh_mod)
+if use_importlib:
+    _ssh = load_source("_ssh", _ssh_mod)
+else:
+    _ssh = imp.load_source("_ssh", _ssh_mod)
 
 # Use same options as the builtin Ansible SSH plugin
 DOCUMENTATION = _ssh.DOCUMENTATION

ansible/roles/vm_set/tasks/control_mux_simulator.yml

@@ -17,27 +17,43 @@
       werkzeug_version: "1.0.1"
       python_command: "python"
 
-  - name: Use newer Flask version for pip3
+  - name: Use newer Flask and Werkzeug version for pip3
     set_fact:
       flask_version: "2.3.3"
-      python_command: "python3"
-    when: pip_executable == "pip3"
-
-  - name: Use newer Werkzeug version for pip3
-    set_fact:
       werkzeug_version: "2.3.7"
       python_command: "python3"
     when: pip_executable == "pip3"
 
-  - name: Install flask
-    pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
-
-  - name: Install werkzeug
-    pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
+  - name: Run python3 in a virtualenv
+    set_fact:
+      python_command: "/tmp/sonic-mgmt-virtualenv/bin/python3"
+    when: host_distribution_version.stdout >= "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout < "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout >= "24.04"
 
   - name: Copy the mux simulator to test server
     copy:

ansible/roles/vm_set/tasks/docker.yml

@@ -55,6 +55,13 @@
   become: yes
   when: host_distribution_version.stdout == "22.04" and docker_repo.matched == 0
 
+- name: Add docker repository for 24.04
+  apt_repository:
+    repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable
+    state: present
+  become: yes
+  when: host_distribution_version.stdout == "24.04" and docker_repo.matched == 0
+
 # In ansible 2.8, there isn't update_cache_retries option in apt module, we can manually run update as a seperate and retryable step
 - name: Run the "apt-get update" as a separate and retryable step
   apt:
@@ -95,4 +102,17 @@
     pip: name=docker version=7.1.0 state=forcereinstall executable={{ pip_executable }}
     become: yes
     environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Update python3 packages
+  block:
+  - name: Install python packages
+    pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+    become: yes
+    environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"

ansible/roles/vm_set/tasks/main.yml

@@ -69,35 +69,27 @@
 
 - block:
     - name: Install necessary packages
-      apt: pkg={{ item }} update_cache=yes cache_valid_time=86400
-      become: yes
+      apt:
+        update_cache: yes
+        cache_valid_time: 86400
+        pkg:
+          - ifupdown
+          - openvswitch-switch
+          - net-tools
+          - bridge-utils
+          - util-linux
+          - iproute2
+          - vlan
+          - apt-transport-https
+          - ca-certificates
+          - curl
+          - software-properties-common
+          - libvirt-clients
       register: apt_res
       retries: 2
       delay: 30
       until: apt_res is success
-      with_items:
-        - ifupdown
-        - qemu
-        - openvswitch-switch
-        - net-tools
-        - bridge-utils
-        - util-linux
-
-    - name: Install necessary packages
-      apt: pkg={{ item }} update_cache=yes cache_valid_time=86400
       become: yes
-      register: apt_res
-      retries: 2
-      delay: 30
-      until: apt_res is success
-      with_items:
-        - iproute2
-        - vlan
-        - apt-transport-https
-        - ca-certificates
-        - curl
-        - software-properties-common
-        - libvirt-clients
 
     - name: Install necessary packages
       register: apt_res
@@ -125,23 +117,18 @@
         - libvirt-daemon-system
         - qemu-system-x86
       become: yes
-      when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04"
+      when: host_distribution_version.stdout >= "20.04"
   when: package_installation|bool
 
 - name: Get default pip_executable
   set_fact:
     pip_executable: pip
-  when: pip_executable is not defined and host_distribution_version.stdout != "20.04" and host_distribution_version.stdout != "22.04"
+  when: pip_executable is not defined and host_distribution_version.stdout < "20.04"
 
 - name: Get default pip_executable
   set_fact:
     pip_executable: pip3
-  when: pip_executable is not defined and (host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04")
-
-- name: remove old python packages
-  pip: name=docker-py state=absent executable={{ pip_executable }}
-  environment: "{{ proxy_env | default({}) }}"
-  ignore_errors: yes
+  when: pip_executable is not defined and (host_distribution_version.stdout >= "20.04")
 
 - include_tasks: docker.yml
   when: package_installation|bool
@@ -150,7 +137,13 @@
   pip: name=requests version=2.32.3 state=present executable={{ pip_executable }}
   become: yes
   environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Install requests package
+  pip: name=requests version=2.32.3 state=present virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+  become: yes
+  environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
 
 - name: Ensure {{ ansible_user }} in docker,sudo group
   user:
@@ -165,7 +158,7 @@
     append: yes
     groups: libvirt
   become: yes
-  when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04"
+  when: host_distribution_version.stdout >= "20.04"
 
 - name: Install br_netfilter kernel module
   become: yes
@@ -223,7 +216,7 @@
     # root_path is supposed to be absolute path.
     - set_fact:
         root_path: "{{ home_path + '/' + root_path }}"
-      when: "not '{{ root_path }}'.startswith('/')"
+      when: "not root_path.startswith('/')"
 
     - debug: msg="home_path = {{ home_path }} root_path = {{ root_path }}"
 
@@ -382,3 +375,8 @@
   loop_control:
     loop_var: dut_name
   when: duts_name is defined
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"

tests/arp/test_unknown_mac.py

@@ -30,7 +30,8 @@ def initClassVars(func):
     """
     Automatically assign instance variables. currently handles only arg list
     """
-    names, varargs, keywords, defaults = inspect.getargspec(func)
+    signature = inspect.signature(func)
+    names = list(signature.parameters.keys())
 
     @functools.wraps(func)
     def wrapper(self, *args):

tests/common/cache/facts_cache.py

@@ -209,10 +209,8 @@ def _get_default_zone(function, func_args, func_kargs):
         raise ValueError("Failed to get attribute 'hostname' of type string from instance of type %s."
                          % type(func_args[0]))
     zone = hostname
-    if sys.version_info.major > 2:
-        arg_names = inspect.getfullargspec(function)[0]
-    else:
-        arg_names = inspect.getargspec(function)[0]
+    signature = inspect.signature(function)
+    arg_names = list(signature.parameters.keys())
     if 'namespace' in arg_names:
         try:
             index = arg_names.index('namespace')

tests/common/connections/base_console_conn.py

@@ -5,7 +5,10 @@
 import logging
 
 from netmiko.cisco_base_connection import CiscoBaseConnection
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 
 # For interactive shell
 import sys

tests/common/connections/ssh_console_conn.py

@@ -1,7 +1,10 @@
 import time
 import re
 from .base_console_conn import CONSOLE_SSH_DIGI_CONFIG, BaseConsoleConn, CONSOLE_SSH
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 from paramiko.ssh_exception import SSHException
 
 

tests/common/connections/telnet_console_conn.py

@@ -1,7 +1,10 @@
 import time
 import re
 from .base_console_conn import BaseConsoleConn
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 
 
 class TelnetConsoleConn(BaseConsoleConn):

tests/common/devices/duthosts.py

@@ -174,7 +174,7 @@ def __getitem__(self, index):
         unicode_type = str if sys.version_info.major >= 3 else unicode      # noqa: F821
         if type(index) == int:
             return self.nodes[index]
-        elif type(index) in [str, unicode_type]:
+        elif type(index) in [str, unicode_type] or isinstance(index, str):
             for node in self.nodes:
                 if node.hostname == index:
                     return node

tests/common/helpers/platform_api/scripts/platform_api_server.py

@@ -73,11 +73,8 @@ def do_platform_api(self):
         while len(path) != 1:
             _dir = path.pop()
 
-            # TODO: Clean this up once we no longer need to support Python 2
-            if sys.version_info.major == 3:
-                args = inspect.getfullargspec(getattr(obj, 'get_' + _dir)).args
-            else:
-                args = inspect.getargspec(getattr(obj, 'get_' + _dir)).args
+            signature = inspect.signature(getattr(obj, 'get_' + _dir))
+            args = list(signature.parameters.keys())
 
             if 'index' in args:
                 _idx = int(path.pop())

tests/ssh/conftest.py

@@ -1,4 +1,10 @@
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 import subprocess
 import pytest
 import logging
@@ -25,6 +31,17 @@
 ]
 
 
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+
 def generate_ssh_ciphers(request, typename):
     if typename == "enc":
         remote_cmd = "ssh -Q cipher"
@@ -43,7 +60,10 @@ def generate_ssh_ciphers(request, typename):
 
     testbed_name = request.config.option.testbed
     testbed_file = request.config.option.testbed_file
-    testbed_module = imp.load_source('testbed', 'common/testbed.py')
+    if use_importlib:
+        testbed_module = load_source('testbed', 'common/testbed.py')
+    else:
+        testbed_module = imp.load_source('testbed', 'common/testbed.py')
     tbinfo = testbed_module.TestbedInfo(
         testbed_file).testbed_topo.get(testbed_name, None)