Add support for Ubuntu 24.04 as the host and sonic-mgmt container (sonic-net#16039)

Approach
What is the motivation for this PR?
When running with Ubuntu 24.04 host, many tasks from a sonic-mgmt container may fail. For example, running add-topo for a KVM testbed will fail. This is because of a newer version of Python being used in Ubuntu 24.04 that doesn't support some of the packages that ansible modules attempt to import.

How did you do it?
Update the code in sonic-mgmt to add support for Ubuntu 24.04 as a host and for the sonic-mgmt container to be running Ubuntu 24.04.

How did you verify/test it?
Tested on a Ubuntu 24.04 host and sonic-mgmt container based on Ubuntu 24.04.

Signed-off-by: Guy Shemesh <gshemesh@nvidia.com>

Author: saiarcot895

ansible/ansible.cfg

@@ -170,7 +170,7 @@ fact_caching_timeout = 86400
 
 [privilege_escalation]
 #become=True
-become_method='sudo'
+become_method=sudo
 #become_user='root'
 #become_ask_pass=False
 

ansible/plugins/connection/multi_passwd_ssh.py

@@ -1,5 +1,11 @@
 import hashlib
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 import logging
 import os
 
@@ -10,9 +16,24 @@
 
 logger = logging.getLogger(__name__)
 
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+
 # HACK: workaround to import the SSH connection plugin
 _ssh_mod = os.path.join(os.path.dirname(connection.__file__), "ssh.py")
-_ssh = imp.load_source("_ssh", _ssh_mod)
+if use_importlib:
+    _ssh = load_source("_ssh", _ssh_mod)
+else:
+    _ssh = imp.load_source("_ssh", _ssh_mod)
 
 # Use same options as the builtin Ansible SSH plugin
 DOCUMENTATION = _ssh.DOCUMENTATION

ansible/roles/vm_set/tasks/control_mux_simulator.yml

@@ -17,27 +17,43 @@
       werkzeug_version: "1.0.1"
       python_command: "python"
 
-  - name: Use newer Flask version for pip3
+  - name: Use newer Flask and Werkzeug version for pip3
     set_fact:
       flask_version: "2.3.3"
-      python_command: "python3"
-    when: pip_executable == "pip3"
-
-  - name: Use newer Werkzeug version for pip3
-    set_fact:
       werkzeug_version: "2.3.7"
       python_command: "python3"
     when: pip_executable == "pip3"
 
-  - name: Install flask
-    pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
-
-  - name: Install werkzeug
-    pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
-    become: yes
-    environment: "{{ proxy_env | default({}) }}"
+  - name: Run python3 in a virtualenv
+    set_fact:
+      python_command: "/tmp/sonic-mgmt-virtualenv/bin/python3"
+    when: host_distribution_version.stdout >= "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }}
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout < "24.04"
+
+  - name: Install Flask and Werkzeug
+    block:
+    - name: Install flask
+      pip: name=flask version={{ flask_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Install werkzeug
+      pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true
+      become: yes
+      environment: "{{ proxy_env | default({}) }}"
+    when: host_distribution_version.stdout >= "24.04"
 
   - name: Copy the mux simulator to test server
     copy:

ansible/roles/vm_set/tasks/docker.yml

@@ -55,6 +55,13 @@
   become: yes
   when: host_distribution_version.stdout == "22.04" and docker_repo.matched == 0
 
+- name: Add docker repository for 24.04
+  apt_repository:
+    repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable
+    state: present
+  become: yes
+  when: host_distribution_version.stdout == "24.04" and docker_repo.matched == 0
+
 # In ansible 2.8, there isn't update_cache_retries option in apt module, we can manually run update as a seperate and retryable step
 - name: Run the "apt-get update" as a separate and retryable step
   apt:
@@ -95,4 +102,17 @@
     pip: name=docker version=7.1.0 state=forcereinstall executable={{ pip_executable }}
     become: yes
     environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Update python3 packages
+  block:
+  - name: Install python packages
+    pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+    become: yes
+    environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"

ansible/roles/vm_set/tasks/main.yml

@@ -69,35 +69,27 @@
 
 - block:
     - name: Install necessary packages
-      apt: pkg={{ item }} update_cache=yes cache_valid_time=86400
-      become: yes
+      apt:
+        update_cache: yes
+        cache_valid_time: 86400
+        pkg:
+          - ifupdown
+          - openvswitch-switch
+          - net-tools
+          - bridge-utils
+          - util-linux
+          - iproute2
+          - vlan
+          - apt-transport-https
+          - ca-certificates
+          - curl
+          - software-properties-common
+          - libvirt-clients
       register: apt_res
       retries: 2
       delay: 30
       until: apt_res is success
-      with_items:
-        - ifupdown
-        - qemu
-        - openvswitch-switch
-        - net-tools
-        - bridge-utils
-        - util-linux
-
-    - name: Install necessary packages
-      apt: pkg={{ item }} update_cache=yes cache_valid_time=86400
       become: yes
-      register: apt_res
-      retries: 2
-      delay: 30
-      until: apt_res is success
-      with_items:
-        - iproute2
-        - vlan
-        - apt-transport-https
-        - ca-certificates
-        - curl
-        - software-properties-common
-        - libvirt-clients
 
     - name: Install necessary packages
       register: apt_res
@@ -125,23 +117,18 @@
         - libvirt-daemon-system
         - qemu-system-x86
       become: yes
-      when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04"
+      when: host_distribution_version.stdout >= "20.04"
   when: package_installation|bool
 
 - name: Get default pip_executable
   set_fact:
     pip_executable: pip
-  when: pip_executable is not defined and host_distribution_version.stdout != "20.04" and host_distribution_version.stdout != "22.04"
+  when: pip_executable is not defined and host_distribution_version.stdout < "20.04"
 
 - name: Get default pip_executable
   set_fact:
     pip_executable: pip3
-  when: pip_executable is not defined and (host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04")
-
-- name: remove old python packages
-  pip: name=docker-py state=absent executable={{ pip_executable }}
-  environment: "{{ proxy_env | default({}) }}"
-  ignore_errors: yes
+  when: pip_executable is not defined and (host_distribution_version.stdout >= "20.04")
 
 - include_tasks: docker.yml
   when: package_installation|bool
@@ -150,7 +137,13 @@
   pip: name=requests version=2.32.3 state=present executable={{ pip_executable }}
   become: yes
   environment: "{{ proxy_env | default({}) }}"
-  when: pip_executable=="pip3"
+  when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04"
+
+- name: Install requests package
+  pip: name=requests version=2.32.3 state=present virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
+  become: yes
+  environment: "{{ proxy_env | default({}) }}"
+  when: host_distribution_version.stdout >= "24.04"
 
 - name: Ensure {{ ansible_user }} in docker,sudo group
   user:
@@ -165,7 +158,7 @@
     append: yes
     groups: libvirt
   become: yes
-  when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04"
+  when: host_distribution_version.stdout >= "20.04"
 
 - name: Install br_netfilter kernel module
   become: yes
@@ -223,7 +216,7 @@
     # root_path is supposed to be absolute path.
     - set_fact:
         root_path: "{{ home_path + '/' + root_path }}"
-      when: "not '{{ root_path }}'.startswith('/')"
+      when: "not root_path.startswith('/')"
 
     - debug: msg="home_path = {{ home_path }} root_path = {{ root_path }}"
 
@@ -382,3 +375,8 @@
   loop_control:
     loop_var: dut_name
   when: duts_name is defined
+
+- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer
+  set_fact:
+    ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python"
+  when: host_distribution_version.stdout >= "24.04"

tests/arp/test_unknown_mac.py

@@ -30,7 +30,8 @@ def initClassVars(func):
     """
     Automatically assign instance variables. currently handles only arg list
     """
-    names, varargs, keywords, defaults = inspect.getargspec(func)
+    signature = inspect.signature(func)
+    names = list(signature.parameters.keys())
 
     @functools.wraps(func)
     def wrapper(self, *args):

tests/common/cache/facts_cache.py

@@ -209,10 +209,8 @@ def _get_default_zone(function, func_args, func_kargs):
         raise ValueError("Failed to get attribute 'hostname' of type string from instance of type %s."
                          % type(func_args[0]))
     zone = hostname
-    if sys.version_info.major > 2:
-        arg_names = inspect.getfullargspec(function)[0]
-    else:
-        arg_names = inspect.getargspec(function)[0]
+    signature = inspect.signature(function)
+    arg_names = list(signature.parameters.keys())
     if 'namespace' in arg_names:
         try:
             index = arg_names.index('namespace')

tests/common/connections/base_console_conn.py

@@ -5,7 +5,10 @@
 import logging
 
 from netmiko.cisco_base_connection import CiscoBaseConnection
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 
 # For interactive shell
 import sys

tests/common/connections/ssh_console_conn.py

@@ -1,7 +1,10 @@
 import time
 import re
 from .base_console_conn import CONSOLE_SSH_DIGI_CONFIG, BaseConsoleConn, CONSOLE_SSH
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 from paramiko.ssh_exception import SSHException
 
 

tests/common/connections/telnet_console_conn.py

@@ -1,7 +1,10 @@
 import time
 import re
 from .base_console_conn import BaseConsoleConn
-from netmiko.ssh_exception import NetMikoAuthenticationException
+try:
+    from netmiko.ssh_exception import NetMikoAuthenticationException
+except ImportError:
+    from netmiko.exceptions import NetMikoAuthenticationException
 
 
 class TelnetConsoleConn(BaseConsoleConn):