Support Ubuntu 24.04 server in KVM (#17883)

What is the motivation for this PR?
When running with Ubuntu 24.04 host, many tasks from a sonic-mgmt container may fail.
One reason is that a newer version of Python being used in Ubuntu 24.04 that doesn't support some of the packages that ansible modules attempt to import. Such as imp module is removed in Python 3.11, need to use importlib, and
PR #16039 did some replacement but still need to replace in somewhere else.
Another reason is that some ansible modules must be running in a python3 venv for scecurity reason.
Also there are some access issues need to fix.

How did you do it?
Try to use import importlib instead of importing imp
Install python3-venv in Ubuntu 24.04 server
Update libvirt qemu configuration in Ubuntu 24.04 server
* fix

* remove

---------

Co-authored-by: xwjiang2021 <[email protected]>

Author: xwjiang-ms

ansible/devutil/conn_graph_helper.py

@@ -1,11 +1,32 @@
 import os
 import inspect
 import sys
-import imp
+try:
+    import importlib.util
+    import importlib.machinery
+    use_importlib = True
+except ImportError:
+    import imp
+    use_importlib = False
 
 CONN_GRAPH_LOG = "/tmp/conn_graph_debug.txt"
 
 
+def load_source(modname, filename):
+    if use_importlib:
+        loader = importlib.machinery.SourceFileLoader(modname, filename)
+        spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+        module = importlib.util.module_from_spec(spec)
+        # The module is always executed and not cached in sys.modules.
+        # Uncomment the following line to cache the module.
+        # sys.modules[module.__name__] = module
+        loader.exec_module(module)
+    else:
+        # For Python 2.x compatibility
+        module = imp.load_source(modname, filename)
+    return module
+
+
 def get_conn_graph_facts(hostnames):
     """
     @summary: Load conn_graph_facts from conn_graph_facts.xml
@@ -18,7 +39,7 @@ def get_conn_graph_facts(hostnames):
     if ansible_path not in sys.path:
         sys.path.append(ansible_path)
 
-    utils = imp.load_source('conn_graph_utils', os.path.join(
+    utils = load_source('conn_graph_utils', os.path.join(
         ansible_path, 'library/conn_graph_facts.py'))
     utils.LAB_GRAPHFILE_PATH = os.path.join(
         ansible_path, utils.LAB_GRAPHFILE_PATH)

ansible/health_checker.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-
-import imp
 import os
 import logging
 import sys
@@ -13,6 +11,7 @@
 except ImportError:
     # ToDo: Support running without Ansible
     has_ansible = False
+from devutil.conn_graph_helper import load_source
 
 ANSIBLE_DIR = os.path.abspath(os.path.dirname(__file__))
 SONIC_MGMT_DIR = os.path.dirname(ANSIBLE_DIR)
@@ -35,7 +34,7 @@
 
 def get_testbeds_dict():
     """Return a dictionary containing mapping from dut hostname to testbed name."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds_dict = testbed.TestbedInfo(TESTBED_FILE).testbed_topo
     return testbeds_dict

ansible/recover_server.py

@@ -10,15 +10,14 @@
 import argparse
 import collections
 import datetime
-import imp
 import logging
 import os
 import subprocess
 import sys
 import tempfile
 import threading
 import time
-
+from devutil.conn_graph_helper import load_source
 from tabulate import tabulate
 # Add tests path to syspath
 sys.path.append('../')
@@ -40,7 +39,7 @@
 
 def parse_testbed(testbedfile, testbed_servers):
     """Return a dictionary containing mapping from server name to testbeds."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     testbeds = {server_name: list() for server_name in testbed_servers}
     for tbname, tb in testbed.TestbedInfo(testbedfile).testbed_topo.items():
@@ -258,7 +257,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/restart_nightly_ptf.py

@@ -2,15 +2,14 @@
 
 import argparse
 import logging
-import imp
 import os
 import recover_server
 import sys
 import collections
 import datetime
 import time
 import tempfile
-
+from devutil.conn_graph_helper import load_source
 # Add tests path to syspath
 sys.path.append('../')
 
@@ -69,7 +68,7 @@ def __call__(self):
 
 def parse_testbed(testbedfile, servers):
     """Return a dictionary containing mapping from server name to nightly testbeds that need restart-ptf."""
-    testbed = imp.load_source('testbed', os.path.join(
+    testbed = load_source('testbed', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/testbed.py'))
     all_testbeds = testbed.TestbedInfo(testbedfile).testbed_topo
     nightly_dir = os.path.join(SONIC_MGMT_DIR, ".azure-pipelines", "nightly")
@@ -117,7 +116,7 @@ def _join_all(threads):
                 break
             time.sleep(5)
 
-    utilities = imp.load_source('utilities', os.path.join(
+    utilities = load_source('utilities', os.path.join(
         SONIC_MGMT_DIR, 'tests/common/utilities.py'))
 
     curr_date = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

ansible/roles/vm_set/tasks/docker.yml

@@ -106,6 +106,12 @@
 
 - name: Update python3 packages
   block:
+  - name: Ensure python3-venv is installed
+    apt:
+      name: python3-venv
+      state: present
+      update_cache: yes
+    become: yes
   - name: Install python packages
     pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv"
     become: yes

ansible/roles/vm_set/tasks/main.yml

@@ -160,6 +160,31 @@
   become: yes
   when: host_distribution_version.stdout >= "20.04"
 
+- name: Update libvirt qemu configuration
+  block:
+  - name: Set user to root in qemu.conf
+    lineinfile:
+      path: /etc/libvirt/qemu.conf
+      regexp: '^#?user\s*=.*'
+      line: 'user = "root"'
+      state: present
+      backrefs: yes
+    become: yes
+  - name: Set group to root in qemu.conf
+    lineinfile:
+      path: /etc/libvirt/qemu.conf
+      regexp: '^#?group\s*=.*'
+      line: 'group = "root"'
+      state: present
+      backrefs: yes
+    become: yes
+  - name: Restart libvirtd to apply qemu.conf changes
+    service:
+      name: libvirtd
+      state: restarted
+    become: yes
+  when: host_distribution_version.stdout >= "24.04"
+
 - name: Install br_netfilter kernel module
   become: yes
   modprobe: name=br_netfilter state=present