diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 7607f61900f..0283e831f63 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -170,7 +170,7 @@ fact_caching_timeout = 86400 [privilege_escalation] #become=True -become_method='sudo' +become_method=sudo #become_user='root' #become_ask_pass=False diff --git a/ansible/plugins/connection/multi_passwd_ssh.py b/ansible/plugins/connection/multi_passwd_ssh.py index 9834c0afe1f..73997fd7631 100644 --- a/ansible/plugins/connection/multi_passwd_ssh.py +++ b/ansible/plugins/connection/multi_passwd_ssh.py @@ -1,5 +1,11 @@ import hashlib -import imp +try: + import importlib.util + import importlib.machinery + use_importlib = True +except ImportError: + import imp + use_importlib = False import logging import os @@ -10,9 +16,24 @@ logger = logging.getLogger(__name__) + +def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + + # HACK: workaround to import the SSH connection plugin _ssh_mod = os.path.join(os.path.dirname(connection.__file__), "ssh.py") -_ssh = imp.load_source("_ssh", _ssh_mod) +if use_importlib: + _ssh = load_source("_ssh", _ssh_mod) +else: + _ssh = imp.load_source("_ssh", _ssh_mod) # Use same options as the builtin Ansible SSH plugin DOCUMENTATION = _ssh.DOCUMENTATION diff --git a/ansible/roles/vm_set/tasks/control_mux_simulator.yml b/ansible/roles/vm_set/tasks/control_mux_simulator.yml index 4e837c6895b..9b034e6c2a5 100644 --- a/ansible/roles/vm_set/tasks/control_mux_simulator.yml +++ b/ansible/roles/vm_set/tasks/control_mux_simulator.yml @@ -17,27 +17,43 @@ werkzeug_version: "1.0.1" python_command: "python" - - name: Use newer Flask version for pip3 + - name: Use newer Flask and Werkzeug version for pip3 set_fact: flask_version: "2.3.3" - python_command: "python3" - when: pip_executable == "pip3" - - - name: Use newer Werkzeug version for pip3 - set_fact: werkzeug_version: "2.3.7" python_command: "python3" when: pip_executable == "pip3" - - name: Install flask - pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }} - become: yes - environment: "{{ proxy_env | default({}) }}" - - - name: Install werkzeug - pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }} - become: yes - environment: "{{ proxy_env | default({}) }}" + - name: Run python3 in a virtualenv + set_fact: + python_command: "/tmp/sonic-mgmt-virtualenv/bin/python3" + when: host_distribution_version.stdout >= "24.04" + + - name: Install Flask and Werkzeug + block: + - name: Install flask + pip: name=flask version={{ flask_version }} state=forcereinstall executable={{ pip_executable }} + become: yes + environment: "{{ proxy_env | default({}) }}" + + - name: Install werkzeug + pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall executable={{ pip_executable }} + become: yes + environment: "{{ proxy_env | default({}) }}" + when: host_distribution_version.stdout < "24.04" + + - name: Install Flask and Werkzeug + block: + - name: Install flask + pip: name=flask version={{ flask_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true + become: yes + environment: "{{ proxy_env | default({}) }}" + + - name: Install werkzeug + pip: name=werkzeug version={{ werkzeug_version }} state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true + become: yes + environment: "{{ proxy_env | default({}) }}" + when: host_distribution_version.stdout >= "24.04" - name: Copy the mux simulator to test server copy: diff --git a/ansible/roles/vm_set/tasks/docker.yml b/ansible/roles/vm_set/tasks/docker.yml index 2ae4488c769..cbf53221203 100644 --- a/ansible/roles/vm_set/tasks/docker.yml +++ b/ansible/roles/vm_set/tasks/docker.yml @@ -55,6 +55,13 @@ become: yes when: host_distribution_version.stdout == "22.04" and docker_repo.matched == 0 +- name: Add docker repository for 24.04 + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable + state: present + become: yes + when: host_distribution_version.stdout == "24.04" and docker_repo.matched == 0 + # In ansible 2.8, there isn't update_cache_retries option in apt module, we can manually run update as a seperate and retryable step - name: Run the "apt-get update" as a separate and retryable step apt: @@ -95,4 +102,17 @@ pip: name=docker version=7.1.0 state=forcereinstall executable={{ pip_executable }} become: yes environment: "{{ proxy_env | default({}) }}" - when: pip_executable=="pip3" + when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04" + +- name: Update python3 packages + block: + - name: Install python packages + pip: name=docker version=7.1.0 state=forcereinstall virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv" + become: yes + environment: "{{ proxy_env | default({}) }}" + when: host_distribution_version.stdout >= "24.04" + +- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer + set_fact: + ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python" + when: host_distribution_version.stdout >= "24.04" diff --git a/ansible/roles/vm_set/tasks/main.yml b/ansible/roles/vm_set/tasks/main.yml index 44c47d25982..0a7ec4d4e5f 100644 --- a/ansible/roles/vm_set/tasks/main.yml +++ b/ansible/roles/vm_set/tasks/main.yml @@ -69,35 +69,27 @@ - block: - name: Install necessary packages - apt: pkg={{ item }} update_cache=yes cache_valid_time=86400 - become: yes + apt: + update_cache: yes + cache_valid_time: 86400 + pkg: + - ifupdown + - openvswitch-switch + - net-tools + - bridge-utils + - util-linux + - iproute2 + - vlan + - apt-transport-https + - ca-certificates + - curl + - software-properties-common + - libvirt-clients register: apt_res retries: 2 delay: 30 until: apt_res is success - with_items: - - ifupdown - - qemu - - openvswitch-switch - - net-tools - - bridge-utils - - util-linux - - - name: Install necessary packages - apt: pkg={{ item }} update_cache=yes cache_valid_time=86400 become: yes - register: apt_res - retries: 2 - delay: 30 - until: apt_res is success - with_items: - - iproute2 - - vlan - - apt-transport-https - - ca-certificates - - curl - - software-properties-common - - libvirt-clients - name: Install necessary packages register: apt_res @@ -125,23 +117,18 @@ - libvirt-daemon-system - qemu-system-x86 become: yes - when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04" + when: host_distribution_version.stdout >= "20.04" when: package_installation|bool - name: Get default pip_executable set_fact: pip_executable: pip - when: pip_executable is not defined and host_distribution_version.stdout != "20.04" and host_distribution_version.stdout != "22.04" + when: pip_executable is not defined and host_distribution_version.stdout < "20.04" - name: Get default pip_executable set_fact: pip_executable: pip3 - when: pip_executable is not defined and (host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04") - -- name: remove old python packages - pip: name=docker-py state=absent executable={{ pip_executable }} - environment: "{{ proxy_env | default({}) }}" - ignore_errors: yes + when: pip_executable is not defined and (host_distribution_version.stdout >= "20.04") - include_tasks: docker.yml when: package_installation|bool @@ -150,7 +137,13 @@ pip: name=requests version=2.32.3 state=present executable={{ pip_executable }} become: yes environment: "{{ proxy_env | default({}) }}" - when: pip_executable=="pip3" + when: pip_executable=="pip3" and host_distribution_version.stdout < "24.04" + +- name: Install requests package + pip: name=requests version=2.32.3 state=present virtualenv=/tmp/sonic-mgmt-virtualenv virtualenv_site_packages=true virtualenv_command="python3 -m venv" + become: yes + environment: "{{ proxy_env | default({}) }}" + when: host_distribution_version.stdout >= "24.04" - name: Ensure {{ ansible_user }} in docker,sudo group user: @@ -165,7 +158,7 @@ append: yes groups: libvirt become: yes - when: host_distribution_version.stdout == "20.04" or host_distribution_version.stdout == "22.04" + when: host_distribution_version.stdout >= "20.04" - name: Install br_netfilter kernel module become: yes @@ -223,7 +216,7 @@ # root_path is supposed to be absolute path. - set_fact: root_path: "{{ home_path + '/' + root_path }}" - when: "not '{{ root_path }}'.startswith('/')" + when: "not root_path.startswith('/')" - debug: msg="home_path = {{ home_path }} root_path = {{ root_path }}" @@ -382,3 +375,8 @@ loop_control: loop_var: dut_name when: duts_name is defined + +- name: Use virtualenv for all Python scripts on Ubuntu 24.04 and newer + set_fact: + ansible_python_interpreter: "/tmp/sonic-mgmt-virtualenv/bin/python" + when: host_distribution_version.stdout >= "24.04" diff --git a/tests/arp/test_unknown_mac.py b/tests/arp/test_unknown_mac.py index a31a3b73a83..b6752ee7834 100644 --- a/tests/arp/test_unknown_mac.py +++ b/tests/arp/test_unknown_mac.py @@ -30,7 +30,8 @@ def initClassVars(func): """ Automatically assign instance variables. currently handles only arg list """ - names, varargs, keywords, defaults = inspect.getargspec(func) + signature = inspect.signature(func) + names = list(signature.parameters.keys()) @functools.wraps(func) def wrapper(self, *args): diff --git a/tests/common/cache/facts_cache.py b/tests/common/cache/facts_cache.py index ab4455a46b9..c40e5f9ff0f 100644 --- a/tests/common/cache/facts_cache.py +++ b/tests/common/cache/facts_cache.py @@ -209,10 +209,8 @@ def _get_default_zone(function, func_args, func_kargs): raise ValueError("Failed to get attribute 'hostname' of type string from instance of type %s." % type(func_args[0])) zone = hostname - if sys.version_info.major > 2: - arg_names = inspect.getfullargspec(function)[0] - else: - arg_names = inspect.getargspec(function)[0] + signature = inspect.signature(function) + arg_names = list(signature.parameters.keys()) if 'namespace' in arg_names: try: index = arg_names.index('namespace') diff --git a/tests/common/connections/base_console_conn.py b/tests/common/connections/base_console_conn.py index 31196689351..5e9d63cac88 100644 --- a/tests/common/connections/base_console_conn.py +++ b/tests/common/connections/base_console_conn.py @@ -5,7 +5,10 @@ import logging from netmiko.cisco_base_connection import CiscoBaseConnection -from netmiko.ssh_exception import NetMikoAuthenticationException +try: + from netmiko.ssh_exception import NetMikoAuthenticationException +except ImportError: + from netmiko.exceptions import NetMikoAuthenticationException # For interactive shell import sys diff --git a/tests/common/connections/ssh_console_conn.py b/tests/common/connections/ssh_console_conn.py index be53b050d97..760f6a1449f 100644 --- a/tests/common/connections/ssh_console_conn.py +++ b/tests/common/connections/ssh_console_conn.py @@ -1,7 +1,10 @@ import time import re from .base_console_conn import CONSOLE_SSH_DIGI_CONFIG, BaseConsoleConn, CONSOLE_SSH -from netmiko.ssh_exception import NetMikoAuthenticationException +try: + from netmiko.ssh_exception import NetMikoAuthenticationException +except ImportError: + from netmiko.exceptions import NetMikoAuthenticationException from paramiko.ssh_exception import SSHException diff --git a/tests/common/connections/telnet_console_conn.py b/tests/common/connections/telnet_console_conn.py index e522c221bb9..0efbaa545ff 100644 --- a/tests/common/connections/telnet_console_conn.py +++ b/tests/common/connections/telnet_console_conn.py @@ -1,7 +1,10 @@ import time import re from .base_console_conn import BaseConsoleConn -from netmiko.ssh_exception import NetMikoAuthenticationException +try: + from netmiko.ssh_exception import NetMikoAuthenticationException +except ImportError: + from netmiko.exceptions import NetMikoAuthenticationException class TelnetConsoleConn(BaseConsoleConn): diff --git a/tests/common/devices/duthosts.py b/tests/common/devices/duthosts.py index ca25eeea1d7..a1165950dd1 100644 --- a/tests/common/devices/duthosts.py +++ b/tests/common/devices/duthosts.py @@ -174,7 +174,7 @@ def __getitem__(self, index): unicode_type = str if sys.version_info.major >= 3 else unicode # noqa: F821 if type(index) == int: return self.nodes[index] - elif type(index) in [str, unicode_type]: + elif type(index) in [str, unicode_type] or isinstance(index, str): for node in self.nodes: if node.hostname == index: return node diff --git a/tests/common/helpers/platform_api/scripts/platform_api_server.py b/tests/common/helpers/platform_api/scripts/platform_api_server.py index fcbed445bda..2972149d275 100644 --- a/tests/common/helpers/platform_api/scripts/platform_api_server.py +++ b/tests/common/helpers/platform_api/scripts/platform_api_server.py @@ -73,11 +73,8 @@ def do_platform_api(self): while len(path) != 1: _dir = path.pop() - # TODO: Clean this up once we no longer need to support Python 2 - if sys.version_info.major == 3: - args = inspect.getfullargspec(getattr(obj, 'get_' + _dir)).args - else: - args = inspect.getargspec(getattr(obj, 'get_' + _dir)).args + signature = inspect.signature(getattr(obj, 'get_' + _dir)) + args = list(signature.parameters.keys()) if 'index' in args: _idx = int(path.pop()) diff --git a/tests/ssh/conftest.py b/tests/ssh/conftest.py index 8360f149e67..f3d17101543 100644 --- a/tests/ssh/conftest.py +++ b/tests/ssh/conftest.py @@ -1,4 +1,10 @@ -import imp +try: + import importlib.util + import importlib.machinery + use_importlib = True +except ImportError: + import imp + use_importlib = False import subprocess import pytest import logging @@ -25,6 +31,17 @@ ] +def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + + def generate_ssh_ciphers(request, typename): if typename == "enc": remote_cmd = "ssh -Q cipher" @@ -43,7 +60,10 @@ def generate_ssh_ciphers(request, typename): testbed_name = request.config.option.testbed testbed_file = request.config.option.testbed_file - testbed_module = imp.load_source('testbed', 'common/testbed.py') + if use_importlib: + testbed_module = load_source('testbed', 'common/testbed.py') + else: + testbed_module = imp.load_source('testbed', 'common/testbed.py') tbinfo = testbed_module.TestbedInfo( testbed_file).testbed_topo.get(testbed_name, None)