diff --git a/files/image_config/interfaces/interfaces.j2 b/files/image_config/interfaces/interfaces.j2 index ef3a47ba043..27dcbc6617a 100644 --- a/files/image_config/interfaces/interfaces.j2 +++ b/files/image_config/interfaces/interfaces.j2 @@ -30,6 +30,14 @@ iface lo inet loopback {% endblock loopback %} {% block mgmt_interface %} +{%- if DEVICE_METADATA and 'bmc' in DEVICE_METADATA.keys() and ('bmc_if_name' in DEVICE_METADATA['bmc']) and ('bmc_if_addr' in DEVICE_METADATA['bmc']) and ('bmc_net_mask' in DEVICE_METADATA['bmc']) %} +# BMC usb0 interface +auto {{ DEVICE_METADATA['bmc']['bmc_if_name'] }} +iface {{ DEVICE_METADATA['bmc']['bmc_if_name'] }} inet static + address {{ DEVICE_METADATA['bmc']['bmc_if_addr'] }} + netmask {{ DEVICE_METADATA['bmc']['bmc_net_mask'] }} +{%- endif %} + # The management network interface {% if (ZTP_DHCP_DISABLED is not defined) and (ZTP is defined) and (ZTP['mode'] is defined and ZTP['mode']['profile'] == 'active') %} auto eth0 diff --git a/files/scripts/mlnx-bmc-test.py b/files/scripts/mlnx-bmc-test.py new file mode 100644 index 00000000000..834e07f3209 --- /dev/null +++ b/files/scripts/mlnx-bmc-test.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 + + +import sonic_platform +import sys +import os +import argparse +import time +import tqdm +import subprocess +from sonic_py_common.logger import Logger + + +logger = Logger() +logger.set_min_log_priority_info() + + +def validate_positive_int(value): + ivalue = int(value) + if ivalue <= 0: + raise argparse.ArgumentTypeError(f"{value} is not a positive integer") + return ivalue + + +def ping(host): + """Check if host is reachable via ping""" + command = ['/usr/bin/ping', '-c', '1', '-W', '1', host] + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + return True + except subprocess.CalledProcessError: + return False + + +def is_host_reachable(host, timeout): + """Wait for host to become reachable""" + start_time = time.time() + while time.time() - start_time < timeout: + if ping(host): + return True + time.sleep(1) + return False + + +def test_bmc_login(bmc, timeout=30): + """Test BMC login API with retry logic""" + print("\n=== Testing BMC Login ===") + print(f"Attempting to login to BMC with {timeout}s timeout...") + + start_time = time.time() + + while time.time() - start_time < timeout: + ret = bmc.login() + print(f"Login attempt result: {ret}") + + if ret == 0: + print("V BMC login successful") + token = bmc.get_login_token() + print(f"BMC token: {token[:20]}..." if token else "No token received") + return True + else: + print(f"X BMC login failed (retry in 1s)") + time.sleep(1) + + print(f"X BMC login failed after {timeout}s timeout") + return False + + +def test_bmc_logout(bmc): + """Test BMC logout API""" + print("\n=== Testing BMC Logout ===") + print("Attempting to logout from BMC...") + + ret = bmc.logout() + print(f"Logout result: {ret}") + + if ret == 0: + print("V BMC logout successful") + else: + print("X BMC logout failed") + + return ret == 0 + + +def test_get_bmc_login_user(bmc): + """Test get BMC login user API""" + print("\n=== Testing Get BMC Login User ===") + + try: + user = bmc.get_login_user() + print(f"V BMC login user: {user}") + return True + except Exception as e: + print(f"X Failed to get BMC login user: {e}") + return False + + +def test_get_bmc_ip_addr(bmc): + """Test get BMC IP address API""" + print("\n=== Testing Get BMC IP Address ===") + + try: + ip_addr = bmc.get_ip_addr() + print(f"V BMC IP address: {ip_addr}") + return True + except Exception as e: + print(f"X Failed to get BMC IP address: {e}") + return False + + +def test_get_bmc_eeprom_list(bmc): + """Test get BMC EEPROM list API""" + print("\n=== Testing Get BMC EEPROM List ===") + + try: + ret, eeprom_list = bmc.get_eeprom_list() + print(f"Get EEPROM list result: {ret}") + + if ret == 0: + print(f"V BMC EEPROM list retrieved successfully") + print(f"EEPROM entries: {len(eeprom_list)}") + for eeprom_id, eeprom_data in eeprom_list: + print(f" - EEPROM ID: {eeprom_id}") + print(f" Data: {eeprom_data}") + return True + else: + print(f"X Failed to get BMC EEPROM list: {eeprom_list}") + return False + except Exception as e: + print(f"X Exception getting BMC EEPROM list: {e}") + return False + + +def test_get_bmc_eeprom_info(bmc, eeprom_id): + """Test get BMC EEPROM info API""" + print("\n=== Testing Get BMC EEPROM Info ===") + + if not eeprom_id: + print("X No EEPROM ID provided, skipping get BMC EEPROM info test") + return False + + try: + print(f"Getting EEPROM info for ID: {eeprom_id}") + ret, eeprom_data = bmc.get_eeprom_info(eeprom_id) + print(f"Get EEPROM info result: {ret}") + + if ret == 0: + print(f"V BMC EEPROM info retrieved successfully") + print(f"EEPROM data: {eeprom_data}") + return True + else: + print(f"X Failed to get BMC EEPROM info: {eeprom_data}") + return False + except Exception as e: + print(f"X Exception getting BMC EEPROM info: {e}") + return False + + +def test_get_bmc_firmware_list(bmc): + """Test get BMC firmware list API""" + print("\n=== Testing Get BMC Firmware List ===") + + try: + ret, fw_list = bmc.get_firmware_list() + print(f"Get firmware list result: {ret}") + + if ret == 0: + print(f"V BMC firmware list retrieved successfully") + print(f"Firmware entries: {len(fw_list)}") + for fw_id, fw_version in fw_list: + print(f" - Firmware ID: {fw_id}") + print(f" Version: {fw_version}") + return True + else: + print(f"X Failed to get BMC firmware list: {fw_list}") + return False + except Exception as e: + print(f"X Exception getting BMC firmware list: {e}") + return False + + +def test_get_bmc_firmware_version(bmc, fw_id): + """Test get BMC firmware version API""" + print("\n=== Testing Get BMC Firmware Version ===") + + if not fw_id: + print("X No firmware ID provided, skipping get BMC firmware version test") + return False + + try: + print(f"Getting firmware version for ID: {fw_id}") + ret, version = bmc.get_firmware_version(fw_id) + print(f"Get firmware version result: {ret}") + + if ret == 0: + print(f"V BMC firmware version retrieved successfully") + print(f"Firmware version: {version}") + return True + else: + print(f"X Failed to get BMC firmware version: {version}") + return False + except Exception as e: + print(f"X Exception getting BMC firmware version: {e}") + return False + + +def test_trigger_bmc_debug_log_dump(bmc): + """Test trigger BMC debug log dump API""" + print("\n=== Testing Trigger BMC Debug Log Dump ===") + + try: + ret, (task_id, err_msg) = bmc.trigger_bmc_debug_log_dump() + print(f"Trigger result: {ret}") + + if ret == 0: + print(f"V BMC debug log dump triggered successfully") + print(f"Task ID: {task_id}") + return task_id + else: + print(f"X Failed to trigger BMC debug log dump: {err_msg}") + return None + except Exception as e: + print(f"X Exception triggering BMC debug log dump: {e}") + return None + + +def test_get_bmc_debug_log_dump(bmc, task_id): + """Test get BMC debug log dump API""" + print("\n=== Testing Get BMC Debug Log Dump ===") + + if not task_id: + print("X No task ID provided, skipping get BMC debug log dump test") + return False + + try: + temp_filename = f"bmc_debug_dump_{int(time.time())}.tar.xz" + temp_path = "/tmp" + + print(f"Attempting to get BMC debug log dump with task ID: {task_id}") + print(f"Target file: {temp_path}/{temp_filename}") + + ret, err_msg = bmc.get_bmc_debug_log_dump(task_id, temp_filename, temp_path) + print(f"Get dump result: {ret}") + + if ret == 0: + print(f"V BMC debug log dump retrieved successfully") + print(f"File saved to: {temp_path}/{temp_filename}") + + full_path = f"{temp_path}/{temp_filename}" + if os.path.exists(full_path): + file_size = os.path.getsize(full_path) + print(f"File size: {file_size} bytes") + else: + print("Warning: File not found after successful API call") + + return True + else: + print(f"X Failed to get BMC debug log dump: {err_msg}") + return False + except Exception as e: + print(f"X Exception getting BMC debug log dump: {e}") + return False + + +def test_reset_password(bmc): + """Test reset password API""" + print("\n=== Testing Reset Password ===") + + try: + print("Testing password reset for root user...") + ret = bmc.login() + if ret != 0: + print("Failed to login to BMC") + return False + user = 'root' + password = '0penBmcTempPass!' + + ret, msg = bmc.change_login_password(password, user) + print(f"Change password result: {ret}") + print(f"Message: {msg}") + + if ret == 0: + print("V Root password reset successful") + else: + print("X Root password reset failed") + + return ret == 0 + except Exception as e: + print(f"X Exception during password reset: {e}") + return False + + +def test_upgrade_bmc_firmware(bmc, fw_image, target=None, timeout=1800): + """Test BMC firmware upgrade API""" + print("\n=== Testing BMC Firmware Upgrade ===") + + if not os.path.exists(fw_image): + print(f"X Firmware image file not found: {fw_image}") + return False + + fw_ids = [] + if target: + fw_ids = [fw_id.strip() for fw_id in target.split(",")] + print(f'Flashing {fw_image} to {fw_ids}...') + else: + print(f'Flashing {fw_image} to BMC...') + + pbar = tqdm.tqdm(total=100) + + def create_progress_callback(): + last_percent = 0 + + def callback(percent): + nonlocal last_percent + delta = percent - last_percent + last_percent = percent + pbar.update(delta) + + return callback + + progress_callback = create_progress_callback() + + start = time.time() + + try: + if target: + ret, msg = bmc.update_firmware_on_component(fw_image, + fw_ids, + timeout=timeout, + progress_callback=progress_callback) + else: + ret, msg = bmc.update_firmware(fw_image, + timeout=timeout, + progress_callback=progress_callback) + + pbar.close() + + print(f'Time elapsed: {int((time.time() - start) * 10) / 10}s') + + if ret == 0: + print('V Firmware is successfully updated') + return True + else: + print(f'X Fail to update firmware. {msg}') + return False + except Exception as e: + pbar.close() + print(f'X Exception during firmware update: {e}') + return False + + +def run_api_test(bmc, api_name, **kwargs): + """Run a specific API test""" + print(f"\n{'=' * 60}") + print(f"TESTING API: {api_name.upper()}") + print(f"{'=' * 60}") + + api_tests = { + 'login': lambda bmc: test_bmc_login(bmc), + 'logout': test_bmc_logout, + 'get_user': test_get_bmc_login_user, + 'get_ip': test_get_bmc_ip_addr, + 'get_eeprom_list': test_get_bmc_eeprom_list, + 'get_eeprom_info': lambda bmc: test_get_bmc_eeprom_info(bmc, kwargs.get('eeprom_id')), + 'get_firmware_list': test_get_bmc_firmware_list, + 'get_firmware_version': lambda bmc: test_get_bmc_firmware_version(bmc, kwargs.get('fw_id')), + 'trigger_dump': test_trigger_bmc_debug_log_dump, + 'get_dump': lambda bmc: test_get_bmc_debug_log_dump(bmc, kwargs.get('task_id')), + 'reset_password': test_reset_password, + 'upgrade_firmware': lambda bmc: test_upgrade_bmc_firmware(bmc, kwargs.get('fw_image'), kwargs.get('target')), + } + + if api_name in api_tests: + return api_tests[api_name](bmc) + else: + print(f"X Unknown API test: {api_name}") + print("Available API tests: login, logout, get_user, get_ip, get_eeprom_list, get_eeprom_info, get_firmware_list, get_firmware_version, trigger_dump, get_dump, reset_password, upgrade_firmware") + return False + + +if __name__ == '__main__': + + if os.geteuid() != 0: + print('Please run under root privilege.') + sys.exit(-1) + + parser = argparse.ArgumentParser(description='BMC API Test Tool - Run one test at a time') + parser.add_argument("--test", choices=['login', 'logout', 'get_user', 'get_ip', 'get_eeprom_list', 'get_eeprom_info', 'get_firmware_list', 'get_firmware_version', 'trigger_dump', 'get_dump', 'reset_password', 'upgrade_firmware'], + required=True, help="Test a specific BMC API") + parser.add_argument("--task-id", help="Task ID for get_dump test") + parser.add_argument("--fw-image", help="Firmware image file for upgrade_firmware test") + parser.add_argument("--target", help="Target firmware IDs for upgrade (comma-separated)") + parser.add_argument("--eeprom-id", help="EEPROM ID for get_eeprom_info test") + parser.add_argument("--fw-id", help="Firmware ID for get_firmware_version test") + + args = parser.parse_args() + + chassis = sonic_platform.platform.Platform().get_chassis() + bmc = chassis.get_bmc() + + if bmc is None: + print('X No BMC exists') + sys.exit(0) + + bmc_ip = bmc.get_ip_addr() + print(f"BMC IP address: {bmc_ip}") + + if not is_host_reachable(bmc_ip, 10): + print(f'X BMC {bmc_ip} not reachable') + sys.exit(2) + + print(f'V BMC {bmc_ip} is reachable') + + if args.test == 'get_dump' and not args.task_id: + print("X --task-id is required for get_dump test") + sys.exit(1) + + if args.test == 'upgrade_firmware' and not args.fw_image: + print("X --fw-image is required for upgrade_firmware test") + sys.exit(1) + + if args.test == 'get_eeprom_info' and not args.eeprom_id: + print("X --eeprom-id is required for get_eeprom_info test") + sys.exit(1) + + if args.test == 'get_firmware_version' and not args.fw_id: + print("X --fw-id is required for get_firmware_version test") + sys.exit(1) + + kwargs = {} + if args.task_id: + kwargs['task_id'] = args.task_id + if args.fw_image: + kwargs['fw_image'] = args.fw_image + if args.target: + kwargs['target'] = args.target + if args.eeprom_id: + kwargs['eeprom_id'] = args.eeprom_id + if args.fw_id: + kwargs['fw_id'] = args.fw_id + + run_api_test(bmc, args.test, **kwargs) diff --git a/platform/mellanox/mlnx-platform-api/setup.py b/platform/mellanox/mlnx-platform-api/setup.py index cfa39aa2bf7..9e8d676cb2d 100644 --- a/platform/mellanox/mlnx-platform-api/setup.py +++ b/platform/mellanox/mlnx-platform-api/setup.py @@ -35,7 +35,8 @@ 'pytest-runner' ], install_requires= [ - 'inotify' + 'inotify', + 'filelock', ], tests_require = [ 'pytest', diff --git a/platform/mellanox/mlnx-platform-api/sonic_platform/bmc.py b/platform/mellanox/mlnx-platform-api/sonic_platform/bmc.py new file mode 100644 index 00000000000..433f4966331 --- /dev/null +++ b/platform/mellanox/mlnx-platform-api/sonic_platform/bmc.py @@ -0,0 +1,504 @@ +# +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################# +# Mellanox +# +# Module contains an implementation of new platform api +# +############################################################################# + + +try: + from functools import wraps + import os + import re + import subprocess + import json + from sonic_platform_base.bmc_base import BMCBase + from sonic_py_common import device_info + from sonic_py_common.logger import Logger + from .redfish_client import RedfishClient + from . import utils + import functools + import filelock +except ImportError as e: + raise ImportError (str(e) + "- required module not found") + +logger = Logger() + + +def under_lock(lockfile, timeout=2): + """ Execute operations under lock. """ + + def _under_lock(func): + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + with filelock.FileLock(lockfile, timeout): + return func(*args, **kwargs) + + return wrapped_function + return _under_lock + + +def with_credential_restore(api_func): + @wraps(api_func) + def wrapper(self, *args, **kwargs): + if self.rf_client is None: + raise Exception('Redfish instance initialization failure') + + if not self.rf_client.has_login(): + # No need to check login result here since we can't decide the + # correct type of return value of 'api_func()' for now. + # 'api_func()' will be executed regardless. If login fails, + # 'api_func()' will return code ERR_CODE_NOT_LOGIN, and the correct + # type of additional data. + self.login() + elif api_func.__name__ in ['update_firmware_on_component', 'update_firmware']: + # W/A for the case when we need to install new FW (takes quite long + # time). While this process, current token can be expired, so the code + # to pool task status will fail. To prevent it - invalidate token and + # create the new before running API to do FW update. + self.logout() + self.login() + + ret, data = api_func(self, *args, **kwargs) + if ret == RedfishClient.ERR_CODE_AUTH_FAILURE: + # Trigger credential restore flow + logger.log_notice(f'{api_func.__name__}() returns bad credential. ' \ + 'Trigger BMC TPM based password recovery flow') + restored = self._restore_tpm_credential() + if restored: + # Execute again + logger.log_notice(f'BMC TPM based password recovered. Retry {api_func.__name__}()') + ret, data = api_func(self, *args, **kwargs) + else: + logger.log_notice(f'Fail to recover BMC based password') + return (RedfishClient.ERR_CODE_AUTH_FAILURE, data) + return (ret, data) + return wrapper + + +class BMC(BMCBase): + + ''' + BMC encapsulates BMC device details such as IP address, credential management. + It also acts as wrapper of RedfishClient. + ''' + + CURL_PATH = '/usr/bin/curl' + BMC_ADMIN_ACCOUNT = 'admin' + BMC_ADMIN_ACCOUNT_DEFAULT_PASSWORD = '0penBmc' + BMC_NOS_ACCOUNT = 'yormnAnb' + BMC_NOS_ACCOUNT_DEFAULT_PASSWORD = "ABYX12#14artb51" + BMC_DIR = "/host/bmc" + MAX_LOGIN_ERROR_PROBE_CNT = 5 + # TODO(BMC): change nvos to sonic + BMC_TPM_HEX_FILE = "nvos_const.bin" + + _instance = None + + def __init__(self, addr): + + self.addr = addr + # Use the NOS account by default. If login fails with the NOS account + # then try with the admin account + self.using_nos_account = True + # Login password will be default password while doing TPM password + # recovery. This flag is used for Redfishclient callback to decide + # which password to use in login() + self.using_tpm_password = True + self.probe_cnt = 0 + + self.rf_client = RedfishClient(BMC.CURL_PATH, + addr, + self.get_login_user, + self.get_password_callback, + logger) + + @staticmethod + def get_instance(): + bmc_data = device_info.get_bmc_data() + if not bmc_data: + return None + + if BMC._instance is None: + BMC._instance = BMC(bmc_data['bmc_addr']) + + return BMC._instance + + # TODO(BMC): Implement the DeviceBase interface methods + + def get_ip_addr(self): + return self.addr + + def get_login_user(self): + if self.using_nos_account: + return BMC.BMC_NOS_ACCOUNT + return BMC.BMC_ADMIN_ACCOUNT + + # Password callback function passed to RedfishClient. + # It is not desired to store password in RedfishClient + # instance for security concern. + def get_password_callback(self): + if self.using_tpm_password: + return self.get_login_password() + else: + if self.using_nos_account: + return BMC.BMC_NOS_ACCOUNT_DEFAULT_PASSWORD + return BMC.BMC_ADMIN_ACCOUNT_DEFAULT_PASSWORD + + @under_lock(lockfile=f'{BMC_DIR}/{BMC_TPM_HEX_FILE}.lock', timeout=5) + def get_login_password(self): + try: + pass_len = 13 + attempt = 1 + max_attempts = 100 + max_repeat = int(3 + 0.09 * pass_len) + # TODO(BMC): change nvos to sonic + hex_data = "1300NVOS-BMC-USER-Const" + os.makedirs(self.BMC_DIR, exist_ok=True) + cmd = f'echo "{hex_data}" | xxd -r -p > {self.BMC_DIR}/{self.BMC_TPM_HEX_FILE}' + subprocess.run(cmd, shell=True, check=True) + + tpm_command = ["tpm2_createprimary", "-C", "o", "-u", f"{self.BMC_DIR}/{self.BMC_TPM_HEX_FILE}", "-G", "aes256cfb"] + result = subprocess.run(tpm_command, capture_output=True, check=True, text=True) + + while attempt <= max_attempts: + if attempt > 1: + # TODO(BMC): change nvos to sonic + const = f"1300NVOS-BMC-USER-Const-{attempt}" + mess = f"Password did not meet criteria; retrying with const: {const}" + logger.log_debug(mess) + tpm_command = f'echo -n "{const}" | tpm2_createprimary -C o -G aes -u -' + result = subprocess.run(tpm_command, shell=True, capture_output=True, check=True, text=True) + + symcipher_pattern = r"symcipher:\s+([\da-fA-F]+)" + symcipher_match = re.search(symcipher_pattern, result.stdout) + + if not symcipher_match: + raise Exception("Symmetric cipher not found in TPM output") + + # BMC dictates a password of 13 characters. Random from TPM is used with an append of A! + symcipher_part = symcipher_match.group(1)[:pass_len-2] + if symcipher_part.isdigit(): + symcipher_value = symcipher_part[:pass_len-3] + 'vA!' + elif symcipher_part.isalpha() and symcipher_part.islower(): + symcipher_value = symcipher_part[:pass_len-3] + '9A!' + else: + symcipher_value = symcipher_part + 'A!' + if len (symcipher_value) != pass_len: + raise Exception("Bad cipher length from TPM output") + + # check for monotonic + monotonic_check = True + for i in range(len(symcipher_value) - 3): + seq = symcipher_value[i:i+4] + increments = [ord(seq[j+1]) - ord(seq[j]) for j in range(3)] + if increments == [1, 1, 1] or increments == [-1, -1, -1]: + monotonic_check = False + break + + variety_check = len(set(symcipher_value)) >= 5 + repeating_pattern_check = sum(1 for i in range(pass_len - 1) if symcipher_value[i] == symcipher_value[i + 1]) <= max_repeat + + # check for consecutive_pairs + count = 0 + for i in range(11): + val1 = symcipher_value[i] + val2 = symcipher_value[i + 1] + if val2 == "v" or val1 == "v": + continue + if abs(int(val2, 16) - int(val1, 16)) == 1: + count += 1 + consecutive_pair_check = count <= 4 + + if consecutive_pair_check and variety_check and repeating_pattern_check and monotonic_check: + os.remove(f"{self.BMC_DIR}/{self.BMC_TPM_HEX_FILE}") + return symcipher_value + else: + attempt += 1 + + raise Exception("Failed to generate a valid password after maximum retries.") + + except subprocess.CalledProcessError as e: + logger.log_error(f"Error executing TPM command: {e}") + raise Exception("Failed to communicate with TPM") + + except Exception as e: + logger.log_error(f"Error: {e}") + raise + + def _restore_tpm_credential(self): + + logger.log_notice(f'Start BMC TPM password recovery flow') + + # We are not good with TPM based password here. + # Try to login with default password. + logger.log_notice(f'Try to login with BMC default password') + # Indicate password callback function to switch to BMC_ADMIN_ACCOUNT_DEFAULT_PASSWORD temporarily + self.using_tpm_password = False + ret = self.rf_client.login() + + if ret != RedfishClient.ERR_CODE_OK: + logger.log_error(f'Bad credential: Fail to login BMC with both TPM based and default passwords') + if self.probe_cnt < BMC.MAX_LOGIN_ERROR_PROBE_CNT: + # Need to log the exact failure reason since the /login REST API + # does not return anything. + # Trigger a GET request using user/password instead of token, then + # BMC will report the failure details. + self.rf_client.probe_login_error() + self.probe_cnt += 1 + # Resume to TPM password + self.using_tpm_password = True + return False + + # Indicate RedfishClient to switch to TPM password + self.using_tpm_password = True + + logger.log_notice(f'Login successfully with BMC default password') + try: + password = self.get_login_password() + except Exception as e: + self.rf_client.invalidate_login_token() + logger.log_error(f'Fail to get login password from TPM: {str(e)}') + return False + + logger.log_notice(f'Try to apply TPM based password to BMC') + ret, msg = self.change_login_password(password) + if ret != RedfishClient.ERR_CODE_OK: + self.rf_client.invalidate_login_token() + logger.log_error(f'Fail to apply TPM based password to BMC') + return False + + logger.log_notice(f'BMC password is restored successfully') + + # TPM based password has been restored. + return True + + def get_login_token(self): + if self.rf_client is None: + return None + + return self.rf_client.get_login_token() + + def get_component_list(self): + + # TBD: As the future improvement, the logic of loading all components + # (including non-BMC managed entities) can be implemented in a + # component manager from which BMC can retrieve relevant components. + # Each component is configured with its attributes and class in + # platform_components.json. Thus we can load the components in a + # generic manner. + + platform_path = device_info.get_path_to_platform_dir() + platform_components_json_path = \ + os.path.join(platform_path, 'platform_components.json') + comp_data = utils.load_json_file(platform_components_json_path) + + if not comp_data or len(comp_data.get('chassis', {})) == 0: + return [] + + if 'component' not in comp_data['chassis']: + return [] + + components = comp_data['chassis']['component'] + comp_list = [] + + for comp_name, attrs in components.items(): + # Skip if not managed by BMC + managed_by = attrs.get('managed_by', '') + if managed_by.upper() != 'BMC': + continue + + comp_cls = attrs.get('class', '') + if len(comp_cls) == 0: + logger.log_error(f"Missing 'class' for component {comp_name} in platform_components.json") + continue + + comp = None + from . import component as module + try: + cls = getattr(module, comp_cls) + if cls is None: + logger.log_error(f"Bad value 'class {comp_cls}' for component {comp_name} in platform_components.json") + continue + + comp = cls(comp_name, attrs) + except: + continue + comp_list.append(comp) + + # The reason why comp_list is not cached in BMC is the concern of circular + # reference with ComponentBMCObj. Anyway, future improvement will move + # component list management part to Chassis. Chassis holds component list + # references. + + return comp_list + + def get_component_by_name(self, name): + comp_list = list(filter(lambda comp: comp.name == name, self.get_component_list())) + return comp_list[0] if len(comp_list) > 0 else None + + def get_component_by_fw_id(self, fw_id): + comp_list = list(filter(lambda comp: comp.fw_id == fw_id, self.get_component_list())) + return comp_list[0] if len(comp_list) > 0 else None + + def get_component_list_by_type(self, type_name): + comp_list = list(filter(lambda comp: comp.type_name == type_name, self.get_component_list())) + return comp_list + + def try_login(self): + account = 'NOS' if self.using_nos_account else 'admin' + logger.log_notice(f'Try login to BMC using the {account} account') + + if self.rf_client is None: + return RedfishClient.ERR_CODE_AUTH_FAILURE + + ret = self.rf_client.login() + + if ret == RedfishClient.ERR_CODE_AUTH_FAILURE: + logger.log_notice(f'Fail to login BMC with TPM password. Trigger password recovery flow') + restored = self._restore_tpm_credential() + if restored: + ret = RedfishClient.ERR_CODE_OK + elif ret == RedfishClient.ERR_CODE_PASSWORD_UNAVAILABLE: + logger.log_notice(f'Fail to generate TPM password') + + return ret + + def login(self): + self.using_nos_account = True + ret = self.try_login() + if ret != RedfishClient.ERR_CODE_OK: + self.using_nos_account = False + ret = self.try_login() + return ret + + def logout(self): + if self.rf_client and self.rf_client.has_login(): + return self.rf_client.logout() + else: + return RedfishClient.ERR_CODE_OK + + def change_login_password(self, password, user=None): + if self.rf_client is None: + return (RedfishClient.ERR_CODE_AUTH_FAILURE, "") + + return self.rf_client.redfish_api_change_login_password(password, user) + + # TODO(BMC): Check if should call it in files/scripts/load_system_info + def check_and_reset_tpm_password_for_user(self, user: str = BMC_ADMIN_ACCOUNT) -> bool: + ''' + Check if the provided user has tpm password and if not, + generate a new tpm password and apply it to the user. + + Returns True if the TPM password is restored for the user or + user was already with TPM password, False otherwise. + ''' + self.using_nos_account = True if user == self.BMC_NOS_ACCOUNT else False + + if self.rf_client is None: + return False + + # By default, BMC password callback will use TPM password + (ret, response) = self.rf_client.probe_login_error() + if ret == RedfishClient.ERR_CODE_AUTH_FAILURE: + logger.log_notice(f'User {user} does not have TPM password, restore it') + if self._restore_tpm_credential(): + logger.log_notice(f'TPM password restored for user {user}') + return True + else: + logger.log_error(f'Fail to restore TPM password for user {user}') + return False + elif ret != RedfishClient.ERR_CODE_OK: + logger.log_error(f'Fail to check TPM password for user {user}: {response}') + return False + + return True + + @with_credential_restore + def get_firmware_list(self): + return self.rf_client.redfish_api_get_firmware_list() + + @with_credential_restore + def get_firmware_version(self, fw_id): + return self.rf_client.redfish_api_get_firmware_version(fw_id) + + @with_credential_restore + def get_eeprom_info(self, eeprom_id): + return self.rf_client.redfish_api_get_eeprom_info(eeprom_id) + + @with_credential_restore + def get_eeprom_list(self): + return self.rf_client.redfish_api_get_eeprom_list() + + @with_credential_restore + def update_firmware(self, fw_image, timeout = 1800, progress_callback = None): + logger.log_notice(f'Installing firmware image {fw_image} via BMC') + ret, msg = self.rf_client.redfish_api_update_firmware(fw_image, timeout, progress_callback) + logger.log_notice(f'Firmware update result: {ret}') + if ret: + logger.log_notice(f'{msg}') + + return (ret, msg) + + @with_credential_restore + def update_firmware_on_component(self, fw_image, fw_ids, timeout = 1800, progress_callback = None): + # Set component ID to be updated + logger.log_notice(f'Set BMC update targets: {fw_ids}') + ret, msg = self.rf_client.redfish_api_set_component_update(fw_ids) + if ret != RedfishClient.ERR_CODE_OK: + return (ret, 'Fail to set Component ID attribute') + + logger.log_notice(f'Installing firmware image {fw_image} via BMC') + ret, msg = self.rf_client.redfish_api_update_firmware(fw_image, timeout, progress_callback) + logger.log_notice(f'Firmware update result: {ret}') + if ret: + logger.log_notice(f'{msg}') + + # Reset component ID from to be updated + logger.log_notice(f'Clear BMC update targets') + _, _ = self.rf_client.redfish_api_set_component_update(None) + + return (ret, msg) + + @with_credential_restore + def trigger_bmc_debug_log_dump(self): + return self.rf_client.redfish_api_trigger_bmc_debug_log_dump() + + @with_credential_restore + def get_bmc_debug_log_dump(self, task_id, filename, path): + return self.rf_client.redfish_api_get_bmc_debug_log_dump(task_id, filename, path) + + # TODO(BMC): Verify which functions are needed for BMC + + # @with_credential_restore + # def get_erot_copy_background_status(self, erot_component_id): + # return self.rf_client.redfish_api_get_erot_copy_background_status(erot_component_id) + + # @with_credential_restore + # def get_erots_debug_token_status(self): + # return self.rf_client.redfish_api_get_debug_token_status() + + # @with_credential_restore + # def get_erot_active_and_inactive_flashes(self, erot_id): + # return self.rf_client.redfish_api_get_erot_active_and_inactive_flashes(erot_id) + + # @with_credential_restore + # def get_erot_ap_boot_status(self, erot_id): + # return self.rf_client.redfish_api_get_erot_ap_boot_status(erot_id) diff --git a/platform/mellanox/mlnx-platform-api/sonic_platform/chassis.py b/platform/mellanox/mlnx-platform-api/sonic_platform/chassis.py index 495be1e5a6d..a47c7f1c968 100644 --- a/platform/mellanox/mlnx-platform-api/sonic_platform/chassis.py +++ b/platform/mellanox/mlnx-platform-api/sonic_platform/chassis.py @@ -23,9 +23,11 @@ # ############################################################################# + try: from sonic_platform_base.chassis_base import ChassisBase from sonic_py_common.logger import Logger + from sonic_py_common.device_info import get_bmc_data import os from sonic_py_common import device_info from functools import reduce @@ -33,6 +35,7 @@ from . import module_host_mgmt_initializer from . import utils from .device_data import DeviceDataManager + from .bmc import BMC import re import select import threading @@ -126,6 +129,10 @@ def __init__(self): self.poll_obj = None self.registered_fds = None + self._bmc = None + self._bmc_data = None + self._bmc_initialized = False + logger.log_info("Chassis loaded successfully") def __del__(self): @@ -802,6 +809,11 @@ def initialize_components(self): self._component_list.append(ComponentSSD()) self._component_list.append(DeviceDataManager.get_bios_component()) self._component_list.extend(DeviceDataManager.get_cpld_component_list()) + + # Initialize BMC and its components + self.initialize_bmc() + + # TODO(BMC): verify if need to self.initialize_fw_dir() def get_num_components(self): """ @@ -1085,6 +1097,35 @@ def is_replaceable(self): """ return False + def initialize_bmc(self): + if self._bmc_initialized: + return + + self._bmc_data = get_bmc_data() + + if self._bmc_data: + self._bmc = BMC.get_instance() + + try: + bmc_comp_list = self._bmc.get_component_list() + self._component_list.extend(bmc_comp_list) + except Exception as e: + logger.log_error("Fail to get BMC component list") + + self._bmc_initialized = True + + def _initialize_bmc(self): + self.initialize_components() + self.initialize_bmc() + + def get_bmc_data(self): + self._initialize_bmc() + return self._bmc_data + + def get_bmc(self): + self._initialize_bmc() + return self._bmc + class ModularChassis(Chassis): def __init__(self): diff --git a/platform/mellanox/mlnx-platform-api/sonic_platform/component.py b/platform/mellanox/mlnx-platform-api/sonic_platform/component.py index f1ce4d8b1cc..b2fbc84d069 100644 --- a/platform/mellanox/mlnx-platform-api/sonic_platform/component.py +++ b/platform/mellanox/mlnx-platform-api/sonic_platform/component.py @@ -909,6 +909,21 @@ def _install_firmware(self, image_path): return True + +# TODO(BMC): Verify which classes and functions are needed for BMC components +# And check the usage on them in load_system_info and platform_components.json + +# class ComponentBMCObj(Component) + +# class ComponentBMC(ComponentBMCObj) + +# class ComponentFPGAOnBMC(ComponentBMCObj) + +# class ComponentERoTOnBMC(ComponentBMCObj) + +# class ComponentBIOSOnBMC(ComponentBMCObj) + + class ComponentCPLDSN4280(ComponentCPLD): CPLD_FIRMWARE_UPDATE_COMMAND = ['cpldupdate', '--gpio', '--print-progress', ''] diff --git a/platform/mellanox/mlnx-platform-api/sonic_platform/redfish_client.py b/platform/mellanox/mlnx-platform-api/sonic_platform/redfish_client.py new file mode 100644 index 00000000000..d62abd3fbf2 --- /dev/null +++ b/platform/mellanox/mlnx-platform-api/sonic_platform/redfish_client.py @@ -0,0 +1,1412 @@ +# +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################# +# Mellanox +# +# Module contains an implementation of RedFish client which provides +# firmware upgrade and sensor retrieval functionality +# +############################################################################# + + +import subprocess +import json +import time +import re +import shlex +# TODO(BMC): Verify if pydash is needed according to the commented functions below +# import pydash as py_ + + +''' +A stub logger class which prints log message to screen. +It can be used for debugging standalone file. +''' +class ConsoleLogger: + def __getattr__(self, name): + # Intercept calls to methods that start with 'log_' + supported_methods = ['log_error', + 'log_warning', + 'log_notice', + 'log_info', + 'log_debug'] + if name in supported_methods: + def method(*args, **kwargs): + print(*args, **kwargs) + return method + + # Raise an AttributeError for other methods + err_msg = f"'{self.__class__.__name__}' object has no attribute '{name}'" + raise AttributeError(err_msg) + + +def is_auth_failure(http_status_code): + return (http_status_code == '401') + + +''' +cURL wrapper for Redfish client access +''' +class RedfishClient: + + DEFAULT_TIMEOUT = 3 + DEFAULT_LOGIN_TIMEOUT = 4 + + # Redfish URIs + REDFISH_URI_FW_INVENTORY = '/redfish/v1/UpdateService/FirmwareInventory' + REDFISH_URI_CHASSIS_INVENTORY = '/redfish/v1/Chassis' + REDFISH_URI_TASKS = '/redfish/v1/TaskService/Tasks' + REDFISH_URI_UPDATE_SERVICE = '/redfish/v1/UpdateService' + REDFISH_URI_ACCOUNTS = '/redfish/v1/AccountService/Accounts' + REDFISH_DEBUG_TOKEN = '/redfish/v1/Systems/System_0/LogServices/DebugTokenService' + REDFISH_BMC_LOG_DUMP = '/redfish/v1/Managers/BMC_0/LogServices/Dump/Actions' + + REDFISH_URI_CHASSIS = '/redfish/v1/Chassis' + + # Error code definitions + ERR_CODE_OK = 0 + ERR_CODE_AUTH_FAILURE = -1 + ERR_CODE_INVALID_JSON_FORMAT = -2 + ERR_CODE_UNEXPECTED_RESPONSE = -3 + ERR_CODE_CURL_FAILURE = -4 + ERR_CODE_NOT_LOGIN = -5 + ERR_CODE_TIMEOUT = -6 + ERR_CODE_IDENTICAL_IMAGE = -7 + ERR_CODE_PASSWORD_UNAVAILABLE = -8 + ERR_CODE_URI_NOT_FOUND = -9 + ERR_CODE_SERVER_UNREACHABLE = -10 + ERR_CODE_GENERIC_ERROR = -11 + + CURL_ERR_OK = 0 + CURL_ERR_OPERATION_TIMEDOUT = 28 + CURL_ERR_COULDNT_RESOLVE_HOST = 6 + CURL_ERR_FAILED_CONNECT_TO_HOST = 7 + CURL_ERR_SSL_CONNECT_ERROR = 35 + + CURL_TO_REDFISH_ERROR_MAP = \ + { + CURL_ERR_COULDNT_RESOLVE_HOST : ERR_CODE_SERVER_UNREACHABLE, + CURL_ERR_FAILED_CONNECT_TO_HOST : ERR_CODE_SERVER_UNREACHABLE, + CURL_ERR_SSL_CONNECT_ERROR : ERR_CODE_SERVER_UNREACHABLE, + CURL_ERR_OPERATION_TIMEDOUT : ERR_CODE_TIMEOUT, + CURL_ERR_OK : ERR_CODE_OK + } + + ''' + Constructor + A password_callback parameter is provoided because: + 1. Password is not allowed to be saved for security concern. + 2. If token expires or becomes invalid for some reason (for example, being + revoked from BMC web interface), RedfishClient will do login retry in which + password is required anyway. It will get password from an external password + provider, for example class BMC which holds the responsibility of generating + password from TPM. + ''' + def __init__(self, curl_path, ip_addr, user_callback, password_callback, logger = None): + self.__curl_path = curl_path + self.__svr_ip = ip_addr + self.__user_callback = user_callback + self.__password_callback = password_callback + self.__token = None + self.__default_timeout = RedfishClient.DEFAULT_TIMEOUT + self.__default_login_timeout = RedfishClient.DEFAULT_LOGIN_TIMEOUT + if logger is None: + self.__logger = ConsoleLogger() + else: + self.__logger = logger + + self.__logger.log_notice(f'RedfishClient instance is created\n') + + def get_login_token(self): + return self.__token + + def curl_errors_to_redfish_erros_translation(self, curl_error): + return self.CURL_TO_REDFISH_ERROR_MAP.get( + curl_error, RedfishClient.ERR_CODE_CURL_FAILURE) + + def invalidate_login_token(self): + self.__logger.log_notice(f'Invalidate login token') + self.__token = None + + ''' + Build the POST command to login and get bearer token + ''' + def __build_login_cmd(self, password): + user = self.__user_callback() + cmd = f'{self.__curl_path} -m {self.__default_login_timeout} -k ' \ + f'-H "Content-Type: application/json" ' \ + f'-X POST https://{self.__svr_ip}/login ' \ + f'-d \'{{"username" : "{user}", "password" : "{password}"}}\'' + return cmd + + ''' + Build the POST command to logout and release the token + ''' + def __build_logout_cmd(self): + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-X POST https://{self.__svr_ip}/logout' + + return cmd + + ''' + Build the GET command + ''' + def __build_get_cmd(self, uri, output_file = None): + output_str = '' if not output_file else f'--output {output_file}' + cmd = f'{self.__curl_path} -m {self.__default_timeout} -k ' \ + f'-H "X-Auth-Token: {self.__token}" --request GET ' \ + f'--location https://{self.__svr_ip}{uri} ' \ + f'{output_str}' + return cmd + + ''' + Build a GET command using user/password to probe login account error + ''' + def __build_login_probe_cmd(self): + uri = RedfishClient.REDFISH_URI_ACCOUNTS + user = self.__user_callback() + password = self.__password_callback() + cmd = f'{self.__curl_path} -m {self.__default_timeout} -k ' \ + f'-u {user}:{password} --request GET ' \ + f'--location https://{self.__svr_ip}{uri} ' + return cmd + + ''' + Build the POST command to do firmware upgdate + ''' + def __build_fw_update_cmd(self, fw_image): + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-H "Content-Type: application/octet-stream" -X POST ' \ + f'https://{self.__svr_ip}' \ + f'{RedfishClient.REDFISH_URI_UPDATE_SERVICE} -T {fw_image}' + return cmd + + ''' + Build the PATCH command to change login password + ''' + def __build_change_password_cmd(self, new_password, user): + if user is None: + user = self.__user_callback() + + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-H "Content-Type: application/json" -X PATCH ' \ + f'https://{self.__svr_ip}' \ + f'{RedfishClient.REDFISH_URI_ACCOUNTS}/{user} ' \ + f'-d \'{{"Password" : "{new_password}"}}\'' + return cmd + + ''' + Build the PATCH command to set component attribute to update FW + ''' + def __build_set_component_update_cmd(self, comps): + comps_uris = [f'"{RedfishClient.REDFISH_URI_FW_INVENTORY}/{comp}"' for comp in comps] + comps_uris_str = ', '.join(comps_uris) + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-X PATCH -d \'{{"HttpPushUriTargets":['\ + f'{comps_uris_str}'\ + f']}}\' ' \ + f'https://{self.__svr_ip}' \ + f'{RedfishClient.REDFISH_URI_UPDATE_SERVICE}' + return cmd + + ''' + Build the PATCH command to reset component attribute to update FW + ''' + def __build_set_component_update_reset_cmd(self): + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-X PATCH -d \'{{"HttpPushUriTargets":[]}}\' ' \ + f'https://{self.__svr_ip}' \ + f'{RedfishClient.REDFISH_URI_UPDATE_SERVICE}' + return cmd + + # TODO(BMC): Verify if this function is needed according to the commented functions below + # ''' + # Build the POST command to start debug toke request + # ''' + # def __build_debug_token_cmd(self, debug_token_status=False): + # data_type = "DebugTokenStatus" if debug_token_status else "GetDebugTokenRequest" + # cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + # f'-H "Content-Type: application/json" ' \ + # f'-X POST https://{self.__svr_ip}' \ + # f'{RedfishClient.REDFISH_DEBUG_TOKEN}/LogService.CollectDiagnosticData ' \ + # f'-d \'{{"DiagnosticDataType":"OEM", "OEMDiagnosticDataType":"{data_type}"}}\'' + # return cmd + + ''' + Build the POST command to start BMC debug dump request Redfish Task + ''' + def __build_bmc_debug_log_dump_cmd(self): + cmd = f'{self.__curl_path} -k -H "X-Auth-Token: {self.__token}" ' \ + f'-H "Content-Type: application/json" ' \ + f'-X POST https://{self.__svr_ip}' \ + f'{RedfishClient.REDFISH_BMC_LOG_DUMP}/LogService.CollectDiagnosticData ' \ + '-d \'{"DiagnosticDataType":"Manager"}\'' + return cmd + + ''' + Obfuscate username and password while asking for bearer token + ''' + def __obfuscate_user_password(self, cmd): + # Obfuscate 'username' and 'password' in the payload + # For example: login + pattern = r'"username" : "[^"]*", "password" : "[^"]*"' + replacement = '"username" : "******", "password" : "******"' + obfuscation_cmd = re.sub(pattern, replacement, cmd) + + # Obfuscate username and password in the command line parameter + # For example: use user:password directly in the command to do + # login failure probe + pattern = r'-u [!-~]+:[!-~]+' + replacement = '-u ******:******' + obfuscation_cmd = re.sub(pattern, replacement, obfuscation_cmd) + + return obfuscation_cmd + + ''' + Obfuscate bearer token in the response string + ''' + def __obfuscate_token_response(self, response): + # Credential obfuscation + pattern = r'"token": "[^"]*"' + replacement = '"token": "******"' + obfuscation_response = re.sub(pattern, + replacement, + response) + return obfuscation_response + + ''' + Obfuscate bearer token passed to cURL + ''' + def __obfuscate_auth_token(self, cmd): + pattern = r'X-Auth-Token: [^"]+' + replacement = 'X-Auth-Token: ******' + + obfuscation_cmd = re.sub(pattern, replacement, cmd) + return obfuscation_cmd + + ''' + Obfuscate password while aksing for password change + ''' + def __obfuscate_password(self, cmd): + pattern = r'"Password" : "[^"]*"' + replacement = '"Password" : "******"' + obfuscation_cmd = re.sub(pattern, replacement, cmd) + + return obfuscation_cmd + + ''' + Parse cURL output to extract response and HTTP status code + Return value: + Tuple of JSON response and HTTP status code + ''' + def __parse_curl_output(self, curl_output): + response_str = None + http_status_code = '000' + + pattern = r'([\s\S]*?)\nHTTP Status Code: (\d+)$' + match = re.search(pattern, curl_output, re.MULTILINE) + + if match: + response_str = match.group(1) # The JSON part + http_status_code = match.group(2) # The HTTP status code + else: + # Unlikely to happen. Bug of cURL + self.__logger.log_error(f'Unexpected curl output: {curl_output}\n') + + # response_str 'None' means format error + return (response_str, http_status_code) + + ''' + Execute cURL command and return the output and error messages + Return value: + ERR_CODE_OK + ERR_CODE_TIMEOUT + ERR_CODE_CURL_FAILURE + ''' + def __exec_curl_cmd_internal(self, cmd): + # Will not print task monitor to syslog + task_mon = (RedfishClient.REDFISH_URI_TASKS in cmd) + login_cmd = ('/login ' in cmd) + password_change = (RedfishClient.REDFISH_URI_ACCOUNTS in cmd) + print_to_syslog = not isinstance(self.__logger, ConsoleLogger) + + # Credential obfuscation + obfuscation_cmd = self.__obfuscate_user_password(cmd) + obfuscation_cmd = self.__obfuscate_auth_token(obfuscation_cmd) + + if password_change: + obfuscation_cmd = self.__obfuscate_password(obfuscation_cmd) + + # For syslog, skip logs for task monitor requests + # since there are too many + if print_to_syslog: + if not task_mon: + self.__logger.log_debug(obfuscation_cmd + '\n') + else: + self.__logger.log_debug(cmd + '\n') + + # Instruct curl to append HTTP status code after JSON response + cmd += ' -w "\nHTTP Status Code: %{http_code}"' + process = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = process.communicate() + output_str, http_status_code = self.__parse_curl_output(output.decode('utf-8')) + error_str = error.decode('utf-8') + ret = process.returncode + + if http_status_code != '200': + self.__logger.log_notice(f'HTTP status code {http_status_code}, output {output_str}, error {error_str}') + + # No HTTP status code found + if http_status_code is None: + ret = RedfishClient.ERR_CODE_CURL_FAILURE + error_str = 'Unexpected curl output' + return (ret, http_status_code, output_str, error_str, obfuscation_cmd) + + if (ret == RedfishClient.CURL_ERR_OK): # cURL retuns ok + ret = RedfishClient.ERR_CODE_OK + + if login_cmd: + obfuscation_output_str = \ + self.__obfuscate_token_response(output_str) + else: + obfuscation_output_str = output_str + + # For syslog, skip logs for task monitor responses + # except the last one since there are too many + if print_to_syslog: + if not task_mon: + msg = f'Output:\n{obfuscation_output_str}\n' + self.__logger.log_debug(msg) + else: + complete_str = '"PercentComplete": 100' + task_complete = (complete_str in obfuscation_output_str) + if task_complete: + self.__logger.log_notice(obfuscation_cmd + '\n') + msg = f'Output:\n{obfuscation_output_str}\n' + self.__logger.log_notice(msg) + else: + msg = f'Output:\n{output_str}\n' + self.__logger.log_debug(msg) + else: # cURL returns error + cmd_to_log = obfuscation_cmd if print_to_syslog else cmd + self.__logger.log_notice(f'curl error on executing command: {cmd_to_log}') + self.__logger.log_notice(f'Error: {error_str}') + + ret = self.curl_errors_to_redfish_erros_translation(ret) + + return (ret, http_status_code, output_str, error_str, obfuscation_cmd) + + ''' + Extract URI from the job response + + Example of Payload: + "Payload": { + "HttpHeaders": [ + "Host: 10.0.1.1", + "User-Agent: curl/7.74.0", + "Accept: */*", + "Content-Length: 76", + "Location: /redfish/v1/Systems/System_0/LogServices/DebugTokenService/Entries/0/attachment" + ], + "HttpOperation": "POST", + "JsonBody": "{\n \"DiagnosticDataType\": \"OEM\",\n \"OEMDiagnosticDataType\": \"GetDebugTokenRequest\"\n}", + "TargetUri": "/redfish/v1/Systems/System_0/LogServices/DebugTokenService/LogService.CollectDiagnosticData" + } + ''' + def __get_uri_from_response(self, response): + try: + json_response = json.loads(response) + except Exception as e: + msg = 'Error: Invalid JSON format' + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, msg, None) + + if "Payload" not in json_response: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Error: Missing 'Payload' field" + return (ret, err_msg, None) + + payload = json_response["Payload"] + if "HttpHeaders" not in payload: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Error: Missing 'HttpHeaders' field" + return (ret, err_msg, None) + + http_headers = payload["HttpHeaders"] + uri = None + for header in http_headers: + if "Location" in header: + uri = header.split()[-1] + + if not uri: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Error: Missing 'Location' field" + return (ret, err_msg, None) + + return (RedfishClient.ERR_CODE_OK, "", uri) + + ''' + Log json response + ''' + def __log_json_response(self, response): + lines = response.splitlines() + for line in lines: + self.__logger.log_notice(line) + + ''' + Replace old token in the command. + This happens in case token becomes invalid and re-login is triggered. + ''' + def __update_token_in_command(self, cmd): + pattern = r'X-Auth-Token:\s*[^\s\"\']+' + new_cmd = re.sub(pattern, 'X-Auth-Token: ' + self.__token, cmd) + + return new_cmd + + ''' + Wrapper function to execute the given cURL command which can deal with + invalid bearer token case. + Return value: + ERR_CODE_OK + ERR_CODE_NOT_LOGIN + ERR_CODE_TIMEOUT + ERR_CODE_CURL_FAILURE + ERR_CODE_AUTH_FAILURE + ''' + def exec_curl_cmd(self, cmd, max_retries=2): + is_login_cmd = ('/login ' in cmd) + + # Not login, return + if (not self.has_login()) and (not is_login_cmd): + self.__logger.log_error('Need to login first before executing curl command\n') + return (RedfishClient.ERR_CODE_NOT_LOGIN, 'Not login', 'Not login') + + ret, http_status_code, output_str, error_str, obfuscation_cmd \ + = self.__exec_curl_cmd_internal(cmd) + + # cURL execution timeout, try again + i = 0 + while (i < max_retries) and (ret == RedfishClient.ERR_CODE_TIMEOUT): + + # TBD: + # Add rechability test (interface down/no ip) here. + # If unreachable, no need to retry. Set unreachable flat at meanwhile. + # If this flag is set, exectute_curl_cmd() needs to do reachablity test + # before executing curl command. Then it avoids getting stuck in curl + # until timeout. The flag will be reset once we have a successful curl + # command executed. + + # Increase timeout temporarily + timeout = None + match = re.search(r'-m\s*(\d+)', cmd) + if match: + timeout = int(match.group(1)) + timeout += 2 + cmd = re.sub(r'-m\s*\d+', f'-m {timeout}', cmd) + + msg = f"exec '{cmd}' (retry_number={i}" + f" timeout={timeout}s)" if timeout else ")" + self.__logger.log_debug(msg + '\n') + + ret, http_status_code, output_str, error_str, obfuscation_cmd \ + = self.__exec_curl_cmd_internal(cmd) + + i += 1 + + # Authentication failure might happen in case of: + # - Incorrect password + # - Invalid token (Token may become invalid for some reason. + # For example, remote side may clear the session table or change password. + # - Account locked + if not is_auth_failure(http_status_code): + return (ret, output_str, error_str) + + # Authentication failure on login, report error. + if is_login_cmd: + return (RedfishClient.ERR_CODE_AUTH_FAILURE, 'Authentication failure', 'Authentication failed') + + # Authentication failure for other commands. + # We can't differentiate various scenarios that may cause authentication failure. + # Just do a re-login and retry the command and expect to recover. + self.__logger.log_notice(f"Got HTTP status code '401' response: {obfuscation_cmd}\n") + self.__logger.log_notice(f'Re-login and retry last command...\n') + self.invalidate_login_token() + ret = self.login() + if ret == RedfishClient.ERR_CODE_OK: + self.__logger.log_notice(f'Login successfully. Rerun last command\n') + cmd = self.__update_token_in_command(cmd) + ret, http_status_code, output_str, error_str, _ = self.__exec_curl_cmd_internal(cmd) + if ret != RedfishClient.ERR_CODE_OK: + self.__logger.log_notice(f'Command rerun returns error {ret}\n') + elif is_auth_failure(http_status_code): + self.__logger.log_notice(f'Command rerun fails as authentication failure\n') + self.invalidate_login_token() + ret = RedfishClient.ERR_CODE_AUTH_FAILURE + output_str = error_str = 'Authentication failure' + return (ret, output_str, error_str) + elif ret == RedfishClient.ERR_CODE_AUTH_FAILURE: + # Login fails, invalidate token. + self.__logger.log_notice(f'Failed to login. Return as authentication failure\n') + self.invalidate_login_token() + return (ret, 'Authentication failure', 'Authentication failure') + else: + # Login fails, invalidate token. + self.__logger.log_notice(f'Failed to login, error : {ret}\n') + self.invalidate_login_token() + return (ret, 'Login failure', 'Login failure') + + ''' + Check if already login + ''' + def has_login(self): + return self.__token is not None + + ''' + Login Redfish server and get bearer token + ''' + def login(self): + if self.has_login(): + return RedfishClient.ERR_CODE_OK + + try: + password = self.__password_callback() + except Exception as e: + self.__logger.log_error(f'{str(e)}') + return RedfishClient.ERR_CODE_PASSWORD_UNAVAILABLE + + cmd = self.__build_login_cmd(password) + ret, response, error = self.exec_curl_cmd(cmd) + + if (ret != 0): + msg = f'Login failure: code {ret}, {error}\n' + self.__logger.log_error(msg) + return ret + + if len(response) == 0: + msg = 'Got empty Redfish login response.\n' + self.__logger.log_error(msg) + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + return ret + + try: + json_response = json.loads(response) + if 'error' in json_response: + msg = json_response['error']['message'] + self.__logger.log_error(f'{msg}\n') + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + elif 'token' in json_response: + token = json_response['token'] + if token is not None: + ret = RedfishClient.ERR_CODE_OK + self.__token = token + self.__logger.log_notice('Redfish login successfully and session token updated') + else: + msg = 'Login failure: empty "token" field found\n' + self.__logger.log_error(msg) + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + else: + msg = 'Login failure: no "token" field found\n' + self.__logger.log_error(msg) + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + except Exception as e: + msg = 'Login failure: invalid json format\n' + self.__logger.log_error(msg) + self.__logger.log_json_response(response) + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + + return ret + + ''' + Logout Redfish server + ''' + def logout(self): + if not self.has_login(): + return RedfishClient.ERR_CODE_OK + + self.__logger.log_notice(f'Logout redfish session\n') + + cmd = self.__build_logout_cmd() + ret, response, error = self.exec_curl_cmd(cmd) + + # Invalidate token anyway + self.__token = None + + if (ret != 0): # cURL execution error + msg = 'Logout failure: curl command returns error\n' + self.__logger.log_notice(msg) + return ret + + if len(response) == 0: # Invalid token + msg = 'Got empty Redfish logout response. It indicates an invalid token\n' + self.__logger.log_notice(msg) + return ret + + try: + json_response = json.loads(response) + + if 'status' in json_response: + status = json_response['status'] + if status != 'ok': + self.__logger.log_notice(f'Redfish response for logout failure: \n') + self.__log_json_response(response) + except Exception as e: + msg = 'Logout failure: invalid json format\n' + self.__logger.log_error(msg) + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + + return ret + + ''' + Use GET command with user/password to probe the exact error reason in case + of login failure + ''' + def probe_login_error(self): + cmd = self.__build_login_probe_cmd() + ret, _, response, error, _ = self.__exec_curl_cmd_internal(cmd) + + if (ret != 0): # cURL execution error, + msg = 'Probe login failure: curl command returns error\n' + self.__logger.log_notice(msg) + return (RedfishClient.ERR_CODE_GENERIC_ERROR, response) + + if len(response) == 0: + msg = 'Got empty response.\n' + self.__logger.log_notice(msg) + return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, msg) + + try: + json_response = json.loads(response) + except Exception as e: + msg = 'Probe login failure: invalid json format\n' + self.__logger.log_error(msg) + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, msg) + + if 'error' in json_response: # Error found + # Log just in case of error response + self.__logger.log_notice(f'Redfish response for login failure probe: \n') + self.__log_json_response(response) + + err = json_response['error'] + if 'code' in err: + err_code = err['code'] + if 'ResourceAtUriUnauthorized' in err_code: + ret = RedfishClient.ERR_CODE_AUTH_FAILURE + err_msg = "Account is locked or wrong password" + else: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = f"Not expected error code: {err_code}" + else: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Missing 'error code' field" + + return (ret, f'Error: {err_msg}') + + return (RedfishClient.ERR_CODE_OK, response) + + + ''' + Get firmware inventory + + Parameters: None + Return value: (ret, firmware_list) + ret return code + firmware_list list of tuple (fw_id, version) + ''' + def redfish_api_get_firmware_list(self): + cmd = self.__build_get_cmd(RedfishClient.REDFISH_URI_FW_INVENTORY) + ret, response, error = self.exec_curl_cmd(cmd) + + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, []) + + try: + json_response = json.loads(response) + item_list = json_response["Members"] + except json.JSONDecodeError as e: + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, []) + except Exception as e: + return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, []) + + fw_list = [] + for item in item_list: + fw_id = item["@odata.id"].split('/')[-1] + + ret, version = self.redfish_api_get_firmware_version(fw_id) + if (ret != RedfishClient.ERR_CODE_OK): + version = "N/A" + + fw_list.append((fw_id, version)) + + return (RedfishClient.ERR_CODE_OK, fw_list) + + + ''' + Get firmware version by given ID + + Parameters: + fw_id firmware ID + Return value: (ret, version) + ret return code + version firmware version string + ''' + def redfish_api_get_firmware_version(self, fw_id): + version = 'N/A' + + uri = f'{RedfishClient.REDFISH_URI_FW_INVENTORY}/{fw_id}' + cmd = self.__build_get_cmd(uri) + ret, response, error_msg = self.exec_curl_cmd(cmd) + + if (ret == RedfishClient.ERR_CODE_OK): + try: + json_response = json.loads(response) + if 'Version' in json_response: + version = json_response['Version'] + else: + msg = 'Error: Version not found in Redfish response\n' + self.__logger.log_error(msg) + self.__log_json_response(response) + except json.JSONDecodeError as e: + msg = f'Error: Invalid Redfish response JSON format on querying {fw_id} version\n' + self.__logger.log_notice(msg) + self.__log_json_response(response) + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + except Exception as e: + msg = f'Error: Exception {str(e)} caught on querying {fw_id} version\n' + self.__logger.log_notice(msg) + self.__log_json_response(response) + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + else: + msg = f'Got error {ret} on querying {fw_id} version: {error_msg}\n' + self.__logger.log_notice(msg) + + return (ret, version) + + + ''' + Update firmware + + Parameters: + fw_image firmware image path + timeout timeout value in seconds + Return value: (ret, error_msg) + ret return code + error_msg error message string + ''' + def redfish_api_update_firmware(self, fw_image, timeout = 1800, progress_callback = None): + # Trigger FW upgrade + cmd = self.__build_fw_update_cmd(fw_image) + ret, response, error_msg = self.exec_curl_cmd(cmd) + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, f'Error: {error_msg}') + + try: + json_response = json.loads(response) + except Exception as e: + msg = 'Error: Invalid JSON format' + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, msg) + + # Retrieve task id from response + task_id = '' + if 'error' in json_response: # Error found + err = json_response['error'] + if 'message' in err: + err_msg = err['message'] + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + else: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Missing 'message' field" + return (ret, f'Error: {err_msg}') + elif 'TaskStatus' in json_response: + status = json_response['TaskStatus'] + if status == 'OK': + task_id = json_response['Id'] + else: + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, f'Error: Return status is {status}') + + # Wait for completion + ret, error_msg, _ = self.__wait_task_completion(task_id, timeout, progress_callback) + + return (ret, error_msg) + + + ''' + Common function for both debug token info and debug token status APIs. + It receives the response, parse it, wait for completion and extract + URI with the path to result and return it. + Parameters: + response - JSON response from request command + timeout - in seconds, how long to wait for task completion + Return (ret_code, ret_msg or URI) + ret_code - returned error code + ret_msg - returned error message + URI - path to take the results after task execution + ''' + def _get_debug_token_responce(self, response, timeout): + try: + json_response = json.loads(response) + except Exception as e: + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, 'Error: Invalid JSON format') + + # Retrieve task id from response + task_id = '' + if 'error' in json_response: # Error found + err = json_response['error'] + if 'message' in err: + err_msg = err['message'] + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + else: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Missing 'message' field" + return (ret, f'Error: {err_msg}') + elif 'TaskStatus' in json_response: + status = json_response['TaskStatus'] + if status == 'OK': + task_id = json_response['Id'] + else: + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, f'Error: Return status is {status}') + + # Wait for completion + ret, error_msg, response = self.__wait_task_completion(task_id, timeout, sleep_timeout=1) + + if ret != RedfishClient.ERR_CODE_OK: + return (ret, error_msg) + + # Fetch the file with results + ret, error_msg, uri = self.__get_uri_from_response(response) + if ret != RedfishClient.ERR_CODE_OK or (not uri): + return (ret, error_msg) + + return (RedfishClient.ERR_CODE_OK, uri) + + + # TODO(BMC): Verify which functions are needed for BMC + # ''' + # Get EROT copy-background-status + + # Parameters: + # erot_fw_id erot component ID + # Return value: (ret, data) + # ret return code + # data EROT background-copy-status or error message + # ''' + # def redfish_api_get_erot_copy_background_status(self, erot_fw_id: str) -> tuple(): + # background_copy_status = 'N/A' + # # Make sure ERoT name doesn't have '_FW' part + # erot_id = re.sub('_FW', '', erot_fw_id) + + # uri = f'{RedfishClient.REDFISH_URI_CHASSIS_INVENTORY}/{erot_id}' + # cmd = self.__build_get_cmd(uri) + # ret, response, error = self.exec_curl_cmd(cmd) + + # if (ret == RedfishClient.ERR_CODE_OK): + # try: + # json_response = json.loads(response) + # background_copy_status = py_.get(json_response, 'Oem.Nvidia.BackgroundCopyStatus', default='N/A') + # except json.JSONDecodeError as e: + # return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, 'Error: Invalid JSON format') + # except Exception as e: + # return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, 'Error: unexpected response') + # else: + # self.__logger.log_notice(f"Was not able to read copy-background-status on {erot_fw_id}, return code is {ret}, response {response}") + + # return (ret, background_copy_status) + + + # ''' + + # Get debug token status + + # Parameters: + # timeout timeout value in seconds + # Return value: (ret, (response, error_msg)) + # ret return code + # response dictionary with ERoT component and its installed value + # E.g.: + # { + # "ERoT_CPU_0": { + # "TokenInstalled": false + # }, + # . . . + # "ERoT_BMC_0": { + # "TokenInstalled": false + # } + # } + + # error_msg error message string + # ''' + # def redfish_api_get_debug_token_status(self, timeout = 10): + + # cmd = self.__build_debug_token_cmd(debug_token_status=True) + # ret, response, error_msg = self.exec_curl_cmd(cmd) + # if (ret != RedfishClient.ERR_CODE_OK): + # return (ret, f'Error: {error_msg}') + + # ret, result = self._get_debug_token_responce(response, timeout) + # if ret != RedfishClient.ERR_CODE_OK: + # return (ret, f'Error: {result}') + + # cmd = self.__build_get_cmd(result) + # ret, response, error_msg = self.exec_curl_cmd(cmd) + + # try: + # json_response = json.loads(response) + # except json.JSONDecodeError as e: + # msg = 'Error: Invalid JSON format' + # return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, msg) + # except Exception as e: + # msg = 'Error: unexpected response' + # return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, msg) + + # result = {} + # erots_dict = json_response.get('DebugTokenStatus', []) + # for erot_dict in erots_dict: + # erot_id = erot_dict.get('@odata.id', '').replace('/redfish/v1/Chassis/', '') + # result[erot_id] = {'TokenInstalled': str(erot_dict.get('TokenInstalled', 'N/A'))} + + # return (ret, result) + + + # ''' + # Get EROT active & inactive flashes + + # Parameters: + # erot_fw_id erot-id + + # Return value: (ret, erot_info) + # ret return code + # erot_info erot_info containing active & inactive flashes + # ''' + # def redfish_api_get_erot_active_and_inactive_flashes(self, erot_fw_id: str) -> tuple(): + # active_flash = 'N/A' + # inactive_flash = 'N/A' + + # uri = f'{RedfishClient.REDFISH_URI_FW_INVENTORY}/{erot_fw_id}' + # cmd = self.__build_get_cmd(uri) + # ret, response, error = self.exec_curl_cmd(cmd) + + # if (ret == RedfishClient.ERR_CODE_OK): + # try: + # json_response = json.loads(response) + # active_flash = py_.get(json_response, 'Oem.Nvidia.ActiveFirmwareSlot.SlotId', default='N/A') + # inactive_flash = py_.get(json_response, 'Oem.Nvidia.InactiveFirmwareSlot.SlotId', default='N/A') + # except json.JSONDecodeError as e: + # return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, 'Error: Invalid JSON format') + # except Exception as e: + # return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, 'Error: unexpected response') + # else: + # self.__logger.log_notice(f"Was not able to read active/inactive on {erot_fw_id}, return code is {ret}, response {response}") + + # return (ret, {'active-flash': active_flash, 'inactive-flash': inactive_flash}) + + + # ''' + # Get EROT AP boot status + + # Parameters: + # erot_id erot-id (for example MGX_FW_ERoT_CPU_0) + + # Return value: (ret_code, ret_data) + # ret_code return code + # ret_data ret_data contains boot status (can be extended in the future) + # or error mesasge in case of return code is not 0 + # ''' + # def redfish_api_get_erot_ap_boot_status(self, erot_fw_id: str) -> tuple(): + # boot_status = 'N/A' + # # Make sure ERoT name doesn't have '_FW' part + # erot_id = re.sub('_FW', '', erot_fw_id) + # # The resouce name is expected to be without 'ERoT' part + # resource_name = re.sub('_ERoT', '', erot_id) + + # uri = f'{RedfishClient.REDFISH_URI_CHASSIS_INVENTORY}/{erot_id}/Oem/NvidiaRoT/RoTProtectedComponents/{resource_name}' + # cmd = self.__build_get_cmd(uri) + # ret, response, error = self.exec_curl_cmd(cmd) + + # if (ret == RedfishClient.ERR_CODE_OK): + # try: + # json_response = json.loads(response) + # boot_status = json_response.get('BootStatusCode', 'N/A') + # except json.JSONDecodeError as e: + # return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, 'Error: Invalid JSON format') + # except Exception as e: + # return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, 'Error: unexpected response') + # else: + # self.__logger.log_notice(f"Was not able to read AP boot status on {erot_fw_id}, return code is {ret}, response {response}") + + # return (ret, {'boot-status': boot_status}) + + + ''' + + Trigger BMC debug log dump file + + Return value: (ret, (task_id, error_msg)) + ret return code + task_id Redfish task-id to monitor + error_msg error message string + ''' + def redfish_api_trigger_bmc_debug_log_dump(self): + task_id = '-1' + + # Trigger debug log dump service + cmd = self.__build_bmc_debug_log_dump_cmd() + ret, response, error_msg = self.exec_curl_cmd(cmd) + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, (task_id, f'Error: {error_msg}')) + + try: + json_response = json.loads(response) + except Exception as e: + msg = 'Error: Invalid JSON format' + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, (task_id, msg)) + + # Retrieve task id from response + if 'error' in json_response: # Error found + err = json_response['error'] + if 'message' in err: + err_msg = err['message'] + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + else: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Missing 'message' field" + return (ret, (task_id, f'Error: {err_msg}')) + elif 'TaskStatus' in json_response: + status = json_response['TaskStatus'] + if status == 'OK': + task_id = json_response.get('Id', '') + ret = RedfishClient.ERR_CODE_OK + return (ret, (task_id, None)) + else: + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, (task_id, f'Error: Return status is {status}')) + + + ''' + Get BMC debug log dump file + + Parameters: + filename new file name + file_path location of the new file + timeout timeout value in seconds + Return value: (ret, error_msg) + ret return code + error_msg error message string + ''' + def redfish_api_get_bmc_debug_log_dump(self, task_id, filename, file_path, timeout = 120): + # Wait for completion + ret, error_msg, response = self.__wait_task_completion(task_id, timeout) + + if ret != RedfishClient.ERR_CODE_OK: + return (ret, error_msg) + + # Fetch the file + ret, error_msg, uri = self.__get_uri_from_response(response) + if ret != RedfishClient.ERR_CODE_OK: + return (ret, error_msg) + + if not uri: + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, error_msg) + + output_file = f'{file_path}/{filename}' + uri += '/attachment' + cmd = self.__build_get_cmd(uri, output_file=output_file) + ret, response, error_msg = self.exec_curl_cmd(cmd) + + return (ret, error_msg) + + + ''' + Reads all the eeproms of the bmc + + Parameters: None + Return value: (ret, eeprom_list) + ret return code + eeprom_list list of tuple (component_name, eeprom_data) + eeprom_data return value from redfish_api_get_eeprom_info called with component_name + ''' + def redfish_api_get_eeprom_list(self): + cmd = self.__build_get_cmd(RedfishClient.REDFISH_URI_CHASSIS_INVENTORY) + ret, response, error = self.exec_curl_cmd(cmd) + + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, []) + + try: + json_response = json.loads(response) + item_list = json_response["Members"] + except json.JSONDecodeError as e: + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, []) + except Exception as e: + return (RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE, []) + + eeprom_list = [] + for item in item_list: + component_url = item.get("@odata.id") + if not component_url: + continue + component_name = component_url.split('/')[-1] + if 'eeprom' not in component_name: + # If the name of the component doesn't contain eeprom, + # it is not an eeprom. Ignore it. + # For now, the only valid eeprom we have is BMC_eeprom. + # But we will probably have more in the future + continue + ret, eeprom_values = self.redfish_api_get_eeprom_info(component_name) + # No need for checking ret. + # If it is a bad value, + # redfish_api_get_eeprom_info will return a dictionary which indicates the error + + eeprom_list.append((component_name, eeprom_values)) + + return (RedfishClient.ERR_CODE_OK, eeprom_list) + + ''' + Get eeprom values for a given component + + Parameters: + component_name component name + Return value: (ret, eeprom_data) + ret return code + eeprom_data dictionary containing eeprom data + ''' + def redfish_api_get_eeprom_info(self, component_name): + uri = f'{RedfishClient.REDFISH_URI_CHASSIS_INVENTORY}/{component_name}' + cmd = self.__build_get_cmd(uri) + ret, response, err_msg = self.exec_curl_cmd(cmd) + + bad_eeprom_info = {'State': 'Fail'} + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, bad_eeprom_info) + + try: + json_response = json.loads(response) + except json.JSONDecodeError as e: + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + return (ret, bad_eeprom_info) + except Exception as e: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + return (ret, bad_eeprom_info) + + if 'error' in json_response: # Error found + err = json_response['error'] + if ('code' in err) and ('ResourceNotFound' in err['code']): + ret = RedfishClient.ERR_CODE_URI_NOT_FOUND + else: + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + self.__logger.log_error(f'Got redfish error response for {component_name} query: \n') + self.__log_json_response(response) + return (ret, bad_eeprom_info) + + eeprom_info = {} + for key,value in json_response.items(): + # Remove information that is not the eeprom content itself. + # But part of the redfish protocol + if '@odata' in str(value) or '@odata' in str(key): + continue + # Don't add the status, we will parse it and add it later + if key == 'Status': + continue + eeprom_info[str(key)] = str(value) + + # Add 'Status'. Even if it is not exactly part of the eeprom, + # it was part of the response we got. + # Can be very usefull also to see the value. + + status = json_response.get('Status',{}) + eeprom_info['State'] = status.get('State', 'Ok') + eeprom_info['Health'] = status.get('Health', 'Ok') + eeprom_info['HealthRollup'] = status.get('HealthRollup', 'Ok') + + return (RedfishClient.ERR_CODE_OK, eeprom_info) + + ''' + Wait for given task to complete + ''' + def __wait_task_completion(self, task_id, timeout = 1800, progress_callback = None, sleep_timeout = 2): + # Polling task status by given task id + + uri = f'{RedfishClient.REDFISH_URI_TASKS}/{task_id}' + cmd = self.__build_get_cmd(uri) + + start_tm = time.time() + + while True: + ret, response, err_msg = self.exec_curl_cmd(cmd) + if (ret != RedfishClient.ERR_CODE_OK): + return (ret, f"Error: {err_msg}", response) + + try: + json_response = json.loads(response) + except Exception as e: + msg = 'Error: Invalid JSON format' + return (RedfishClient.ERR_CODE_INVALID_JSON_FORMAT, msg, response) + + percent = None + if 'PercentComplete' in json_response: + percent = json_response['PercentComplete'] + if progress_callback: + progress_callback(percent) + + if "TaskStatus" not in json_response: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + err_msg = "Error: Missing 'TaskStatus' field" + return (ret, err_msg, response) + + status = json_response["TaskStatus"] + + if (status != "OK") and ("Messages" not in json_response): + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + err_msg = f'Error: Fail to execute the task - Taskstatus={status}' + return (ret, err_msg, response) + + same_version = False + err_detected = False + aborted = False + err_msg = '' + + for msg in json_response['Messages']: + msg_id = msg['MessageId'] + + if 'ResourceErrorsDetected' in msg_id: + err_detected = True + err_msg = msg['Message'] + elif 'TaskAborted' in msg_id: + aborted = True + elif 'ComponentUpdateSkipped' in msg_id: + same_version = True + err_msg = msg['Message'] + + if (status != 'OK'): + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + if err_detected: + return (ret, f'Error: {err_msg}', response) + elif aborted: + return (ret, 'Error: The task has been aborted', response) + else: + err_msg = f'Error: Fail to execute the task - Taskstatus={status}' + return (ret, err_msg, response) + elif same_version: + return (RedfishClient.ERR_CODE_IDENTICAL_IMAGE, err_msg, response) + + if percent is None: + continue + + if (percent == 100): + return (RedfishClient.ERR_CODE_OK, '', response) + + if (time.time() - start_tm > timeout): + return (RedfishClient.ERR_CODE_TIMEOUT, 'Wait task completion timeout', response) + + time.sleep(sleep_timeout) + + ''' + Change login password + + Parameters: + new_password new password to change + Return value: (ret, error_msg) + ret return code + error_msg error message string + ''' + def redfish_api_change_login_password(self, new_password, user=None): + self.__logger.log_notice(f'Changing BMC password\n') + + cmd = self.__build_change_password_cmd(new_password, user) + ret, response, error = self.exec_curl_cmd(cmd) + + if (ret != RedfishClient.ERR_CODE_OK): + self.__logger.log_error(f'Fail to change login password: {error}\n') + return (ret, f'Error: {error}') + else: + self.__logger.log_notice(f'Redfish response: \n') + self.__log_json_response(response) + try: + json_response = json.loads(response) + if 'error' in json_response: + msg = json_response['error']['message'] + self.__logger.log_error(f'Fail to change login password: {msg}\n') + + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, msg) + elif 'Password@Message.ExtendedInfo' in json_response: + for info in json_response['Password@Message.ExtendedInfo']: + if info['MessageId'].endswith('Error'): + msg = info['Message'] + self.__logger.log_error(f'Fail to change login password: {msg}\n') + resolution = info['Resolution'] + self.__logger.log_error(f'Resolution: {resolution}\n') + + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + + return (ret, msg) + else: + self.__logger.log_notice(f'Password changed sucessfully\n') + ret = RedfishClient.ERR_CODE_OK + except json.JSONDecodeError as e: + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + return (ret, 'Error: Invalid JSON format') + except Exception as e: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + return (ret, 'Error: Unexpected response format') + + # Logout and re-login for admin user. Do not care about the result. Logout will invalidate token. + # If it doesn't login successully, Redfish API call later on will do retry anyway. + if user is None or user == 'admin': + self.logout() + self.login() + + return (RedfishClient.ERR_CODE_OK, '') + + def redfish_api_set_component_update(self, comps): + if comps: + cmd = self.__build_set_component_update_cmd(comps) + else: + cmd = self.__build_set_component_update_reset_cmd() + ret, response, error = self.exec_curl_cmd(cmd) + + if (ret == RedfishClient.ERR_CODE_OK): + try: + json_response = json.loads(response) + + if 'error' in json_response: + msg = json_response['error']['message'] + self.__logger.log_error(f'{msg}\n') + + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + return (ret, msg) + elif 'ForceUpdate@Message.ExtendedInfo' in json_response: + for info in json_response['ForceUpdate@Message.ExtendedInfo']: + if info['MessageId'].endswith('Error'): + msg = info['Message'] + self.__logger.log_error(f'{msg}\n') + resolution = info['Resolution'] + self.__logger.log_error(f'Resolution: {resolution}\n') + + ret = RedfishClient.ERR_CODE_GENERIC_ERROR + + return (ret, msg) + else: + ret = RedfishClient.ERR_CODE_OK + except json.JSONDecodeError as e: + ret = RedfishClient.ERR_CODE_INVALID_JSON_FORMAT + except Exception as e: + ret = RedfishClient.ERR_CODE_UNEXPECTED_RESPONSE + + return (RedfishClient.ERR_CODE_OK, '') diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index f892929c5ef..f3979638f6e 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -333,12 +333,19 @@ def main(): args = parser.parse_args() platform = device_info.get_platform() + data = {} + + bmc_data = None + if platform: + bmc_data = device_info.get_bmc_data() + if bmc_data: + for key,value in bmc_data.items(): + deep_update(data, {'DEVICE_METADATA': {'bmc': {key:value}}}) db_kwargs = {} if args.redis_unix_sock_file is not None: db_kwargs['unix_socket_path'] = args.redis_unix_sock_file - data = {} hwsku = args.hwsku asic_name = args.namespace asic_id = None diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 47409eb18ca..d6772e08ee1 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -21,6 +21,8 @@ # Port configuration file names PORT_CONFIG_FILE = "port_config.ini" PLATFORM_JSON_FILE = "platform.json" +# TODO(BMC): Create bmc.json files in device/mellanox/<>/bmc.json +BMC_DATA_FILE = 'bmc.json' # Fabric port configuration file names FABRIC_MONITOR_CONFIG_FILE = "fabric_monitor_config.json" @@ -945,6 +947,21 @@ def is_warm_restart_enabled(container_name): return wr_enable_state +def get_bmc_data(): + json_file = None + try: + platform_path = get_path_to_platform_dir() + json_file = os.path.join(platform_path, BMC_DATA_FILE) + if os.path.exists(json_file): + with open(json_file, "r") as f: + bmc_data = json.loads(f.read()) + return bmc_data + else: + return None + except Exception: + return None + + # Check if System fast reboot is enabled. def is_fast_reboot_enabled(): state_db = SonicV2Connector(host='127.0.0.1')