diff --git a/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py b/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py index 062dffe8027..73c735180da 100644 --- a/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py +++ b/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py @@ -1136,3 +1136,21 @@ def deinit(self): :return: """ self.mock_helper.deinit() + + +@mocker('PsuMocker') +class PsuMocker(object): + PSU_PRESENCE = 'psu{}_status' + + def __init__(self, dut): + self.mock_helper = MockerHelper(dut) + + def deinit(self): + """ + Destructor of MinTableMocker. + :return: + """ + self.mock_helper.deinit() + + def mock_psu_status(self, psu_index, status): + self.mock_helper.mock_thermal_value(self.PSU_PRESENCE.format(psu_index), '1' if status else '0') diff --git a/tests/platform_tests/mellanox/test_thermal_control.py b/tests/platform_tests/mellanox/test_thermal_control.py index e00000a08a1..15e81c8e083 100644 --- a/tests/platform_tests/mellanox/test_thermal_control.py +++ b/tests/platform_tests/mellanox/test_thermal_control.py @@ -20,11 +20,14 @@ THERMAL_CONTROL_TEST_WAIT_TIME = 75 THERMAL_CONTROL_TEST_CHECK_INTERVAL = 5 +THERMAL_PATH = '/run/hw-management/thermal/' COOLING_CUR_STATE_PATH = '/run/hw-management/thermal/cooling_cur_state' COOLING_CUR_STATE_THRESHOLD = 7 PSU_PRESENCE_PATH = '/run/hw-management/thermal/psu{}_status' PSU_SPEED_PATH = '/run/hw-management/thermal/psu{}_fan1_speed_get' PSU_MAX_SPEED_PATH = '/run/hw-management/config/psu_fan_max' +PWM_PATH = '/run/hw-management/thermal/pwm1' +MAX_PWM = 255 PSU_SPEED_TOLERANCE = 0.25 MAX_COOLING_LEVEL = 10 @@ -133,6 +136,29 @@ def test_set_psu_fan_speed(duthosts, rand_one_dut_hostname, mocker_factory): assert False, 'Wait for PSU fan speed change to normal failed' +@pytest.mark.disable_loganalyzer +def test_psu_absence_policy(duthosts, rand_one_dut_hostname, mocker_factory): + duthost = duthosts[rand_one_dut_hostname] + platform_data = get_platform_data(duthost) + hot_swappable = platform_data['psus']['hot_swappable'] + if not hot_swappable: + pytest.skip('The platform {} does not support this test case.'.format(duthost.facts["platform"])) + + psu_num = platform_data['psus']['number'] + psu_mocker = mocker_factory(duthost, 'PsuMocker') + psu_index = random.randint(1, psu_num) + psu_mocker.mock_psu_status(psu_index, False) + wait_result = wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, + THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, + check_pwm, + duthost, + MAX_PWM, + operator.eq) + assert wait_result, 'PSU is absent, but PWM value is not turned to {}'.format(MAX_PWM) + assert check_fan_speed(duthost, MAX_PWM), 'Fan speed is not turn to {}'.format(MAX_PWM) + + def _check_psu_fan_speed_in_range(actual_speed, max_speed, cooling_level): expect_speed = max_speed * cooling_level / 10.0 logger.info('Expect speed: {}, actual speed: {}'.format(expect_speed, actual_speed)) @@ -208,3 +234,39 @@ def check_psu_fan_speed(duthost, psu_num, psu_max_speed, op): def check_cooling_level_larger_than_minimum(duthost, expect_minimum_cooling_level): actual_cooling_level = get_cooling_cur_state(duthost) return actual_cooling_level >= expect_minimum_cooling_level + + +def get_pwm_value(dut): + cmd_output = dut.command('cat {}'.format(PWM_PATH)) + try: + pwm_value = int(cmd_output['stdout']) + logger.info('PWM is {}'.format(pwm_value)) + return pwm_value + except Exception as e: + assert False, 'Bad content in {} - {}'.format(PWM_PATH, e) + + +def check_pwm(duthost, expect_value, op): + """Check if FAN PWM value is the expect value + + Args: + duthost (object): DUT host object + expect_value (int): Expect PWM value + op (object): Operator eq or ne + + Returns: + boolean: True if the pwm value is expected + """ + pwm_value = get_pwm_value(duthost) + return op(pwm_value, expect_value) + + +def check_fan_speed(duthost, expect_value): + get_fan_speed_sysfs_cmd = 'ls {}fan*_speed_set'.format(THERMAL_PATH) + file_list = duthost.shell(get_fan_speed_sysfs_cmd)['stdout'].splitlines() + for file in file_list: + actual_speed = int(duthost.shell('cat {}'.format(file))['stdout'].strip()) + if actual_speed != expect_value: + logging.error('For file {}, Expect speed {}, but actual is {}'.format(file, expect_value, actual_speed)) + return False + return True diff --git a/tests/platform_tests/test_platform_info.py b/tests/platform_tests/test_platform_info.py index c1292ef7127..d19246f365b 100644 --- a/tests/platform_tests/test_platform_info.py +++ b/tests/platform_tests/test_platform_info.py @@ -335,97 +335,6 @@ def check_thermal_control_load_invalid_file(duthost, file_name): restart_thermal_control_daemon(duthost) -@pytest.mark.disable_loganalyzer -@pytest.mark.parametrize('ignore_particular_error_log', [SKIP_ERROR_LOG_PSU_ABSENCE], indirect=True) -def test_thermal_control_psu_absence(duthosts, enum_rand_one_per_hwsku_hostname, pdu_controller, mocker_factory, ignore_particular_error_log): - """ - @summary: Turn off/on PSUs, check thermal control is working as expect. - """ - duthost = duthosts[enum_rand_one_per_hwsku_hostname] - psu_num = get_psu_num(duthost) - - pytest_require(psu_num >= 2, "At least 2 PSUs required for rest of the testing in this case") - - logging.info("Create PDU controller for testing") - pdu_ctrl = pdu_controller - - pytest_require(pdu_ctrl, "No PDU controller for %s, skip rest of the testing in this case" % duthost.hostname) - - logging.info("To avoid DUT being shutdown, need to turn on PSUs that are not powered") - turn_all_outlets_on(pdu_ctrl) - - logging.info("Initialize test results") - psu_test_results = {} - pytest_require(check_all_psu_on(duthost, psu_test_results), "Some PSU are still down, skip rest of the testing in this case") - - with ThermalPolicyFileContext(duthost, THERMAL_POLICY_VALID_FILE): - fan_mocker = mocker_factory(duthost, 'FanStatusMocker') - pytest_require(fan_mocker, "No FanStatusMocker for %s, skip rest of the testing in this case" % duthost.facts['asic_type']) - - restart_thermal_control_daemon(duthost) - logging.info('Wait and check all FAN speed turn to 60%...') - wait_result = wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, - THERMAL_CONTROL_TEST_CHECK_INTERVAL, - 0, - fan_mocker.check_all_fan_speed, - 60) - - pytest_require(wait_result, "FAN speed is not 60%, there might be abnormal in FAN/PSU, skip rest of the testing in this case") - - check_thermal_algorithm_status(duthost, mocker_factory, False) - - logging.info('Shutdown first PDU outlet and check thermal control result...') - all_outlet_status = pdu_ctrl.get_outlet_status() - pytest_require(all_outlet_status and len(all_outlet_status) >= 2, 'Skip the test, cannot get at least 2 outlet status: {}'.format(all_outlet_status)) - outlet = all_outlet_status[0] - turn_off_outlet_and_check_thermal_control(duthost, pdu_ctrl, outlet, fan_mocker) - psu_test_results.clear() - pytest_require(check_all_psu_on(duthost, psu_test_results), "Some PSU are still down, skip rest of the testing in this case") - - logging.info('Shutdown second PDU outlet and check thermal control result...') - outlet = all_outlet_status[1] - turn_off_outlet_and_check_thermal_control(duthost, pdu_ctrl, outlet, fan_mocker) - psu_test_results.clear() - pytest_require(check_all_psu_on(duthost, psu_test_results), "Some PSU are still down, skip rest of the testing in this case") - - logging.info('Wait and check all FAN speed turn to 65%...') - pytest_assert(wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, - THERMAL_CONTROL_TEST_CHECK_INTERVAL, - 0, - fan_mocker.check_all_fan_speed, - 65), 'FAN speed not change to 65% according to policy') - - -def turn_off_outlet_and_check_thermal_control(dut, pdu_ctrl, outlet, mocker): - """ - @summary: Turn off PSUs, check all FAN speed are set to 100% according to thermal - control policy file. - """ - logging.info("Turn off outlet %s" % str(outlet["outlet_id"])) - pdu_ctrl.turn_off_outlet(outlet) - time.sleep(5) - - psu_under_test = None - psu_line_pattern = get_dut_psu_line_pattern(dut) - cli_psu_status = dut.command(CMD_PLATFORM_PSUSTATUS) - for line in cli_psu_status["stdout_lines"][2:]: - psu_match = psu_line_pattern.match(line) - pytest_assert(psu_match, "Unexpected PSU status output") - if psu_match.group(2) != "OK": - psu_under_test = psu_match.group(1) - - pytest_assert(psu_under_test is not None, "No PSU is turned off") - logging.info('Wait and check all FAN speed turn to 100%...') - pytest_assert(wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, - THERMAL_CONTROL_TEST_CHECK_INTERVAL, - 0, - mocker.check_all_fan_speed, - 100), 'FAN speed not turn to 100% after PSU off') - - pdu_ctrl.turn_on_outlet(outlet) - time.sleep(5) - - @pytest.mark.disable_loganalyzer def test_thermal_control_fan_status(duthosts, enum_rand_one_per_hwsku_hostname, mocker_factory): """