[system-health] Add regression test cases for system health feature

tests/system_health/device_mocker.py

@@ -0,0 +1,62 @@
+import os
+import pytest
+import sys
+
+
+class DeviceMocker:
+    def deinit(self):
+        pass
+
+    def mock_fan_presence(self, status):
+        return False, None
+
+    def mock_fan_status(self, status):
+        return False, None
+
+    def mock_fan_speed(self, good):
+        return False, None
+
+    def mock_asic_temperature(self, good):
+        return False
+
+    def mock_psu_presence(self, status):
+        return False, None
+
+    def mock_psu_status(self, status):
+        return False, None
+
+    def mock_psu_temperature(self, good):
+        return False, None
+
+    def mock_psu_voltage(self, good):
+        return False, None
+
+
+@pytest.fixture
+def device_mocker_factory():
+    """
+    Fixture for system health data mocker factory.
+    :return: A function for creating system health related data mocker.
+    """
+    mockers = []
+
+    def _create_mocker(dut):
+        """
+        Create vendor specified mocker object by mocker name.
+        :param dut: DUT object representing a SONiC switch under test.
+        :return: Created mocker instance.
+        """
+        asic = dut.facts['asic_type']
+        mocker_object = None
+        if 'mellanox' in asic:
+            from .mellanox.mellanox_device_mocker import MellanoxDeviceMocker
+            mocker_object = MellanoxDeviceMocker(dut)
+            mockers.append(mocker_object)
+        else:
+            pytest.skip("No mocker defined for this platform %s")
+        return mocker_object
+
+    yield _create_mocker
+
+    for m in mockers:
+        m.deinit()

tests/system_health/files/device_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": [],
+    "external_checkers": [],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/external_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": [],
+    "external_checkers": ["cat /tmp/mock_valid_external_checker.txt"],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/ignore_asic_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": ["asic"],
+    "external_checkers": [],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/ignore_device_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": ["asic","fan","psu"],
+    "external_checkers": [],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/ignore_fan_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": ["fan"],
+    "external_checkers": [],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/ignore_psu_check.json

@@ -0,0 +1,12 @@
+{
+    "services_to_ignore": [],
+    "devices_to_ignore": ["psu"],
+    "external_checkers": [],
+    "polling_interval": 10,
+    "led_color": {
+      "fault": "orange",
+      "normal": "green",
+      "booting": "orange_blink"
+    },
+    "boot_timeout": 300
+}

tests/system_health/files/mock_valid_external_checker.txt

@@ -0,0 +1,3 @@
+ExternalCategory
+ExternalService:Service is not working
+ExternalDevice:Device is broken

tests/system_health/mellanox/mellanox_device_mocker.py

@@ -0,0 +1,131 @@
+from ..device_mocker import DeviceMocker
+from tests.common.mellanox_data import SWITCH_MODELS
+from tests.platform_tests.mellanox.mellanox_thermal_control_test_helper import MockerHelper, FanDrawerData, FanData, \
+    FAN_NAMING_RULE
+
+
+class AsicData(object):
+    TEMPERATURE_FILE = '/run/hw-management/thermal/asic'
+    THRESHOLD_FILE = '/run/hw-management/thermal/mlxsw/temp_trip_hot'
+
+    def __init__(self, mock_helper):
+        self.helper = mock_helper
+
+    def mock_asic_temperature(self, value):
+        self.helper.mock_value(AsicData.TEMPERATURE_FILE, str(value))
+
+    def get_asic_temperature_threshold(self):
+        value = self.helper.read_value(AsicData.THRESHOLD_FILE)
+        return int(value)
+
+
+class PsuData(object):
+    PSU_STATUS_FILE = '/run/hw-management/thermal/psu{}_status'
+    PSU_POWER_STATUS_FILE = '/run/hw-management/thermal/psu{}_pwr_status'
+    PSU_TEMPERATURE_FILE = '/run/hw-management/thermal/psu{}_temp'
+    PSU_TEMP_THRESHOLD_FILE = '/run/hw-management/thermal/psu{}_temp_max'
+
+    def __init__(self, mock_helper, index):
+        self.helper = mock_helper
+        self.index = index
+        self.name = 'PSU {}'.format(self.index)
+        power_status_file = PsuData.PSU_POWER_STATUS_FILE.format(index)
+        if self.helper._file_exist(power_status_file):
+            self.power_on = True
+        else:
+            self.power_on = False
+
+    def mock_presence(self, status):
+        value = 1 if status else 0
+        presence_file = PsuData.PSU_STATUS_FILE.format(self.index)
+        self.helper.mock_value(presence_file, str(value))
+
+    def mock_status(self, status):
+        value = 1 if status else 0
+        power_status_file = PsuData.PSU_POWER_STATUS_FILE.format(self.index)
+        self.helper.mock_value(power_status_file, str(value))
+
+    def mock_temperature(self, value):
+        temperature_file = PsuData.PSU_TEMPERATURE_FILE.format(self.index)
+        self.helper.mock_value(temperature_file, str(value))
+
+    def get_psu_temperature_threshold(self):
+        threshold_file = PsuData.PSU_TEMP_THRESHOLD_FILE.format(self.index)
+        value = self.helper.read_value(threshold_file)
+        return int(value)
+
+
+class MellanoxDeviceMocker(DeviceMocker):
+    TARGET_SPEED_VALUE = 60
+    SPEED_TOLERANCE = 20
+    PSU_NUM = 2
+
+    def __init__(self, dut):
+        self.mock_helper = MockerHelper(dut)
+        self.asic_data = AsicData(self.mock_helper)
+        naming_rule = FAN_NAMING_RULE['fan']
+        self.fan_drawer_data = FanDrawerData(self.mock_helper, naming_rule, 1)
+        self.fan_data = FanData(self.mock_helper, naming_rule, 1)
+
+        for i in range(MellanoxDeviceMocker.PSU_NUM):
+            self.psu_data = PsuData(self.mock_helper, i + 1)
+            if self.psu_data.power_on:
+                break
+
+    def deinit(self):
+        self.mock_helper.deinit()
+
+    def mock_fan_presence(self, status):
+        dut_hwsku = self.mock_helper.dut.facts["hwsku"]
+        always_present = not SWITCH_MODELS[dut_hwsku]['fans']['hot_swappable']
+        if always_present:
+            return False, None
+
+        value = 1 if status else 0
+        self.fan_drawer_data.mock_presence(value)
+        return True, self.fan_data.name
+
+    def mock_fan_status(self, status):
+        value = 0 if status else 1
+        self.fan_data.mock_status(value)
+        return True, self.fan_data.name
+
+    def mock_fan_speed(self, good):
+        target_speed = self.fan_data.get_target_speed()
+        if good:
+            actual_speed = target_speed
+        else:
+            actual_speed = target_speed * (5 + ((MellanoxDeviceMocker.SPEED_TOLERANCE + 1) / float(100)))
+            actual_speed = int(actual_speed)
+        self.fan_data.mock_speed(actual_speed)
+        return True, self.fan_data.name
+
+    def mock_asic_temperature(self, good):
+        threshold = self.asic_data.get_asic_temperature_threshold()
+        if good:
+            value = threshold - 1000
+        else:
+            value = threshold + 1000
+        self.asic_data.mock_asic_temperature(value)
+        return True
+
+    def mock_psu_presence(self, status):
+        self.psu_data.mock_presence(1 if status else 0)
+        return True, self.psu_data.name
+
+    def mock_psu_status(self, status):
+        self.psu_data.mock_status(1 if status else 0)
+        return True, self.psu_data.name
+
+    def mock_psu_temperature(self, good):
+        threshold = self.psu_data.get_psu_temperature_threshold()
+        if good:
+            value = threshold - 1000
+        else:
+            value = threshold + 1000
+        self.psu_data.mock_temperature(value)
+        return True, self.psu_data.name
+
+    def mock_psu_voltage(self, good):
+        # Not Supported for now
+        return False, None