service_checker should consider k8s rollout containers (#25660)

Why I did it
K8s rollout containers have long names, and we will use container label to identify its original container/service names such as "restapi". This PR will extend service_checker to respect either image native containers or k8s rollout containers for the same service.

Work item tracking
Microsoft ADO (number only):
How I did it
Check all containers labels.

How to verify it
Unit test. Manually verified on a sonic device with k8s rollout restapi container.

Signed-off-by: Feng Pan <[email protected]>

Author: qiluo-msft

src/system-health/health_checker/service_checker.py

@@ -152,6 +152,20 @@ def get_current_running_containers(self):
             lst = ctrs.list(filters={"status": "running"})
 
             for ctr in lst:
+                # Check if this is a Kubernetes-managed container
+                labels = ctr.labels or {}
+                ns = labels.get("io.kubernetes.pod.namespace")
+                dtype = labels.get("io.kubernetes.docker.type")
+                kname = labels.get("io.kubernetes.container.name")
+
+                if ns == "sonic":
+                    # Kubernetes-managed container - add service name to running containers
+                    # but skip critical process checking (k8s has its own health mechanisms)
+                    if dtype == "container" and kname and kname not in ("<no value>", "POD"):
+                        running_containers.add(kname)
+                    continue
+
+                # Regular Docker container - use the container name
                 running_containers.add(ctr.name)
                 if ctr.name not in self.container_critical_processes:
                     self.fill_critical_process_by_container(ctr.name)

src/system-health/tests/test_system_health.py

@@ -354,6 +354,144 @@ def test_service_checker_check_by_monit(mock_run):
     assert checker._info['diskCheck'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
 
 
+@patch('swsscommon.swsscommon.ConfigDBConnector.connect', MagicMock())
+@patch('health_checker.service_checker.ServiceChecker._get_container_folder', MagicMock(return_value=test_path))
+@patch('sonic_py_common.multi_asic.is_multi_asic', MagicMock(return_value=False))
+@patch('docker.DockerClient')
+@patch('health_checker.utils.run_command')
+@patch('swsscommon.swsscommon.ConfigDBConnector')
+def test_service_checker_k8s_containers(mock_config_db, mock_run, mock_docker_client):
+    """Test that service checker recognizes Kubernetes-managed containers by labels"""
+    setup()
+    mock_db_data = MagicMock()
+    mock_get_table = MagicMock()
+    mock_db_data.get_table = mock_get_table
+    mock_config_db.return_value = mock_db_data
+    mock_get_table.return_value = {
+        'snmp': {
+            'state': 'enabled',
+            'has_global_scope': 'True',
+            'has_per_asic_scope': 'False',
+        },
+        'restapi': {
+            'state': 'enabled',
+            'has_global_scope': 'True',
+            'has_per_asic_scope': 'False',
+        }
+    }
+    
+    # Mock Kubernetes containers with labels
+    mock_containers = MagicMock()
+    mock_snmp_container = MagicMock()
+    mock_snmp_container.name = 'k8s_snmp_snmp-pod-test_sonic_12345678-1234-1234-1234-123456789abc_0'
+    mock_snmp_container.labels = {
+        'io.kubernetes.pod.namespace': 'sonic',
+        'io.kubernetes.docker.type': 'container',
+        'io.kubernetes.container.name': 'snmp'
+    }
+    
+    mock_restapi_container = MagicMock()
+    mock_restapi_container.name = 'k8s_restapi_restapi-pod-test_sonic_87654321-4321-4321-4321-cba987654321_0'
+    mock_restapi_container.labels = {
+        'io.kubernetes.pod.namespace': 'sonic',
+        'io.kubernetes.docker.type': 'container',
+        'io.kubernetes.container.name': 'restapi'
+    }
+    
+    # Mock POD container (should be ignored)
+    mock_pod_container = MagicMock()
+    mock_pod_container.name = 'k8s_POD_snmp-pod-test_sonic_12345678-1234-1234-1234-123456789abc_0'
+    mock_pod_container.labels = {
+        'io.kubernetes.pod.namespace': 'sonic',
+        'io.kubernetes.docker.type': 'container',
+        'io.kubernetes.container.name': 'POD'
+    }
+    
+    mock_containers.list = MagicMock(return_value=[mock_snmp_container, mock_restapi_container, mock_pod_container])
+    mock_docker_client_object = MagicMock()
+    mock_docker_client.return_value = mock_docker_client_object
+    mock_docker_client_object.containers = mock_containers
+    
+    mock_run.return_value = mock_supervisorctl_output
+    
+    checker = ServiceChecker()
+    config = Config()
+    checker.check(config)
+    
+    # Verify k8s containers are recognized by their label names
+    running_containers = checker.get_current_running_containers()
+    assert 'snmp' in running_containers
+    assert 'restapi' in running_containers
+    assert 'POD' not in running_containers
+    
+    # Verify k8s containers are NOT added to critical processes (k8s has its own health checks)
+    assert 'snmp' not in checker.container_critical_processes
+    assert 'restapi' not in checker.container_critical_processes
+
+
+@patch('swsscommon.swsscommon.ConfigDBConnector.connect', MagicMock())
+@patch('health_checker.service_checker.ServiceChecker._get_container_folder', MagicMock(return_value=test_path))
+@patch('sonic_py_common.multi_asic.is_multi_asic', MagicMock(return_value=False))
+@patch('docker.DockerClient')
+@patch('health_checker.utils.run_command')
+@patch('swsscommon.swsscommon.ConfigDBConnector')
+def test_service_checker_mixed_containers(mock_config_db, mock_run, mock_docker_client):
+    """Test that service checker handles both regular Docker and Kubernetes containers"""
+    setup()
+    mock_db_data = MagicMock()
+    mock_get_table = MagicMock()
+    mock_db_data.get_table = mock_get_table
+    mock_config_db.return_value = mock_db_data
+    mock_get_table.return_value = {
+        'swss': {
+            'state': 'enabled',
+            'has_global_scope': 'True',
+            'has_per_asic_scope': 'False',
+        },
+        'database': {
+            'state': 'enabled',
+            'has_global_scope': 'True',
+            'has_per_asic_scope': 'False',
+        }
+    }
+    
+    mock_containers = MagicMock()
+    
+    # Regular Docker container
+    mock_swss_container = MagicMock()
+    mock_swss_container.name = 'swss'
+    mock_swss_container.labels = {}
+    
+    # Kubernetes container
+    mock_database_container = MagicMock()
+    mock_database_container.name = 'k8s_database_database-pod-test_sonic_12345678_0'
+    mock_database_container.labels = {
+        'io.kubernetes.pod.namespace': 'sonic',
+        'io.kubernetes.docker.type': 'container',
+        'io.kubernetes.container.name': 'database'
+    }
+    
+    mock_containers.list = MagicMock(return_value=[mock_swss_container, mock_database_container])
+    mock_docker_client_object = MagicMock()
+    mock_docker_client.return_value = mock_docker_client_object
+    mock_docker_client_object.containers = mock_containers
+    
+    mock_run.return_value = mock_supervisorctl_output
+    
+    checker = ServiceChecker()
+    config = Config()
+    checker.check(config)
+    
+    # Verify both types of containers are recognized
+    running_containers = checker.get_current_running_containers()
+    assert 'swss' in running_containers
+    assert 'database' in running_containers
+    
+    # Verify only regular Docker containers are monitored for critical processes
+    assert 'swss' in checker.container_critical_processes
+    assert 'database' not in checker.container_critical_processes  # k8s container, not monitored
+
+
 def test_hardware_checker():
     MockConnector.data.update({
         'TEMPERATURE_INFO|ASIC': {