Skip to content

Commit b52393c

Browse files
crazysaltfishcrazysaltfish
authored
Fix parallel healthcheck (#1052)
1 parent b28980c commit b52393c

File tree

8 files changed

+120
-22
lines changed

8 files changed

+120
-22
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
<description>SOFABoot Build</description>
3838

3939
<properties>
40-
<revision>3.16.2</revision>
40+
<revision>3.16.3</revision>
4141
<sofa.boot.version>${revision}</sofa.boot.version>
4242
<!--maven plugin-->
4343
<maven.staging.plugin>1.6.7</maven.staging.plugin>

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthCheckerProcessor.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,9 @@ public boolean readinessHealthCheck(Map<String, Health> healthMap) {
176176
} catch (InterruptedException e) {
177177
logger.error(ErrorCode.convert("01-22005"), e);
178178
}
179+
if (!finished) {
180+
healthMap.put(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY, Health.unknown().withDetail(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY,SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG).build());
181+
}
179182
result = finished && parallelResult.get();
180183
} else {
181184
result = readinessHealthCheckers.entrySet().stream()

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthIndicatorProcessor.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,9 @@ public boolean readinessHealthCheck(Map<String, Health> healthMap) {
181181
} catch (InterruptedException e) {
182182
logger.error(ErrorCode.convert("01-21004"), e);
183183
}
184+
if (!finished) {
185+
healthMap.put(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY, Health.unknown().withDetail(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY,SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG).build());
186+
}
184187
result = finished && parallelResult.get();
185188
} else {
186189
result = healthIndicators.entrySet().stream()

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/ReadinessCheckListener.java‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -238,29 +238,30 @@ public Health aggregateReadinessHealth() {
238238
SofaBootConstants.SOFABOOT_HEALTH_CHECK_NOT_READY_MSG).build());
239239
} else {
240240
boolean healthCheckerStatus = getHealthCheckerStatus();
241+
boolean healthIndicatorStatus = getHealthIndicatorStatus();
242+
boolean afterReadinessCheckCallbackStatus = getHealthCallbackStatus();
241243
Map<String, Health> healthCheckerDetails = getHealthCheckerDetails();
242244
Map<String, Health> healthIndicatorDetails = getHealthIndicatorDetails();
243-
244-
boolean afterReadinessCheckCallbackStatus = getHealthCallbackStatus();
245245
Map<String, Health> afterReadinessCheckCallbackDetails = getHealthCallbackDetails();
246246

247247
Health.Builder builder;
248-
if (healthCheckerStatus && afterReadinessCheckCallbackStatus) {
249-
builder = Health.up();
250-
} else {
251-
builder = Health.down();
252-
}
248+
builder = healthCheckerStatus ? Health.up():Health.down();
253249
if (!CollectionUtils.isEmpty(healthCheckerDetails)) {
254-
builder = builder.withDetail("HealthChecker", healthCheckerDetails);
250+
builder = builder.withDetails(healthCheckerDetails);
255251
}
256-
if (!CollectionUtils.isEmpty(afterReadinessCheckCallbackDetails)) {
257-
builder = builder.withDetail("ReadinessCheckCallback",
258-
afterReadinessCheckCallbackDetails);
252+
healths.put("HealthCheckerInfo", builder.build());
253+
254+
builder = healthIndicatorStatus ? Health.up():Health.down();
255+
if (!CollectionUtils.isEmpty(healthIndicatorDetails)) {
256+
builder = builder.withDetails(healthIndicatorDetails);
259257
}
260-
healths.put("SOFABootReadinessHealthCheckInfo", builder.build());
258+
healths.put("HealthIndicatorInfo", builder.build());
261259

262-
// HealthIndicator
263-
healths.putAll(healthIndicatorDetails);
260+
builder = afterReadinessCheckCallbackStatus ? Health.up():Health.down();
261+
if (!CollectionUtils.isEmpty(afterReadinessCheckCallbackDetails)) {
262+
builder = builder.withDetails(afterReadinessCheckCallbackDetails);
263+
}
264+
healths.put("ReadinessCheckCallbackInfo", builder.build());
264265
}
265266
Status overallStatus = this.statusAggregator.getAggregateStatus(
266267
healths.values().stream().map(Health::getStatus).collect(Collectors.toSet()));

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/test/java/com/alipay/sofa/healthcheck/test/HealthCheckerProcessorParallelTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,21 @@ public void testComponentHealthCheckerFailedFirst() {
223223
}
224224
}
225225

226+
@Test
227+
public void testHealthCheckerParallelTimeout() {
228+
initApplicationContextTimeout(0, false, 4);
229+
230+
HashMap<String, Health> hashMap = new HashMap<>();
231+
HealthCheckerProcessor healthCheckerProcessor = applicationContext
232+
.getBean(HealthCheckerProcessor.class);
233+
boolean result = healthCheckerProcessor.readinessHealthCheck(hashMap);
234+
Assert.assertFalse(result);
235+
Health timeoutHealth = hashMap.get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY);
236+
Assert.assertEquals(Status.UNKNOWN, timeoutHealth.getStatus());
237+
Assert.assertEquals(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG, timeoutHealth
238+
.getDetails().get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY));
239+
}
240+
226241
private void initApplicationContext(int count, boolean strict, int retryCount) {
227242
Map<String, Object> properties = new LinkedHashMap<>();
228243
properties.put("memory-health-checker.count", count);
@@ -237,4 +252,20 @@ private void initApplicationContext(int count, boolean strict, int retryCount) {
237252
springApplication.setWebApplicationType(WebApplicationType.NONE);
238253
applicationContext = springApplication.run();
239254
}
255+
256+
private void initApplicationContextTimeout(int count, boolean strict, int retryCount) {
257+
Map<String, Object> properties = new LinkedHashMap<>();
258+
properties.put("memory-health-checker.count", count);
259+
properties.put("memory-health-checker.strict", strict);
260+
properties.put("memory-health-checker.retry-count", retryCount);
261+
properties.put("spring.application.name", "HealthCheckerProcessorTest");
262+
properties.put(SofaBootConstants.SOFABOOT_SKIP_COMPONENT_HEALTH_CHECK, true);
263+
properties.put("com.alipay.sofa.boot.health-check-parallel-timeout", "1");
264+
265+
SpringApplication springApplication = new SpringApplication(
266+
HealthCheckerProcessorTestConfiguration.class);
267+
springApplication.setDefaultProperties(properties);
268+
springApplication.setWebApplicationType(WebApplicationType.NONE);
269+
applicationContext = springApplication.run();
270+
}
240271
}

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/test/java/com/alipay/sofa/healthcheck/test/HealthIndicatorCheckProcessorParallelTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package com.alipay.sofa.healthcheck.test;
1818

19+
import com.alipay.sofa.boot.constant.SofaBootConstants;
1920
import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
2021
import com.alipay.sofa.healthcheck.HealthCheckProperties;
2122
import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
@@ -30,6 +31,7 @@
3031
import org.springframework.boot.SpringApplication;
3132
import org.springframework.boot.WebApplicationType;
3233
import org.springframework.boot.actuate.health.Health;
34+
import org.springframework.boot.actuate.health.Status;
3335
import org.springframework.boot.context.properties.EnableConfigurationProperties;
3436
import org.springframework.context.ApplicationContext;
3537
import org.springframework.context.annotation.Bean;
@@ -121,10 +123,39 @@ public void testCheckIndicatorFailed() {
121123
Assert.assertEquals(2, hashMap.size());
122124
}
123125

126+
@Test
127+
public void testCheckIndicatorParallelTimeout() {
128+
initApplicationContextTimeout();
129+
ReadinessCheckListener readinessCheckListener = applicationContext
130+
.getBean(ReadinessCheckListener.class);
131+
Health health = readinessCheckListener.aggregateReadinessHealth();
132+
Health healthIndicatorInfo = (Health) health.getDetails().get("HealthIndicatorInfo");
133+
Health timeoutHealth = (Health) healthIndicatorInfo.getDetails().get(
134+
SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY);
135+
Assert.assertEquals(Status.DOWN, health.getStatus());
136+
Assert.assertEquals(Status.UNKNOWN, timeoutHealth.getStatus());
137+
Assert.assertEquals(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG, timeoutHealth
138+
.getDetails().get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY));
139+
}
140+
124141
private void initApplicationContext(boolean health) {
125142
Map<String, Object> properties = new LinkedHashMap<>();
126143
properties.put("disk-health-indicator.health", health);
127144
properties.put("spring.application.name", "HealthIndicatorCheckProcessorTest");
145+
properties.put(SofaBootConstants.SOFABOOT_SKIP_HEALTH_INDICATOR_CHECK, "true");
146+
SpringApplication springApplication = new SpringApplication(
147+
HealthIndicatorConfiguration.class);
148+
springApplication.setDefaultProperties(properties);
149+
springApplication.setWebApplicationType(WebApplicationType.NONE);
150+
applicationContext = springApplication.run();
151+
}
152+
153+
private void initApplicationContextTimeout() {
154+
Map<String, Object> properties = new LinkedHashMap<>();
155+
properties.put("disk-health-indicator.health", true);
156+
properties.put("spring.application.name", "HealthIndicatorCheckProcessorTest");
157+
// properties.put(SofaBootConstants.SOFABOOT_SKIP_HEALTH_INDICATOR_CHECK, "true");
158+
properties.put("com.alipay.sofa.boot.health-check-parallel-timeout", "1");
128159
SpringApplication springApplication = new SpringApplication(
129160
HealthIndicatorConfiguration.class);
130161
springApplication.setDefaultProperties(properties);

‎sofa-boot-project/sofa-boot-core/healthcheck-sofa-boot/src/test/java/com/alipay/sofa/healthcheck/test/ReadinessCheckListenerTest.java‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,19 @@
1616
*/
1717
package com.alipay.sofa.healthcheck.test;
1818

19+
import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
1920
import com.alipay.sofa.healthcheck.HealthCheckProperties;
21+
import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
22+
import com.alipay.sofa.healthcheck.HealthIndicatorProcessor;
23+
import com.alipay.sofa.healthcheck.ReadinessCheckListener;
2024
import com.alipay.sofa.healthcheck.core.HealthCheckExecutor;
25+
import com.alipay.sofa.healthcheck.test.bean.DiskHealthIndicator;
26+
import com.alipay.sofa.healthcheck.test.bean.MemoryHealthChecker;
27+
import com.alipay.sofa.healthcheck.test.bean.MiddlewareHealthCheckCallback;
28+
import com.alipay.sofa.runtime.SofaRuntimeProperties;
2129
import com.alipay.sofa.runtime.configure.SofaRuntimeConfigurationProperties;
2230
import org.junit.Assert;
31+
import org.junit.BeforeClass;
2332
import org.junit.Test;
2433
import org.junit.runner.RunWith;
2534
import org.springframework.beans.BeansException;
@@ -40,13 +49,9 @@
4049
import org.springframework.test.context.TestPropertySource;
4150
import org.springframework.test.context.junit4.SpringRunner;
4251

43-
import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
44-
import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
45-
import com.alipay.sofa.healthcheck.HealthIndicatorProcessor;
46-
import com.alipay.sofa.healthcheck.ReadinessCheckListener;
47-
import com.alipay.sofa.healthcheck.test.bean.DiskHealthIndicator;
48-
import com.alipay.sofa.healthcheck.test.bean.MemoryHealthChecker;
49-
import com.alipay.sofa.healthcheck.test.bean.MiddlewareHealthCheckCallback;
52+
import java.lang.reflect.Field;
53+
import java.lang.reflect.Modifier;
54+
import java.util.concurrent.ConcurrentHashMap;
5055

5156
/**
5257
* @author liangen
@@ -120,6 +125,20 @@ public HealthCheckExecutor healthCheckExecutor(HealthCheckProperties properties)
120125
}
121126
}
122127

128+
@BeforeClass
129+
public static void beforeClass() {
130+
try {
131+
Field f1 = SofaRuntimeProperties.class.getDeclaredField("manualReadinessCallbackMap");
132+
f1.setAccessible(true);
133+
Field modifiers = f1.getClass().getDeclaredField("modifiers");
134+
modifiers.setAccessible(true);
135+
modifiers.setInt(f1, f1.getModifiers() & ~Modifier.FINAL);
136+
f1.set(SofaRuntimeProperties.class, new ConcurrentHashMap<>());
137+
} catch (Exception e) {
138+
System.out.println(e);
139+
}
140+
}
141+
123142
@Test
124143
public void testReadinessCheck() throws BeansException {
125144
ReadinessCheckListener readinessCheckListener = applicationContext

‎sofa-boot-project/sofa-boot/src/main/java/com/alipay/sofa/boot/constant/SofaBootConstants.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,21 @@ public class SofaBootConstants {
200200
*/
201201
public static final String SOFABOOT_HEALTH_CHECK_NOT_READY_KEY = "HEALTH-CHECK-NOT-READY";
202202

203+
/**
204+
* health check timeout key
205+
*/
206+
public static final String SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY = "HEALTH-CHECK-TIMEOUT";
207+
203208
/**
204209
* health check not ready result
205210
*/
206211
public static final String SOFABOOT_HEALTH_CHECK_NOT_READY_MSG = "App is still in startup process, please try later!";
207212

213+
/**
214+
* health check timeout result
215+
*/
216+
public static final String SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG = "Timeout when wait for readiness check result!";
217+
208218
/** framework constants **/
209219
public static String APPLICATION = "SOFABOOT-APPLICATION";
210220
public static String PROCESSORS_OF_ROOT_APPLICATION_CONTEXT = "PROCESSORS_OF_ROOT_APPLICATION_CONTEXT";

0 commit comments

Comments
 (0)