feat(configservice): Enhanced instance configuration auditing and caching

CHANGES.md

@@ -7,6 +7,7 @@ Apollo 2.5.0
 ------------------
 * [Refactor: align permission validator api between openapi and portal](https://github.com/apolloconfig/apollo/pull/5337)
 * [Feature: Provide a new configfiles API to return the raw content of configuration files directly](https://github.com/apolloconfig/apollo/pull/5336)
+* [Feature: Enhanced instance configuration auditing and caching](https://github.com/apolloconfig/apollo/pull/5361)
 
 ------------------
 All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/16?closed=1)

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java

@@ -60,6 +60,11 @@ public class BizConfig extends RefreshableConfig {
   private static final int DEFAULT_LONG_POLLING_TIMEOUT = 60; //60s
   public static final int DEFAULT_RELEASE_HISTORY_RETENTION_SIZE = -1;
 
+  private static final int DEFAULT_INSTANCE_CONFIG_AUDIT_MAX_SIZE = 10000;
+  private static final int DEFAULT_INSTANCE_CACHE_MAX_SIZE = 50000;
+  private static final int DEFAULT_INSTANCE_CONFIG_CACHE_MAX_SIZE = 50000;
+  private static final int DEFAULT_INSTANCE_CONFIG_AUDIT_TIME_THRESHOLD_IN_MINUTE = 10;//10 minutes
+
   private static final Gson GSON = new Gson();
 
   private static final Type appIdValueLengthOverrideTypeReference =
@@ -100,7 +105,8 @@ public int grayReleaseRuleScanInterval() {
   public long longPollingTimeoutInMilli() {
     int timeout = getIntProperty("long.polling.timeout", DEFAULT_LONG_POLLING_TIMEOUT);
     // java client's long polling timeout is 90 seconds, so server side long polling timeout must be less than 90
-    return 1000 * checkInt(timeout, 1, 90, DEFAULT_LONG_POLLING_TIMEOUT);
+    timeout = checkInt(timeout, 1, 90, DEFAULT_LONG_POLLING_TIMEOUT);
+    return TimeUnit.SECONDS.toMillis(timeout);
   }
 
   public int itemKeyLengthLimit() {
@@ -240,6 +246,27 @@ public boolean isConfigServiceCacheKeyIgnoreCase() {
     return getBooleanProperty("config-service.cache.key.ignore-case", false);
   }
 
+  public int getInstanceConfigAuditMaxSize() {
+    int auditMaxSize = getIntProperty("instance.config.audit.max.size", DEFAULT_INSTANCE_CONFIG_AUDIT_MAX_SIZE);
+    return checkInt(auditMaxSize, 10, Integer.MAX_VALUE, DEFAULT_INSTANCE_CONFIG_AUDIT_MAX_SIZE);
+  }
+
+  public int getInstanceCacheMaxSize() {
+    int cacheMaxSize = getIntProperty("instance.cache.max.size", DEFAULT_INSTANCE_CACHE_MAX_SIZE);
+    return checkInt(cacheMaxSize, 10, Integer.MAX_VALUE, DEFAULT_INSTANCE_CACHE_MAX_SIZE);
+  }
+
+  public int getInstanceConfigCacheMaxSize() {
+    int cacheMaxSize = getIntProperty("instance.config.cache.max.size", DEFAULT_INSTANCE_CONFIG_CACHE_MAX_SIZE);
+    return checkInt(cacheMaxSize, 10, Integer.MAX_VALUE, DEFAULT_INSTANCE_CONFIG_CACHE_MAX_SIZE);
+  }
+
+  public long getInstanceConfigAuditTimeThresholdInMilli() {
+    int timeThreshold = getIntProperty("instance.config.audit.time.threshold.minutes", DEFAULT_INSTANCE_CONFIG_AUDIT_TIME_THRESHOLD_IN_MINUTE);
+    timeThreshold = checkInt(timeThreshold, 5, Integer.MAX_VALUE, DEFAULT_INSTANCE_CONFIG_AUDIT_TIME_THRESHOLD_IN_MINUTE);
+    return TimeUnit.MINUTES.toMillis(timeThreshold);
+  }
+
   int checkInt(int value, int min, int max, int defaultValue) {
     if (value >= min && value <= max) {
       return value;

apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/InstanceConfigAuditUtil.java

@@ -16,6 +16,7 @@
  */
 package com.ctrip.framework.apollo.configservice.util;
 
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
 import com.google.common.cache.Cache;
@@ -30,6 +31,9 @@
 import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory;
 import com.ctrip.framework.apollo.tracer.Tracer;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics;
+import javax.annotation.PostConstruct;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.stereotype.Service;
@@ -48,29 +52,33 @@
  */
 @Service
 public class InstanceConfigAuditUtil implements InitializingBean {
-  private static final int INSTANCE_CONFIG_AUDIT_MAX_SIZE = 10000;
-  private static final int INSTANCE_CACHE_MAX_SIZE = 50000;
-  private static final int INSTANCE_CONFIG_CACHE_MAX_SIZE = 50000;
-  private static final long OFFER_TIME_LAST_MODIFIED_TIME_THRESHOLD_IN_MILLI = TimeUnit.MINUTES.toMillis(10);//10 minutes
+
   private static final Joiner STRING_JOINER = Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR);
   private final ExecutorService auditExecutorService;
   private final AtomicBoolean auditStopped;
-  private BlockingQueue<InstanceConfigAuditModel> audits = Queues.newLinkedBlockingQueue
-      (INSTANCE_CONFIG_AUDIT_MAX_SIZE);
+  private BlockingQueue<InstanceConfigAuditModel> audits;
   private Cache<String, Long> instanceCache;
   private Cache<String, String> instanceConfigReleaseKeyCache;
 
   private final InstanceService instanceService;
+  private final BizConfig bizConfig;
+  private final MeterRegistry meterRegistry;
 
-  public InstanceConfigAuditUtil(final InstanceService instanceService) {
+  public InstanceConfigAuditUtil(final InstanceService instanceService, final BizConfig bizConfig, final MeterRegistry meterRegistry) {
     this.instanceService = instanceService;
+    this.bizConfig = bizConfig;
+    this.meterRegistry = meterRegistry;
+
+    audits = Queues.newLinkedBlockingQueue(this.bizConfig.getInstanceConfigAuditMaxSize());
     auditExecutorService = Executors.newSingleThreadExecutor(
         ApolloThreadFactory.create("InstanceConfigAuditUtil", true));
     auditStopped = new AtomicBoolean(false);
-    instanceCache = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS)
-        .maximumSize(INSTANCE_CACHE_MAX_SIZE).build();
-    instanceConfigReleaseKeyCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS)
-        .maximumSize(INSTANCE_CONFIG_CACHE_MAX_SIZE).build();
+  }
+
+  @PostConstruct
+  void initialize() {
+    buildInstanceCache();
+    buildInstanceConfigReleaseKeyCache();
   }
 
   public boolean audit(String appId, String clusterName, String dataCenter, String
@@ -138,8 +146,7 @@ void doAudit(InstanceConfigAuditModel auditModel) {
   }
 
   private boolean offerTimeAndLastModifiedTimeCloseEnough(Date offerTime, Date lastModifiedTime) {
-    return (offerTime.getTime() - lastModifiedTime.getTime()) <
-        OFFER_TIME_LAST_MODIFIED_TIME_THRESHOLD_IN_MILLI;
+    return (offerTime.getTime() - lastModifiedTime.getTime()) < this.bizConfig.getInstanceConfigAuditTimeThresholdInMilli();
   }
 
   private long prepareInstanceId(InstanceConfigAuditModel auditModel) {
@@ -178,6 +185,30 @@ public void afterPropertiesSet() throws Exception {
     });
   }
 
+  private void buildInstanceCache() {
+    CacheBuilder instanceCacheBuilder = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS)
+        .maximumSize(this.bizConfig.getInstanceCacheMaxSize());
+    if (bizConfig.isConfigServiceCacheStatsEnabled()) {
+      instanceCacheBuilder.recordStats();
+    }
+    instanceCache = instanceCacheBuilder.build();
+    if (bizConfig.isConfigServiceCacheStatsEnabled()) {
+      GuavaCacheMetrics.monitor(meterRegistry, instanceCache, "instance_cache");
+    }
+  }
+
+  private void buildInstanceConfigReleaseKeyCache() {
+    CacheBuilder instanceConfigReleaseKeyCacheBuilder = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS)
+        .maximumSize(this.bizConfig.getInstanceConfigCacheMaxSize());
+    if (bizConfig.isConfigServiceCacheStatsEnabled()) {
+      instanceConfigReleaseKeyCacheBuilder.recordStats();
+    }
+    instanceConfigReleaseKeyCache = instanceConfigReleaseKeyCacheBuilder.build();
+    if (bizConfig.isConfigServiceCacheStatsEnabled()) {
+      GuavaCacheMetrics.monitor(meterRegistry, instanceCache, "instance_config_cache");
+    }
+  }
+
   private String assembleInstanceKey(String appId, String cluster, String ip, String datacenter) {
     List<String> keyParts = Lists.newArrayList(appId, cluster, ip);
     if (!Strings.isNullOrEmpty(datacenter)) {

apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/util/InstanceConfigAuditUtilTest.java

@@ -16,9 +16,11 @@
  */
 package com.ctrip.framework.apollo.configservice.util;
 
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Instance;
 import com.ctrip.framework.apollo.biz.entity.InstanceConfig;
 import com.ctrip.framework.apollo.biz.service.InstanceService;
+import io.micrometer.core.instrument.MeterRegistry;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -41,6 +43,10 @@ public class InstanceConfigAuditUtilTest {
 
   @Mock
   private InstanceService instanceService;
+  @Mock
+  private BizConfig bizConfig;
+  @Mock
+  private MeterRegistry meterRegistry;
   private BlockingQueue<InstanceConfigAuditUtil.InstanceConfigAuditModel> audits;
 
   private String someAppId;
@@ -56,7 +62,12 @@ public class InstanceConfigAuditUtilTest {
 
   @Before
   public void setUp() throws Exception {
-    instanceConfigAuditUtil = new InstanceConfigAuditUtil(instanceService);
+    when(bizConfig.getInstanceConfigAuditMaxSize()).thenReturn(100);
+    when(bizConfig.getInstanceCacheMaxSize()).thenReturn(100);
+    when(bizConfig.getInstanceConfigCacheMaxSize()).thenReturn(100);
+
+    instanceConfigAuditUtil = new InstanceConfigAuditUtil(instanceService, bizConfig, meterRegistry);
+    instanceConfigAuditUtil.initialize();
 
     audits = (BlockingQueue<InstanceConfigAuditUtil.InstanceConfigAuditModel>)
         ReflectionTestUtils.getField(instanceConfigAuditUtil, "audits");

docs/en/deployment/distributed-deployment-guide.md

@@ -1632,3 +1632,33 @@ json
 }
 ```
 The above configuration specifies that the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace1, and branchName=bj is 10, and the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace2, and branchName=bj is 20. In general, branchName equals clusterName. It is only different during gray release, where the branchName needs to be confirmed by querying the ReleaseHistory table in the database.
+
+### 3.2.14 instance.config.audit.max.size - The size of the queue for clients to pull audit records
+
+> For version 2.5.0 and above
+
+The default value is 10000 and the minimum value is 10. It is used to control the queue size for the client to pull audit records. When the queue size is exceeded, the earliest audit record will be discarded.
+
+After the modification, you need to restart for it to take effect.
+
+### 3.2.15 instance.cache.max.size - The maximum number of caches for the instance
+
+> For version 2.5.0 and above
+
+The default value is 50000, and the minimum value is 10. It is used to control the maximum number of instance caches. When the cache exceeds the maximum capacity, the cache eviction mechanism is triggered.
+
+After the modification, you need to restart for it to take effect.
+
+### 3.2.16 instance.config.cache.max.size - The maximum number of caches for the instance config
+
+> For version 2.5.0 and above
+
+The default value is 50000 and the minimum value is 10. It is used to control the maximum number of caches for the instance config. When the cache exceeds the maximum capacity, the cache eviction mechanism is triggered.
+
+After the modification, you need to restart for it to take effect.
+
+### 3.2.17 instance.config.audit.time.threshold.minutes - The interval between instances pulling audit records
+
+> For version 2.5.0 and above
+
+The time threshold unit is minutes, the default is 10, and the minimum is 5. It is used to control when saving/updating the client pull configuration audit record. When the interval between two request records is greater than this value, the pull record will be saved/updated. When it is less than this value, the pull record will not be saved/updated.

docs/zh/deployment/distributed-deployment-guide.md

@@ -1571,3 +1571,35 @@ json
 }
 ```
 以上配置指定了 appId=kl、clusterName=bj、namespaceName=namespace1、branchName=bj 的发布历史保留数量为 10，appId=kl、clusterName=bj、namespaceName=namespace2、branchName=bj 的发布历史保留数量为 20，branchName 一般等于 clusterName，只有灰度发布时才会不同，灰度发布的 branchName 需要查询数据库 ReleaseHistory 表确认。
+
+### 3.2.14 instance.config.audit.max.size - 客户端拉取审计记录的队列大小
+
+> 适用于2.5.0及以上版本
+
+默认为 10000，最小为10，用于控制客户端拉取审计记录的队列大小，超过队列大小后会丢弃最早的审计记录。
+
+修改完需要重启生效。
+
+### 3.2.15 instance.cache.max.size - 实例缓存的最大数量
+
+> 适用于2.5.0及以上版本
+
+默认为 50000，最小为10，用于控制实例缓存的最大数量，当缓存超过最大容量时，会触发 缓存淘汰（Eviction） 机制。
+
+修改完需要重启生效。
+
+
+### 3.2.16 instance.config.cache.max.size - 实例配置的缓存最大数量
+
+> 适用于2.5.0及以上版本
+
+默认为 50000，最小为10，用于控制实例配置的缓存最大数量，当缓存超过最大容量时，会触发 缓存淘汰（Eviction） 机制。
+
+修改完需要重启生效。
+
+
+### 3.2.17 instance.config.audit.time.threshold.minutes - 实例拉取审计记录的间隔时间
+
+> 适用于2.5.0及以上版本
+
+时间阈值单位为分钟，默认为 10，最小为5，用于控制在保存/更新客户端拉取配置审计记录时，当2次请求记录间隔大于该值时，才会保存/更新拉取记录，小于该值时，不会保存/更新拉取记录。