From 2a422f9478404fd4157e934072933329313e938e Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Mon, 16 Jun 2025 11:01:45 +0800 Subject: [PATCH] BIGTOP-4442: Support resolve nested variables for service config --- .../stack/core/executor/StackExecutor.java | 1 + .../stack/core/spi/param/BaseParams.java | 84 ++++++++++++++++++- .../manager/stack/core/spi/param/Params.java | 2 + .../stack/core/spi/param/BaseParamsTest.java | 22 +++++ .../infra/v1_0_0/grafana/GrafanaParams.java | 6 +- pom.xml | 2 - 6 files changed, 109 insertions(+), 8 deletions(-) diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java index 4bfde11bf..16067ffdd 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java @@ -83,6 +83,7 @@ public static ShellResult execute(ComponentCommandPayload payload) { .getClass() .getDeclaredConstructor(ComponentCommandPayload.class) .newInstance(payload); + params.initGlobalParams(); if (Environments.isDevMode()) { log.info("Executing {}::{} on dev mode", script.getName(), method.getName()); return ShellResult.success(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java index 9f6f47a0c..80693276c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java @@ -29,7 +29,6 @@ import org.apache.bigtop.manager.stack.core.exception.StackException; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; -import lombok.Getter; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -44,17 +43,49 @@ @NoArgsConstructor public abstract class BaseParams implements Params { - @Getter protected final Map globalParamsMap = new HashMap<>(); + private boolean globalParamsResolved = false; + + private static int CURRENT_RESOLVING_DEPTH = 0; + private static final int MAX_RECURSION_DEPTH = 5; + public static final String LIMITS_CONF_DIR = "/etc/security/limits.d"; protected ComponentCommandPayload payload; - @SuppressWarnings("unchecked") protected BaseParams(ComponentCommandPayload payload) { this.payload = payload; + } + + public void putGlobalParam(String key, Object value) { + if (value instanceof String s) { + if (s.contains("${") && s.contains("}")) { + globalParamsResolved = false; + } + } + + globalParamsMap.put(key, value); + } + + public Object getGlobalParam(String key) { + if (!globalParamsResolved) { + recursivelyResolveGlobalParams(); + } + + return globalParamsMap.get(key); + } + + public Map getGlobalParamsMap() { + if (!globalParamsResolved) { + recursivelyResolveGlobalParams(); + } + + return globalParamsMap; + } + @SuppressWarnings("unchecked") + public void initGlobalParams() { // Global Parameters Injection Method[] declaredMethods = this.getClass().getDeclaredMethods(); for (Method declaredMethod : declaredMethods) { @@ -67,7 +98,54 @@ protected BaseParams(ComponentCommandPayload payload) { log.warn("Get {} Params error", declaredMethod, e); } } + globalParamsMap.remove("content"); + + recursivelyResolveGlobalParams(); + globalParamsResolved = true; + } + + private void recursivelyResolveGlobalParams() { + CURRENT_RESOLVING_DEPTH++; + boolean allResolved = true; + for (Map.Entry entry : globalParamsMap.entrySet()) { + if (entry.getValue() instanceof String value) { + while (value.contains("${") && value.contains("}")) { + allResolved = false; + int start = -1, end = -1; + for (int i = 0; i < value.length(); i++) { + if (value.startsWith("${", i)) { + start = i; + } else if (value.charAt(i) == '}') { + end = i; + break; + } + } + if (start != -1 && end != -1) { + String key = value.substring(start + 2, end); + Object resolvedValue = globalParamsMap.get(key); + if (resolvedValue != null) { + value = value.substring(0, start) + resolvedValue + value.substring(end + 1); + entry.setValue(value); + } else { + // Log a warning if the key is not found + log.warn("Parameter '{}' not found for replacement in '{}'", key, entry.getKey()); + break; + } + } + } + } + } + + if (!allResolved) { + if (CURRENT_RESOLVING_DEPTH >= MAX_RECURSION_DEPTH) { + CURRENT_RESOLVING_DEPTH = 0; + } else { + recursivelyResolveGlobalParams(); + } + } else { + CURRENT_RESOLVING_DEPTH = 0; + } } public String hostname() { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java index 231a9dd1b..3ab509dcf 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java @@ -27,6 +27,8 @@ public interface Params extends PrioritySPI { + void initGlobalParams(); + String confDir(); String user(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java index 59d53a818..782e6d36f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mockStatic; @@ -105,6 +106,27 @@ public void tearDown() { netUtilsMockedStatic.close(); } + @Test + public void testInitGlobalParams() { + Map globalParamsMap = mockBaseParams.getGlobalParamsMap(); + globalParamsMap.put("a1", "k1"); + globalParamsMap.put("a2", "k2"); + globalParamsMap.put("a3", "k3"); + globalParamsMap.put("k1_k2", "kk1"); + globalParamsMap.put("kk1_k3", "value"); + globalParamsMap.put("key", "${key3}"); + globalParamsMap.put("key1", "${k1_${a2}}_${a3}"); + globalParamsMap.put("key2", "${${k1_${a2}}_${a3}}"); + globalParamsMap.put("key3", "${key1}"); + globalParamsMap.put("key4", "${not_exists}"); + mockBaseParams.initGlobalParams(); + assertEquals("kk1_k3", globalParamsMap.get("key")); + assertEquals("kk1_k3", globalParamsMap.get("key1")); + assertEquals("value", mockBaseParams.getGlobalParamsMap().get("key2")); + assertEquals("kk1_k3", globalParamsMap.get("key3")); + assertEquals("${not_exists}", globalParamsMap.get("key4")); + } + @Test public void testHostname() { String hostname = mockBaseParams.hostname(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaParams.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaParams.java index 118471c63..ff7e2ca24 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaParams.java @@ -110,11 +110,11 @@ public Map prometheus() { } prometheusServer = prometheusServers.get(0); - Map prometheus_configuration = LocalSettings.configurations("prometheus", "prometheus"); - if (prometheus_configuration == null) { + Map prometheusConf = LocalSettings.configurations("prometheus", "prometheus"); + if (prometheusConf == null) { return configuration; } - prometheusPort = (String) prometheus_configuration.get("port"); + prometheusPort = (String) prometheusConf.get("port"); return configuration; } diff --git a/pom.xml b/pom.xml index ea20a7086..b5f1f4747 100644 --- a/pom.xml +++ b/pom.xml @@ -226,9 +226,7 @@ ${java.version} false - true ${maven.javadoc.skip} - -Xdoclint:none