diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml index 2e915dc95..1e6fea731 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml @@ -31,7 +31,7 @@ zookeeper.connect - <#list zk_server_list as host>${host}:2181<#sep>,<#else>localhost:2181]]> + localhost:2181 Zookeeper also allows you to add a "chroot" path which will make all kafka data for this cluster appear under a particular path. @@ -43,14 +43,14 @@ listeners - ${host}:9092<#else>localhost:9092]]> + PLAINTEXT://${host}:9092 host and port where kafka broker will be accepting connections. localhost will be substituted with hostname. advertised.listeners - ${host}:9092<#else>localhost:9092]]> + PLAINTEXT://${host}:9092 Listeners to publish to ZooKeeper for clients to use, if different than the listeners config property. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml index 405ac5144..5784c0cc5 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/configuration/solr.in.xml @@ -140,8 +140,7 @@ solr_zookeeper_quorum - - <#list zookeeper_quorum as host>${host}:2181<#sep>,<#else>localhost:2181]]> + localhost:2181 Placeholder for Solr Zookeeper connection string. (Use the cluster one by default, you can override this with a custom one if ZK needs to be external) @@ -187,7 +186,7 @@ solr_extra_java_opts - + Extra Solr java options (e.g.: -Dproperty=value), that will be added to SOLR_OPTS environment variable @@ -195,11 +194,11 @@ solr_gc_log_opts - + solr_gc_tune - + solr_min_mem diff --git a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml index 2e915dc95..1e6fea731 100644 --- a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml +++ b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-broker.xml @@ -31,7 +31,7 @@ zookeeper.connect - <#list zk_server_list as host>${host}:2181<#sep>,<#else>localhost:2181]]> + localhost:2181 Zookeeper also allows you to add a "chroot" path which will make all kafka data for this cluster appear under a particular path. @@ -43,14 +43,14 @@ listeners - ${host}:9092<#else>localhost:9092]]> + PLAINTEXT://${host}:9092 host and port where kafka broker will be accepting connections. localhost will be substituted with hostname. advertised.listeners - ${host}:9092<#else>localhost:9092]]> + PLAINTEXT://${host}:9092 Listeners to publish to ZooKeeper for clients to use, if different than the listeners config property. diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java index 42f1a8e6c..d1027b124 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java @@ -24,11 +24,15 @@ import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.springframework.util.CollectionUtils; + import com.google.auto.service.AutoService; import lombok.Getter; import lombok.NoArgsConstructor; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Getter @AutoService(Params.class) @@ -49,12 +53,20 @@ public KafkaParams(ComponentCommandPayload componentCommandPayload) { globalParamsMap.put("java_home", javaHome()); globalParamsMap.put("kafka_conf_dir", confDir()); globalParamsMap.put("security_enabled", false); + globalParamsMap.put("host", hostname()); } @GlobalParams public Map kafkaBroker() { Map kafkaBroker = LocalSettings.configurations(getServiceName(), "kafka-broker"); kafkaDataDir = (String) kafkaBroker.get("log.dirs"); + + List zookeeperServerHosts = LocalSettings.componentHosts("zookeeper_server"); + if (!CollectionUtils.isEmpty(zookeeperServerHosts)) { + String zkServerListStr = + zookeeperServerHosts.stream().map(s -> s + ":2181").collect(Collectors.joining(",")); + kafkaBroker.put("zookeeper.connect", zkServerListStr); + } return kafkaBroker; } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java index e7e13b39e..adddb041d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java @@ -21,7 +21,6 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.enums.ConfigType; import org.apache.bigtop.manager.stack.core.spi.param.Params; -import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; import lombok.AccessLevel; @@ -29,9 +28,6 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; @@ -53,10 +49,6 @@ public static ShellResult configure(Params params) { LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - List zookeeperServerHosts = LocalSettings.componentHosts("zookeeper_server"); - Map paramMap = new HashMap<>(); - paramMap.put("zk_server_list", zookeeperServerHosts); - paramMap.put("host", kafkaParams.hostname()); LinuxFileUtils.toFile( ConfigType.PROPERTIES, MessageFormat.format("{0}/server.properties", confDir), @@ -64,7 +56,7 @@ public static ShellResult configure(Params params) { kafkaGroup, PERMISSION_644, kafkaParams.kafkaBroker(), - paramMap); + kafkaParams.getGlobalParamsMap()); LinuxFileUtils.toFileByTemplate( kafkaParams.getKafkaEnvContent(), diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java index e4408d241..b43252c87 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java @@ -32,6 +32,7 @@ import java.text.MessageFormat; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Getter @Slf4j @@ -71,12 +72,17 @@ public Map solrEnv() { solrPort = (String) solrEnv.get("solr_port"); solrPidFile = solrPidDir + "/solr-" + solrPort + ".pid"; - List ZookeeperServerHosts = LocalSettings.componentHosts("zookeeper_server"); - Map ZKPort = LocalSettings.configurations("zookeeper", "zoo.cfg"); - String clientPort = (String) ZKPort.get("clientPort"); + List zookeeperServerHosts = LocalSettings.componentHosts("zookeeper_server"); + Map zKPort = LocalSettings.configurations("zookeeper", "zoo.cfg"); + String clientPort = (String) zKPort.get("clientPort"); zNode = (String) solrEnv.get("solr_znode"); - zkString = MessageFormat.format("{0}:{1}", ZookeeperServerHosts.get(0), clientPort); + zkString = MessageFormat.format("{0}:{1}", zookeeperServerHosts.get(0), clientPort); zkHost = MessageFormat.format("{0}{1}", zkString, zNode); + + String zkServerListStr = + zookeeperServerHosts.stream().map(s -> s + ":2181").collect(Collectors.joining(",")); + solrEnv.put("solr_zookeeper_quorum", zkServerListStr); + return solrEnv; }