diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/grpc/GrpcClient.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/grpc/GrpcClient.java index 340d17114..8e5e7d3e0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/grpc/GrpcClient.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/grpc/GrpcClient.java @@ -129,6 +129,22 @@ private static ManagedChannel getChannel(String host, Integer grpcPort) { } } + public static void removeChannel(String host) { + ManagedChannel channel = CHANNELS.remove(host); + if (channel != null) { + try { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.warn("Channel shutdown interrupted", e); + } + + BLOCKING_STUBS.remove(host); + ASYNC_STUBS.remove(host); + FUTURE_STUBS.remove(host); + log.info("Channel to host: {} removed.", host); + } + } + private static > AbstractStub.StubFactory getFactory(Class clazz) { return (channel, callOptions) -> { try { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java index bcb49ed23..f9d869676 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.bigtop.manager.server.enums.HostAuthTypeEnum; import org.apache.bigtop.manager.server.enums.InstalledStatusEnum; import org.apache.bigtop.manager.server.exception.ApiException; +import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.model.converter.ComponentConverter; import org.apache.bigtop.manager.server.model.converter.HostConverter; import org.apache.bigtop.manager.server.model.dto.HostDTO; @@ -337,6 +338,9 @@ private Boolean execAgentScript(Long hostId, String action) { String command = path + "/bigtop-manager-agent/bin/agent.sh " + action; command = "export GRPC_PORT=" + grpcPort + " ; " + command; + // Remove channel before operations + GrpcClient.removeChannel(hostname); + ShellResult result = execCommandOnRemoteHost(hostDTO, hostname, command); if (result.getExitCode() != MessageConstants.SUCCESS_CODE) { log.error("Unable to {} agent, hostname: {}, msg: {}", action, hostname, result); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java index 3ac226945..86e3a76a8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java @@ -18,6 +18,7 @@ */ package org.apache.bigtop.manager.server.utils; +import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.PropertyAction; import org.apache.bigtop.manager.server.model.dto.AttrsDTO; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; @@ -29,8 +30,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -79,8 +78,9 @@ public static List mergeServiceConfigs( // To avoid to change the original configs, we use cloned object List mergedConfigs = new ArrayList<>(); for (ServiceConfigDTO oriConfig : oriConfigs) { - ServiceConfigDTO mergedConfig = new ServiceConfigDTO(); - BeanUtils.copyProperties(oriConfig, mergedConfig); + // Deep clone via JSON serialization/deserialization + ServiceConfigDTO mergedConfig = + JsonUtils.readFromString(JsonUtils.writeAsString(oriConfig), ServiceConfigDTO.class); mergedConfigs.add(mergedConfig); } diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/configuration/flink-conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/configuration/flink-conf.xml index a865f5d2d..157ecf009 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/configuration/flink-conf.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/configuration/flink-conf.xml @@ -20,35 +20,25 @@ - jobmanager.archive.fs.dir + jobmanager_archive_fs_dir hdfs:///completed-jobs/ Directory for JobManager to store the archives of completed jobs. - historyserver.archive.fs.dir + historyserver_archive_fs_dir hdfs:///completed-jobs/ Comma separated list of directories to fetch archived jobs from. - historyserver.web.port + historyserver_web_port 8082 The port under which the web-based HistoryServer listens. - historyserver.archive.fs.refresh-interval + historyserver_archive_fs_refresh_interval 10000 Interval in milliseconds for refreshing the monitored directories. - - security.kerberos.login.keytab - Flink keytab path - none - - - security.kerberos.login.principal - Flink principal name - none - content diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml index 170d3a546..c1c1da094 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml @@ -63,9 +63,9 @@ # Set Hadoop-specific environment variables here. USER="$(whoami)" -export JAVA_HOME=${java_home!} -export HADOOP_HOME=${hadoop_home!} -export HADOOP_CONF_DIR=${hadoop_conf_dir!} +export JAVA_HOME=${java_home} +export HADOOP_HOME=${hadoop_home} +export HADOOP_CONF_DIR=${hadoop_conf_dir} export HADOOP_LOG_DIR=${hadoop_log_dir} export HADOOP_PID_DIR=${hadoop_pid_dir} export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max} @@ -79,7 +79,7 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} export HADOOP_ROOT_LOGGER=${hadoop_root_logger} export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} -export LD_LIBRARY_PATH=${hadoop_home!}/lib/native +export LD_LIBRARY_PATH=${hadoop_home}/lib/native ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml index 095de4957..c299d0cce 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml @@ -30,7 +30,7 @@ mapred-env template This is the freemarker template for mapred-env.sh file mapreduce.application.classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/* + ${hadoop_conf_dir},${hadoop_home}/*,${hadoop_home}/lib/* CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. @@ -296,7 +296,7 @@ mapreduce.admin.user.env - LD_LIBRARY_PATH=${hadoop_home!}/lib/native + LD_LIBRARY_PATH=${hadoop_home}/lib/native Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml index a67158646..78ea3d517 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml @@ -93,7 +93,7 @@ yarn.application.classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/* + ${hadoop_conf_dir},${hadoop_home}/*,${hadoop_home}/lib/* Classpath for typical applications. @@ -308,7 +308,7 @@ yarn.timeline-service.enabled - true + false Indicate to clients whether timeline service is enabled or not. If enabled, clients will put entities and events to the timeline server. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hbase/configuration/hbase-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hbase/configuration/hbase-env.xml index ecc41ced0..40d52a6fe 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hbase/configuration/hbase-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hbase/configuration/hbase-env.xml @@ -89,7 +89,7 @@ # Set environment variables here. # The java implementation to use. Java 1.6 required. -export JAVA_HOME=${java_home!} +export JAVA_HOME=${java_home} # HBase Configuration directory export HBASE_CONF_DIR=${hbase_conf_dir} diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml index ac167a788..ca73d071f 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml @@ -52,10 +52,10 @@ # Set KAFKA specific environment variables here. # The java implementation to use. -export JAVA_HOME=${java_home!} +export JAVA_HOME=${java_home} <#noparse>export PATH=$PATH:${JAVA_HOME}/bin -export PID_DIR=${kafka_pid_dir!} -export LOG_DIR=${kafka_log_dir!} +export PID_DIR=${kafka_pid_dir} +export LOG_DIR=${kafka_log_dir} export CLASSPATH=$CLASSPATH:${kafka_conf_dir} ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml index 0f9c9ffab..bf0bf4f89 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml @@ -26,13 +26,13 @@ This is the freemarker template for tez-env.sh file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml index 130ac59cb..5d3efcb55 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml @@ -22,7 +22,7 @@ tez.lib.uris.classpath TEZ Lib URIs Classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!} + ${hadoop_conf_dir},${hadoop_home}/*,${hadoop_home}/lib/*,${tez_home}/*,${tez_home}/lib/*,${tez_conf_dir} Comma-delimited list of the location of the Tez libraries Classpath which will be localized for DAGs. @@ -30,7 +30,7 @@ tez.lib.uris TEZ Lib URIs - ${tez_lib_uris!} + ${tez_lib_uris} Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the @@ -69,7 +69,7 @@ tez.am.launch.cmd-opts - -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts} Java options for the Tez AppMaster process. The Xmx value is derived based on tez.am.resource.memory.mb and is 80% of the value by default. Used only if the value is not specified explicitly by the DAG definition. @@ -84,7 +84,7 @@ tez.am.launch.env - LD_LIBRARY_PATH=${hadoop_home!}/lib/native + LD_LIBRARY_PATH=${hadoop_home}/lib/native Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if @@ -101,7 +101,7 @@ tez.task.launch.cmd-opts - -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts} Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of this value by default. Used only if the value is not specified explicitly by the DAG definition. @@ -116,7 +116,7 @@ tez.task.launch.env - LD_LIBRARY_PATH=${hadoop_home!}/lib/native + LD_LIBRARY_PATH=${hadoop_home}/lib/native Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml index a947ff694..d780de517 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml @@ -37,17 +37,17 @@ zookeeper-env template This is the freemarker template for zookeeper-env.sh file export JAVA=${JAVA_HOME}/bin/java export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* <#if securityEnabled?? && securityEnabled > -export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config=${zk_server_jaas_file!}" -export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config=${zk_client_jaas_file!} -Dzookeeper.sasl.client.username=${zk_principal_user!}" +export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config=${zk_server_jaas_file}" +export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config=${zk_client_jaas_file} -Dzookeeper.sasl.client.username=${zk_principal_user}" ]]> diff --git a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml index ac167a788..ca73d071f 100644 --- a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml +++ b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/kafka/configuration/kafka-env.xml @@ -52,10 +52,10 @@ # Set KAFKA specific environment variables here. # The java implementation to use. -export JAVA_HOME=${java_home!} +export JAVA_HOME=${java_home} <#noparse>export PATH=$PATH:${JAVA_HOME}/bin -export PID_DIR=${kafka_pid_dir!} -export LOG_DIR=${kafka_log_dir!} +export PID_DIR=${kafka_pid_dir} +export LOG_DIR=${kafka_log_dir} export CLASSPATH=$CLASSPATH:${kafka_conf_dir} ]]> diff --git a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml index a947ff694..d780de517 100644 --- a/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml +++ b/bigtop-manager-server/src/test/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zookeeper-env.xml @@ -37,17 +37,17 @@ zookeeper-env template This is the freemarker template for zookeeper-env.sh file export JAVA=${JAVA_HOME}/bin/java export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* <#if securityEnabled?? && securityEnabled > -export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config=${zk_server_jaas_file!}" -export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config=${zk_client_jaas_file!} -Dzookeeper.sasl.client.username=${zk_principal_user!}" +export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config=${zk_server_jaas_file}" +export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config=${zk_client_jaas_file} -Dzookeeper.sasl.client.username=${zk_principal_user}" ]]> diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java index 4cdca6ab6..b0eea0557 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java @@ -47,11 +47,6 @@ public class FlinkParams extends BigtopParams { private String flinkLog4jConsolePropertiesContent; private String flinkLog4jSessionPropertiesContent; - private String jobManagerArchiveFsDir; - private String historyServerWebPort; - private String historyServerArchiveFsDir; - private String historyServerArchiveFsRefreshInterval; - public FlinkParams(ComponentCommandPayload componentCommandPayload) { super(componentCommandPayload); globalParamsMap.put("flink_user", user()); @@ -59,23 +54,12 @@ public FlinkParams(ComponentCommandPayload componentCommandPayload) { globalParamsMap.put("java_home", javaHome()); globalParamsMap.put("hadoop_home", hadoopHome()); globalParamsMap.put("hadoop_conf_dir", hadoopConfDir()); - - globalParamsMap.put("jobmanager_archive_fs_dir", jobManagerArchiveFsDir); - globalParamsMap.put("historyserver_web_port", historyServerWebPort); - globalParamsMap.put("historyserver_archive_fs_dir", historyServerArchiveFsDir); - globalParamsMap.put("historyserver_archive_fs_refresh_interval", historyServerArchiveFsRefreshInterval); } @GlobalParams public Map flinkConf() { Map configurations = LocalSettings.configurations(getServiceName(), "flink-conf"); flinkConfContent = (String) configurations.get("content"); - - jobManagerArchiveFsDir = (String) configurations.get("jobmanager.archive.fs.dir"); - historyServerWebPort = (String) configurations.get("historyserver.web.port"); - historyServerArchiveFsDir = (String) configurations.get("historyserver.archive.fs.dir"); - historyServerArchiveFsRefreshInterval = - (String) configurations.get("historyserver.archive.fs.refresh-interval"); return configurations; }