Merge branch 'release-12.x'

* release-12.x:
  [release] prepare release for proxy-12.2
  update open_source_licenses.txt for release 12.2
  Fix failing HttpEndtoEndTest (#835)
  12.2 version (#834)
  [release] prepare for next development iteration

Author: laullon

docker/Dockerfile

@@ -16,6 +16,9 @@ RUN chown -R wavefront:wavefront /opt/java/openjdk/lib/security/cacerts
 RUN mkdir -p /var/spool/wavefront-proxy
 RUN chown -R wavefront:wavefront /var/spool/wavefront-proxy
 
+RUN mkdir -p /var/log/wavefront
+RUN chown -R wavefront:wavefront /var/log/wavefront
+
 # Run the agent
 EXPOSE 3878
 EXPOSE 2878

open_source_licenses.txt

@@ -8,11 +8,11 @@
 # suffix -- or Wavefront provides the URL.
 server=CHANGE_ME
 
-# The hostname will be used to identify the internal proxy statistics around point rates, JVM info, etc.
+# The proxyname will be used to identify the internal proxy statistics around point rates, JVM info, etc.
 # We strongly recommend setting this to a name that is unique among your entire infrastructure, to make this
-# proxy easy to identify. This hostname does not need to correspond to any actual hostname or DNS entry; it's merely
+# proxy easy to identify. This proxyname does not need to correspond to any actual proxyname or DNS entry; it's merely
 # a string that we pass with the internal stats and ~proxy.* metrics.
-#hostname=my.proxy.host.com
+#proxyname=my.proxy.name.com
 
 # The Token is any valid API token for an account that has *Proxy Management* permissions. To get to the token:
 # 1. Click the gear icon at the top right in the Wavefront UI.
@@ -41,9 +41,9 @@ pushListenerPorts=2878
 #graphitePorts=2003
 ## Comma-separated list of ports to listen on for Graphite pickle formatted data (from carbon-relay) (Default: none)
 #picklePorts=2004
-## Which fields (1-based) should we extract and concatenate (with dots) as the hostname?
+## Which fields (1-based) should we extract and concatenate (with dots) as the proxyname?
 #graphiteFormat=2
-## Which characters should be replaced by dots in the hostname, after extraction? (Default: _)
+## Which characters should be replaced by dots in the proxyname, after extraction? (Default: _)
 #graphiteDelimiters=_
 ## Comma-separated list of fields (metric segments) to remove (1-based). This is an optional setting. (Default: none)
 #graphiteFieldsToRemove=3,4,5
@@ -83,7 +83,7 @@ pushListenerPorts=2878
 ## source= or host= (preferring source=) and treat that as the source/host within Wavefront.
 ## customSourceTags is a comma-separated, ordered list of additional tag keys to use if neither
 ## source= or host= is present
-#customSourceTags=fqdn, hostname
+#customSourceTags=fqdn, proxyname
 
 ## The prefix should either be left undefined, or can be any  prefix you want
 ## prepended to all data points coming through this proxy (such as `production`).

pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default

@@ -1,10 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <groupId>com.wavefront</groupId>
   <artifactId>proxy</artifactId>
-  <version>12.1</version>
+  <version>12.2</version>
 
   <name>Wavefront Proxy</name>
   <description>Service for batching and relaying metric traffic to Wavefront</description>
@@ -263,7 +265,7 @@
       <dependency>
         <groupId>com.google.protobuf</groupId>
         <artifactId>protobuf-bom</artifactId>
-        <version>3.21.0</version>
+        <version>3.21.12</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
@@ -280,7 +282,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-bom</artifactId>
-        <version>4.1.77.Final</version>
+        <version>4.1.89.Final</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
@@ -386,7 +388,7 @@
     <dependency>
       <groupId>com.wavefront</groupId>
       <artifactId>java-lib</artifactId>
-      <version>2022-11.1</version>
+      <version>2023-02.1</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.module</groupId>
@@ -444,7 +446,7 @@
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-sqs</artifactId>
-      <version>1.12.229</version>
+      <version>1.12.297</version>
       <scope>compile</scope>
     </dependency>
     <dependency>

proxy/pom.xml

@@ -12,7 +12,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.wavefront.agent.api.APIContainer;
 import com.wavefront.agent.config.LogsIngestionConfig;
@@ -42,7 +41,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.UUID;
@@ -51,8 +49,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 import javax.net.ssl.SSLException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -67,10 +63,8 @@ public abstract class AbstractAgent {
   final Counter activeListeners =
       Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName);
   /** A set of commandline parameters to hide when echoing command line arguments */
-  protected static final Set<String> PARAMETERS_TO_HIDE =
-      ImmutableSet.of("-t", "--token", "--proxyPassword");
-
   protected final ProxyConfig proxyConfig = new ProxyConfig();
+
   protected APIContainer apiContainer;
   protected final List<ExecutorService> managedExecutors = new ArrayList<>();
   protected final List<Runnable> shutdownTasks = new ArrayList<>();
@@ -223,6 +217,15 @@ private void postProcessConfig() {
   @VisibleForTesting
   void parseArguments(String[] args) {
     // read build information and print version.
+    String versionStr =
+        "Wavefront Proxy version "
+            + getBuildVersion()
+            + " (pkg:"
+            + getPackage()
+            + ")"
+            + ", runtime: "
+            + getJavaVersion();
+    logger.info(versionStr);
     try {
       if (!proxyConfig.parseArguments(args, this.getClass().getCanonicalName())) {
         System.exit(0);
@@ -231,13 +234,6 @@ void parseArguments(String[] args) {
       logger.severe("Parameter exception: " + e.getMessage());
       System.exit(1);
     }
-    logger.info(
-        "Arguments: "
-            + IntStream.range(0, args.length)
-                .mapToObj(
-                    i -> (i > 0 && PARAMETERS_TO_HIDE.contains(args[i - 1])) ? "<HIDDEN>" : args[i])
-                .collect(Collectors.joining(" ")));
-    proxyConfig.verifyAndInit();
   }
 
   /**

proxy/src/main/java/com/wavefront/agent/AbstractAgent.java

@@ -0,0 +1,33 @@
+package com.wavefront.agent;
+
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.common.TaggedMetricName;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricsRegistry;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/** @author Sumit Deo ([email protected]) */
+public class LogsUtil {
+
+  public static final Set<DataFormat> LOGS_DATA_FORMATS =
+      new HashSet<>(
+          Arrays.asList(
+              DataFormat.LOGS_JSON_ARR,
+              DataFormat.LOGS_JSON_LINES,
+              DataFormat.LOGS_JSON_CLOUDWATCH));
+
+  public static Counter getOrCreateLogsCounterFromRegistry(
+      MetricsRegistry registry, DataFormat format, String group, String name) {
+    return registry.newCounter(
+        new TaggedMetricName(group, name, "format", format.name().toLowerCase()));
+  }
+
+  public static Histogram getOrCreateLogsHistogramFromRegistry(
+      MetricsRegistry registry, DataFormat format, String group, String name) {
+    return registry.newHistogram(
+        new TaggedMetricName(group, name, "format", format.name().toLowerCase()), false);
+  }
+}

proxy/src/main/java/com/wavefront/agent/LogsUtil.java

@@ -9,6 +9,7 @@
 import com.google.common.base.Throwables;
 import com.google.common.collect.Maps;
 import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.preprocessor.PreprocessorConfigManager;
 import com.wavefront.api.agent.AgentConfiguration;
 import com.wavefront.common.Clock;
 import com.wavefront.common.NamedThreadFactory;
@@ -54,18 +55,16 @@ public class ProxyCheckInScheduler {
   private final BiConsumer<String, AgentConfiguration> agentConfigurationConsumer;
   private final Runnable shutdownHook;
   private final Runnable truncateBacklog;
-  private String hostname;
-  private String serverEndpointUrl = null;
-
-  private volatile JsonNode agentMetrics;
   private final AtomicInteger retries = new AtomicInteger(0);
   private final AtomicLong successfulCheckIns = new AtomicLong(0);
-  private boolean retryImmediately = false;
-
   /** Executors for support tasks. */
   private final ScheduledExecutorService executor =
       Executors.newScheduledThreadPool(2, new NamedThreadFactory("proxy-configuration"));
 
+  private String serverEndpointUrl = null;
+  private volatile JsonNode agentMetrics;
+  private boolean retryImmediately = false;
+
   /**
    * @param proxyId Proxy UUID.
    * @param proxyConfig Proxy settings.
@@ -87,13 +86,18 @@ public ProxyCheckInScheduler(
     this.agentConfigurationConsumer = agentConfigurationConsumer;
     this.shutdownHook = shutdownHook;
     this.truncateBacklog = truncateBacklog;
+
     updateProxyMetrics();
+
     Map<String, AgentConfiguration> configList = checkin();
+    sendConfig();
+
     if (configList == null && retryImmediately) {
       // immediately retry check-ins if we need to re-attempt
       // due to changing the server endpoint URL
       updateProxyMetrics();
       configList = checkin();
+      sendPreprocessorRules();
     }
     if (configList != null && !configList.isEmpty()) {
       logger.info("initial configuration is available, setting up proxy");
@@ -102,7 +106,6 @@ public ProxyCheckInScheduler(
         successfulCheckIns.incrementAndGet();
       }
     }
-    hostname = proxyConfig.getHostname();
   }
 
   /** Set up and schedule regular check-ins. */
@@ -126,6 +129,27 @@ public void shutdown() {
     executor.shutdown();
   }
 
+  private void sendConfig() {
+    try {
+      apiContainer
+          .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME)
+          .proxySaveConfig(proxyId, proxyConfig.getJsonConfig());
+    } catch (javax.ws.rs.NotFoundException ex) {
+      logger.debug("'proxySaveConfig' api end point not found", ex);
+    }
+  }
+
+  /** Send preprocessor rules */
+  private void sendPreprocessorRules() {
+    try {
+      apiContainer
+          .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME)
+          .proxySavePreprocessorRules(proxyId, PreprocessorConfigManager.getJsonRules());
+    } catch (javax.ws.rs.NotFoundException ex) {
+      logger.debug("'proxySavePreprocessorRules' api end point not found", ex);
+    }
+  }
+
   /**
    * Perform agent check-in and fetch configuration of the daemon from remote server.
    *
@@ -307,6 +331,7 @@ private Map<String, AgentConfiguration> checkin() {
   void updateConfiguration() {
     try {
       Map<String, AgentConfiguration> configList = checkin();
+      sendPreprocessorRules();
       if (configList != null && !configList.isEmpty()) {
         AgentConfiguration config;
         for (Map.Entry<String, AgentConfiguration> configEntry : configList.entrySet()) {
@@ -345,8 +370,7 @@ void updateProxyMetrics() {
     try {
       Map<String, String> pointTags = new HashMap<>(proxyConfig.getAgentMetricsPointTags());
       pointTags.put("processId", ID);
-      // MONIT-27856 Adds real hostname (fqdn if possible) as internal metric tag
-      pointTags.put("hostname", hostname);
+      pointTags.put("hostname", proxyConfig.getHostname());
       synchronized (executor) {
         agentMetrics =
             JsonMetricsGenerator.generateJsonMetrics(