fix metadata normalization (#43)

Co-authored-by: Joao Grassi <[email protected]>

Author: pirgeo

lib/build.gradle

@@ -20,7 +20,7 @@ plugins {
 }
 
 group 'com.dynatrace.metric.util'
-version = '2.3.0'
+version = '2.4.0'
 
 repositories {
     mavenCentral()

lib/src/main/java/com/dynatrace/metric/util/CodePoints.java

@@ -20,6 +20,7 @@ final class CodePoints {
   static final int EQUALS = "=".codePointAt(0);
   static final int BLANK = " ".codePointAt(0);
   static final int COLON = ":".codePointAt(0);
+  static final int NEWLINE = "\n".codePointAt(0);
 
   static final int ZERO = "0".codePointAt(0);
   static final int NINE = "9".codePointAt(0);
@@ -47,6 +48,8 @@ final class CodePoints {
 
   static final String QUOTATION_MARK = "\"";
   static final String EMPTY_STRING = "";
+  static final String ESCAPED_NEWLINE = "\\n";
+  static final String ESCAPED_QUOTES = "\\\"";
 
   private CodePoints() {}
 }

lib/src/main/java/com/dynatrace/metric/util/MetricLineBuilderImpl.java

@@ -447,7 +447,7 @@ public String build() {
                       this.description));
         } else {
           NormalizationResult normalizationResult =
-              Normalizer.normalizeDimensionValue(
+              Normalizer.normalizeMetadataString(
                   this.description, MetadataLineConstants.Limits.MAX_DESCRIPTION_LENGTH);
           if (normalizationResult.messageType() != NormalizationResult.MessageType.NONE) {
             logger.warning(
@@ -503,7 +503,7 @@ public String build() {
                       this.displayName));
         } else {
           NormalizationResult normalizationResult =
-              Normalizer.normalizeDimensionValue(
+              Normalizer.normalizeMetadataString(
                   this.displayName, MetadataLineConstants.Limits.MAX_DISPLAY_NAME_LENGTH);
           if (normalizationResult.messageType() != NormalizationResult.MessageType.NONE) {
             logger.warning(

lib/src/main/java/com/dynatrace/metric/util/MetricLineConstants.java

@@ -100,6 +100,8 @@ private ValidationMessages() {}
         "Dimension key normalized from '%s' to '%s'";
     static final String DIMENSION_VALUE_NORMALIZED_MESSAGE =
         "Dimension value normalized from '%s' to '%s'";
+    static final String METADATA_VALUE_NORMALIZED_MESSAGE =
+        "Metadata value normalized from '%s' to '%s'";
     static final String DIMENSION_DROPPED_KEY_EMPTY_MESSAGE =
         "Dimension with empty dimension key has been dropped";
     static final String DIMENSION_DROPPED_KEY_EMPTY_WITH_METRIC_KEY_MESSAGE =

lib/src/main/java/com/dynatrace/metric/util/Normalizer.java

@@ -14,6 +14,7 @@
 package com.dynatrace.metric.util;
 
 import com.dynatrace.metric.util.MetricLineConstants.ValidationMessages;
+import java.util.function.Supplier;
 
 /** Offers normalization methods for metric key, dimension key and dimension value */
 final class Normalizer {
@@ -257,6 +258,139 @@ static NormalizationResult normalizeDimensionValue(String value, int maxDimensio
     return normalizeUnquotedStringDimValue(value, maxDimensionValueLength);
   }
 
+  static NormalizationResult normalizeMetadataString(String value, int maxLength) {
+    if (StringValueValidator.isNullOrEmpty(value)) {
+      return NormalizationResult.newValid(CodePoints.EMPTY_STRING);
+    }
+
+    // in a quoted string, quotes do not count towards the string character limit
+    boolean isQuoted =
+        value.startsWith(CodePoints.QUOTATION_MARK) && value.endsWith(CodePoints.QUOTATION_MARK);
+
+    int numValidBytes = 0;
+    boolean needsNormalization = false;
+    boolean needsEscaping = false;
+
+    // if quoted, start at index 1 (the one after the quote)
+    int start = isQuoted ? 1 : 0;
+    // if quoted, stop at length - 1 (the last byte before the quote)
+    int end = isQuoted ? value.length() - 1 : value.length();
+
+    for (int i = start; i < end; ) {
+      final int codePoint = value.codePointAt(i);
+      final int codePointLength = Character.charCount(codePoint);
+
+      // code point is valid but needs escaping
+      if (codePointNeedsEscaping(codePoint)) {
+        needsEscaping = true;
+      }
+
+      // code point is not valid, and needs normalizing (e.g. unicode control char)
+      if (codePointNeedsNormalizing(codePoint)) {
+        // if there are control characters, note that normalization is necessary but don't break yet
+        // this way we know how many characters there will be in the output
+        needsNormalization = true;
+      } else {
+        // if the char is valid, count up
+        numValidBytes += codePointLength;
+        // if the maxLength is reached, break.
+        if (numValidBytes > maxLength) {
+          needsNormalization = true;
+          break;
+        }
+      }
+      i += codePointLength;
+    }
+
+    // return early if no normalization is needed.
+    if (!needsNormalization) {
+      // if the string is already quoted or doesn't need escaping, return as is
+      if (isQuoted || !needsEscaping) {
+        return NormalizationResult.newValid(value);
+      } else {
+        // not quoted, needs escaping, but no normalization (e.g., it contains a '=' character).
+        // turn the string into a quoted string, where the '=' sign does not need escaping.
+        return NormalizationResult.newValid("\"" + value + "\"");
+      }
+    }
+
+    // do normalization
+    StringBuilder builder = new StringBuilder(numValidBytes + 2);
+    builder.append(CodePoints.QUOTATION_MARK);
+    Supplier<String> warningMessageSupplier = null;
+
+    // don't need to escape anything since we're working in a quoted string.
+    for (int i = start; i < end; ) {
+      final int codePoint = value.codePointAt(i);
+      final int codePointLength = Character.charCount(codePoint);
+
+      // special handling: only newlines are considered, since the resulting string will be quoted
+      // either way.
+      if (codePoint == CodePoints.NEWLINE) {
+        // check if the escaped character fits, exit if it does not.
+        if (i + CodePoints.ESCAPED_NEWLINE.length() > maxLength) {
+          break;
+        }
+        builder.append(CodePoints.ESCAPED_NEWLINE);
+      } else if (codePoint == CodePoints.QUOTE) {
+        // check if the escaped character fits, exit if it does not.
+        if (i + CodePoints.ESCAPED_QUOTES.length() > maxLength) {
+          break;
+        }
+        builder.append(CodePoints.ESCAPED_QUOTES);
+      } else if (!codePointNeedsNormalizing(codePoint)) {
+        // code point is valid, check if it fits, then add.
+        if (i + codePointLength > maxLength) {
+          break;
+        }
+        builder.appendCodePoint(codePoint);
+      }
+
+      i += codePointLength;
+    }
+    builder.append(CodePoints.QUOTATION_MARK);
+
+    String normalized = builder.toString();
+    if (normalized.length() == 2) {
+      return NormalizationResult.newInvalid(
+          () -> "no valid characters after normalization (input: " + value + ")");
+    } else
+      return NormalizationResult.newWarning(
+          normalized,
+          () ->
+              String.format(
+                  ValidationMessages.METADATA_VALUE_NORMALIZED_MESSAGE, value, normalized));
+  }
+
+  private static boolean codePointNeedsEscaping(int codePoint) {
+    return codePoint == CodePoints.NEWLINE
+        || codePoint == CodePoints.BLANK
+        || codePoint == CodePoints.COMMA
+        || codePoint == CodePoints.EQUALS
+        || codePoint == CodePoints.QUOTE
+        || codePoint == CodePoints.BACKSLASH;
+  }
+
+  private static boolean codePointNeedsNormalizing(int codePoint) {
+    int type = Character.getType(codePoint);
+
+    // unassigned characters outside the range of Unicode 10.0 "Supplemental Symbols and
+    // Pictographs" are not allowed.
+    if (type == Character.UNASSIGNED) {
+      // Unicode version 10 allowed Supplemental Symbols and pictographs
+      // https://www.unicode.org/charts/PDF/Unicode-10.0/U100-1F900.pdf
+      return codePoint <= CodePoints.UC_SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS_START
+          || codePoint >= CodePoints.UC_SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS_END;
+    }
+
+    return type == Character.CONTROL
+        || type == Character.PRIVATE_USE
+        || type == Character.SURROGATE
+        || type == Character.LINE_SEPARATOR
+        || type == Character.PARAGRAPH_SEPARATOR
+        || codePoint == CodePoints.QUOTE;
+  }
+
   /**
    * Applies normalization to an unquoted string dimension value.
    *

lib/src/test/java/com/dynatrace/metric/util/MetadataBuilderTest.java

@@ -105,8 +105,15 @@ private static Stream<Arguments> provideMetadataInformation() {
                 "do_not_normalize_description",
                 "do_not_normalize_description"),
             Tuple.of(
-                "description to normalize/escape", "escape description", "escape\\ description"),
-            Tuple.of("all invalid characters description", "\u0000", "_"),
+                "description to normalize/escape", "escape description", "\"escape description\""),
+            Tuple.of("all invalid characters description", "\u0000", null),
+            Tuple.of(
+                "multiline description", "multiline\ndescription", "\"multiline\\ndescription\""),
+            Tuple.of("description with emoji (no escaping needed)", "\uD83E\uDD20", "\uD83E\uDD20"),
+            Tuple.of(
+                "description with emoji (with char to escape)",
+                "\uD83E\uDD20 \uD83C\uDF1E",
+                "\"\uD83E\uDD20 \uD83C\uDF1E\""),
             Tuple.of("empty description", "", null),
             Tuple.of("empty quoted description", "\"\"", null),
             Tuple.of("null description", null, null));
@@ -127,9 +134,16 @@ private static Stream<Arguments> provideMetadataInformation() {
             Tuple.of(
                 "displayName to normalize/escape",
                 "escape display name",
-                "escape\\ display\\ name"),
-            Tuple.of("all invalid characters displayName", "\u0000", "_"),
+                "\"escape display name\""),
+            Tuple.of("all invalid characters displayName", "\u0000", null),
             Tuple.of("empty displayName", "", null),
+            Tuple.of(
+                "multiline displayName", "multiline\ndisplayName", "\"multiline\\ndisplayName\""),
+            Tuple.of("displayName with emoji (no escaping needed)", "\uD83E\uDD20", "\uD83E\uDD20"),
+            Tuple.of(
+                "displayName with emoji and escapable char",
+                "\uD83C\uDF35 \uD83C\uDF1E",
+                "\"\uD83C\uDF35 \uD83C\uDF1E\""),
             Tuple.of("empty quoted displayName", "\"\"", null),
             Tuple.of("null displayName", null, null));
 

lib/src/test/java/com/dynatrace/metric/util/MetricBuilderTest.java

@@ -666,7 +666,7 @@ void testCreateMetadataLineWithDescription() throws MetricException {
 
     assertEquals("prefix.name gauge,3", builder.value(3.).build());
     assertEquals(
-        "#prefix.name gauge dt.meta.description=my\\ description\\ goes\\ here",
+        "#prefix.name gauge dt.meta.description=\"my description goes here\"",
         builder.metadata().description("my description goes here").build());
   }
 
@@ -679,7 +679,7 @@ void testCreateMetadataLineWithDisplayName() throws MetricException {
 
     assertEquals("prefix.name gauge,3", builder.value(3.).build());
     assertEquals(
-        "#prefix.name gauge dt.meta.displayName=my\\ displayName\\ goes\\ here",
+        "#prefix.name gauge dt.meta.displayName=\"my displayName goes here\"",
         builder.metadata().displayName("my displayName goes here").build());
   }
 
@@ -692,7 +692,7 @@ void testCreateMetadataLineWithUnitAndDescriptionAndDisplayName() throws MetricE
 
     assertEquals("prefix.name gauge,3", builder.value(3.).build());
     assertEquals(
-        "#prefix.name gauge dt.meta.description=my\\ description\\ goes\\ here,dt.meta.unit=unit,dt.meta.displayName=displayName",
+        "#prefix.name gauge dt.meta.description=\"my description goes here\",dt.meta.unit=unit,dt.meta.displayName=displayName",
         builder
             .metadata()
             .description("my description goes here")