[EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter

CMakeLists.txt

@@ -448,6 +448,16 @@ if((NOT WITH_API_ONLY) AND WITH_HTTP_CLIENT_CURL)
   message(STATUS "Found CURL: ${CURL_LIBRARIES}, version ${CURL_VERSION}")
 endif()
 
+#
+# Do we need ZLIB ?
+#
+
+if((NOT WITH_API_ONLY) AND WITH_HTTP_CLIENT_CURL)
+  # No specific version required.
+  find_package(ZLIB REQUIRED)
+  message(STATUS "Found ZLIB: ${ZLIB_LIBRARIES}, version ${ZLIB_VERSION}")
+endif()
+
 #
 # Do we need NLOHMANN_JSON ?
 #

bazel/repository.bzl

@@ -183,3 +183,16 @@ def opentelemetry_cpp_deps():
             "https://github.com/opentracing/opentracing-cpp/archive/refs/tags/v1.6.0.tar.gz",
         ],
     )
+
+    # Zlib (optional)
+    maybe(
+        http_archive,
+        name = "zlib",
+        build_file = "@io_opentelemetry_cpp//bazel:zlib.BUILD",
+        sha256 = "d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98",
+        strip_prefix = "zlib-1.2.13",
+        urls = [
+            "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.xz",
+            "https://zlib.net/zlib-1.2.13.tar.xz",
+        ],
+    )

bazel/zlib.BUILD

@@ -0,0 +1,74 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# Builds ZLIB from a distribution.
+# Copied from https://github.com/protocolbuffers/protobuf/blob/master/third_party/zlib.BUILD
+
+licenses(["notice"])  # BSD/MIT-like license (for zlib)
+
+exports_files(["zlib.BUILD"])
+
+_ZLIB_HEADERS = [
+    "crc32.h",
+    "deflate.h",
+    "gzguts.h",
+    "inffast.h",
+    "inffixed.h",
+    "inflate.h",
+    "inftrees.h",
+    "trees.h",
+    "zconf.h",
+    "zlib.h",
+    "zutil.h",
+]
+
+_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS]
+
+# In order to limit the damage from the `includes` propagation
+# via `:zlib`, copy the public headers to a subdirectory and
+# expose those.
+genrule(
+    name = "copy_public_headers",
+    srcs = _ZLIB_HEADERS,
+    outs = _ZLIB_PREFIXED_HEADERS,
+    cmd_bash = "cp $(SRCS) $(@D)/zlib/include/",
+    cmd_bat = " && ".join(
+        ["@copy /Y $(location %s) $(@D)\\zlib\\include\\  >NUL" %
+         s for s in _ZLIB_HEADERS],
+    ),
+)
+
+cc_library(
+    name = "zlib",
+    srcs = [
+        "adler32.c",
+        "compress.c",
+        "crc32.c",
+        "deflate.c",
+        "gzclose.c",
+        "gzlib.c",
+        "gzread.c",
+        "gzwrite.c",
+        "infback.c",
+        "inffast.c",
+        "inflate.c",
+        "inftrees.c",
+        "trees.c",
+        "uncompr.c",
+        "zutil.c",
+        # Include the un-prefixed headers in srcs to work
+        # around the fact that zlib isn't consistent in its
+        # choice of <> or "" delimiter when including itself.
+    ] + _ZLIB_HEADERS,
+    hdrs = _ZLIB_PREFIXED_HEADERS,
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": [
+            "-Wno-deprecated-non-prototype",
+            "-Wno-unused-variable",
+            "-Wno-implicit-function-declaration",
+        ],
+    }),
+    includes = ["zlib/include/"],
+    visibility = ["//visibility:public"],
+)

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h

@@ -148,6 +148,10 @@ inline OtlpHeaders GetOtlpDefaultHeaders()
   return GetOtlpDefaultTracesHeaders();
 }
 
+std::string GetOtlpDefaultTracesCompression();
+std::string GetOtlpDefaultMetricsCompression();
+std::string GetOtlpDefaultLogsCompression();
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h

@@ -55,6 +55,9 @@ struct OtlpGrpcClientOptions
   /** max number of threads that can be allocated from this */
   std::size_t max_threads;
 
+  /** Compression type. */
+  std::string compression;
+
 #ifdef ENABLE_ASYNC_EXPORT
   // Concurrent requests
   std::size_t max_concurrent_requests;

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h

@@ -59,6 +59,9 @@ struct OtlpHttpClientOptions
   // This option is ignored if content_type is not kJson
   JsonBytesMappingKind json_bytes_mapping = JsonBytesMappingKind::kHexId;
 
+  // By default, do not compress data
+  std::string compression = "none";
+
   // If using the json name of protobuf field to set the key of json. By default, we will use the
   // field name just like proto files.
   bool use_json_name = false;
@@ -94,6 +97,7 @@ struct OtlpHttpClientOptions
                                nostd::string_view input_ssl_cipher_suite,
                                HttpRequestContentType input_content_type,
                                JsonBytesMappingKind input_json_bytes_mapping,
+                               nostd::string_view input_compression,
                                bool input_use_json_name,
                                bool input_console_debug,
                                std::chrono::system_clock::duration input_timeout,
@@ -116,6 +120,7 @@ struct OtlpHttpClientOptions
                     input_ssl_cipher_suite),
         content_type(input_content_type),
         json_bytes_mapping(input_json_bytes_mapping),
+        compression(input_compression),
         use_json_name(input_use_json_name),
         console_debug(input_console_debug),
         timeout(input_timeout),

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h

@@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h

@@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h

@@ -103,6 +103,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/src/otlp_environment.cc

@@ -997,6 +997,57 @@ OtlpHeaders GetOtlpDefaultLogsHeaders()
   return GetHeaders(kSignalEnv, kGenericEnv);
 }
 
+std::string GetOtlpDefaultTracesCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
+std::string GetOtlpDefaultMetricsCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
+std::string GetOtlpDefaultLogsCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/src/otlp_grpc_client.cc

@@ -291,6 +291,11 @@ std::shared_ptr<grpc::Channel> OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO
     grpc_arguments.SetResourceQuota(quota);
   }
 
+  if (options.compression == "gzip")
+  {
+    grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
+  }
+
   if (options.use_ssl_credentials)
   {
     grpc::SslCredentialsOptions ssl_opts;

exporters/otlp/src/otlp_grpc_exporter_options.cc

@@ -31,6 +31,8 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions()
   user_agent = GetOtlpDefaultUserAgent();
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultTracesCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc

@@ -29,6 +29,8 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions()
   user_agent = GetOtlpDefaultUserAgent();
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultLogsCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_grpc_metric_exporter_options.cc

@@ -31,6 +31,8 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions()
   aggregation_temporality = PreferredAggregationTemporality::kCumulative;
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultMetricsCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_http_client.cc

@@ -958,6 +958,11 @@ OtlpHttpClient::createSession(
   request->ReplaceHeader("Content-Type", content_type);
   request->ReplaceHeader("User-Agent", options_.user_agent);
 
+  if (options_.compression == "gzip")
+  {
+    request->SetCompression(opentelemetry::ext::http::client::Compression::kGzip);
+  }
+
   // Returns the created session data
   return HttpSessionData{
       std::move(session),

exporters/otlp/src/otlp_http_exporter.cc

@@ -40,6 +40,7 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options)
                                                             options.ssl_cipher_suite,
                                                             options.content_type,
                                                             options.json_bytes_mapping,
+                                                            options.compression,
                                                             options.use_json_name,
                                                             options.console_debug,
                                                             options.timeout,

exporters/otlp/src/otlp_http_exporter_options.cc

@@ -41,6 +41,8 @@ OtlpHttpExporterOptions::OtlpHttpExporterOptions()
   ssl_max_tls      = GetOtlpDefaultTracesSslTlsMaxVersion();
   ssl_cipher       = GetOtlpDefaultTracesSslTlsCipher();
   ssl_cipher_suite = GetOtlpDefaultTracesSslTlsCipherSuite();
+
+  compression = GetOtlpDefaultTracesCompression();
 }
 
 OtlpHttpExporterOptions::~OtlpHttpExporterOptions() {}

exporters/otlp/src/otlp_http_log_record_exporter.cc

@@ -43,6 +43,7 @@ OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter(
                                                             options.ssl_cipher_suite,
                                                             options.content_type,
                                                             options.json_bytes_mapping,
+                                                            options.compression,
                                                             options.use_json_name,
                                                             options.console_debug,
                                                             options.timeout,

exporters/otlp/src/otlp_http_log_record_exporter_options.cc

@@ -41,6 +41,8 @@ OtlpHttpLogRecordExporterOptions::OtlpHttpLogRecordExporterOptions()
   ssl_max_tls      = GetOtlpDefaultLogsSslTlsMaxVersion();
   ssl_cipher       = GetOtlpDefaultLogsSslTlsCipher();
   ssl_cipher_suite = GetOtlpDefaultLogsSslTlsCipherSuite();
+
+  compression = GetOtlpDefaultLogsCompression();
 }
 
 OtlpHttpLogRecordExporterOptions::~OtlpHttpLogRecordExporterOptions() {}

exporters/otlp/src/otlp_http_metric_exporter.cc

@@ -43,6 +43,7 @@ OtlpHttpMetricExporter::OtlpHttpMetricExporter(const OtlpHttpMetricExporterOptio
                                                             options.ssl_cipher_suite,
                                                             options.content_type,
                                                             options.json_bytes_mapping,
+                                                            options.compression,
                                                             options.use_json_name,
                                                             options.console_debug,
                                                             options.timeout,

exporters/otlp/src/otlp_http_metric_exporter_options.cc

@@ -42,6 +42,8 @@ OtlpHttpMetricExporterOptions::OtlpHttpMetricExporterOptions()
   ssl_max_tls      = GetOtlpDefaultMetricsSslTlsMaxVersion();
   ssl_cipher       = GetOtlpDefaultMetricsSslTlsCipher();
   ssl_cipher_suite = GetOtlpDefaultMetricsSslTlsCipherSuite();
+
+  compression = GetOtlpDefaultMetricsCompression();
 }
 
 OtlpHttpMetricExporterOptions::~OtlpHttpMetricExporterOptions() {}

exporters/otlp/test/otlp_http_exporter_test.cc

@@ -70,7 +70,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t
       "",                                 /* ssl_max_tls */
       "",                                 /* ssl_cipher */
       "",                                 /* ssl_cipher_suite */
-      options.content_type, options.json_bytes_mapping, options.use_json_name,
+      options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name,
       options.console_debug, options.timeout, options.http_headers);
   if (!async_mode)
   {

exporters/otlp/test/otlp_http_log_record_exporter_test.cc

@@ -69,7 +69,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t
       "",                                 /* ssl_max_tls */
       "",                                 /* ssl_cipher */
       "",                                 /* ssl_cipher_suite */
-      options.content_type, options.json_bytes_mapping, options.use_json_name,
+      options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name,
       options.console_debug, options.timeout, options.http_headers);
   if (!async_mode)
   {

exporters/otlp/test/otlp_http_metric_exporter_test.cc

@@ -76,7 +76,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t
       "",                                 /* ssl_max_tls */
       "",                                 /* ssl_cipher */
       "",                                 /* ssl_cipher_suite */
-      options.content_type, options.json_bytes_mapping, options.use_json_name,
+      options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name,
       options.console_debug, options.timeout, options.http_headers);
   if (!async_mode)
   {