[exporter/datadog] use the same default batch configs as exporterhelper (#43848)

#### Description
Set sending queue batch default values to match exporter helper default:
flush timeout 200ms, min size 8192, no max size.

Context: when we applied the default batch values in
#43419
to datadog-agent, we observed a
[regression](DataDog/datadog-agent#41661 (comment))
in performance tests. Setting the batch values back to batch processor
default values (also the exporter helper default) fixes the regression,
thus this change.

A downside is if users ran into 413 payload too large errors, they now
need to set the batch values in datadog::sending_queue::batch. This is
reflected in the readme.

#### Link to tracking issue
related to #43082

Author: songy23

.chloggen/dd-exporter-queue.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: exporter/datadog
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Set sending queue batch default values to match exporter helper default: flush timeout 200ms, min size 8192, no max size."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [43848]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "The default values now match exactly the default in batch processor."
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/datadogexporter/README.md

@@ -27,16 +27,18 @@ Find the full configs of Datadog exporter and their usage in [collector.yaml](./
 
 This error indicates the payload size sent by the Datadog exporter exceeds the size limit (see previous examples https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16834, https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/17566).
 
-This is usually caused by the pipeline batching too many telemetry data before sending to the Datadog API intake. To fix that, prefer using the Datadog exporter `sending_queue::batch` section instead of the batch processor, since the former has the defaults that work correctly with Datadog:
+This is usually caused by the pipeline batching too many telemetry data before sending to the Datadog API intake. To fix that, prefer using the Datadog exporter `sending_queue::batch` section instead of the batch processor:
 
 ```yaml
 exporters:
   datadog:
     api:
       key: ${env:DD_API_KEY}
     sending_queue:
-      # An empty section means that the defaults will be used.
       batch:
+        min_size: 10
+        max_size: 100
+        flush_timeout: 10s
 ```
 
 

pkg/datadog/config/config.go

@@ -15,7 +15,6 @@ import (
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configopaque"
-	"go.opentelemetry.io/collector/config/configoptional"
 	"go.opentelemetry.io/collector/config/configretry"
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
@@ -344,31 +343,12 @@ func defaultClientConfig() confighttp.ClientConfig {
 	return client
 }
 
-// newDefaultQueueConfig creates the default exporter queue configuration.
-func newDefaultQueueConfig() exporterhelper.QueueBatchConfig {
-	queueSet := exporterhelper.NewDefaultQueueConfig()
-	// Override batching options since the defaults cause payloads that are over our limits
-	// See e.g. https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16834 or https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/17566
-	//
-	// The limits were chosen based on the API intake limits:
-	// - Trace intake: 3.2MB
-	// - Log intake: https://docs.datadoghq.com/api/latest/logs/
-	// - Metrics V2 intake: https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-	queueSet.Batch = configoptional.Default(exporterhelper.BatchConfig{
-		FlushTimeout: 10 * time.Second,
-		Sizer:        exporterhelper.RequestSizerTypeItems,
-		MinSize:      10,
-		MaxSize:      100,
-	})
-	return queueSet
-}
-
 // CreateDefaultConfig creates the default exporter configuration
 func CreateDefaultConfig() component.Config {
 	return &Config{
 		ClientConfig:  defaultClientConfig(),
 		BackOffConfig: configretry.NewDefaultBackOffConfig(),
-		QueueSettings: newDefaultQueueConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 
 		API: APIConfig{
 			Site: "datadoghq.com",

pkg/datadog/config/config_test.go

@@ -22,6 +22,7 @@ import (
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
 	"go.opentelemetry.io/collector/confmap/xconfmap"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 )
 
 func TestValidate(t *testing.T) {
@@ -408,7 +409,7 @@ func TestCreateDefaultConfig(t *testing.T) {
 	assert.Equal(t, &Config{
 		ClientConfig:  defaultClientConfig(),
 		BackOffConfig: configretry.NewDefaultBackOffConfig(),
-		QueueSettings: newDefaultQueueConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 
 		API: APIConfig{
 			Site: "datadoghq.com",
@@ -484,7 +485,7 @@ func TestLoadConfig(t *testing.T) {
 			expected: &Config{
 				ClientConfig:  defaultClientConfig(),
 				BackOffConfig: configretry.NewDefaultBackOffConfig(),
-				QueueSettings: newDefaultQueueConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 				API: APIConfig{
 					Key:              "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 					Site:             "datadoghq.com",
@@ -545,7 +546,7 @@ func TestLoadConfig(t *testing.T) {
 			expected: &Config{
 				ClientConfig:  defaultClientConfig(),
 				BackOffConfig: configretry.NewDefaultBackOffConfig(),
-				QueueSettings: newDefaultQueueConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 				TagsConfig: TagsConfig{
 					Hostname: "customhostname",
 				},
@@ -613,7 +614,7 @@ func TestLoadConfig(t *testing.T) {
 			expected: &Config{
 				ClientConfig:  defaultClientConfig(),
 				BackOffConfig: configretry.NewDefaultBackOffConfig(),
-				QueueSettings: newDefaultQueueConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 				TagsConfig: TagsConfig{
 					Hostname: "customhostname",
 				},
@@ -679,7 +680,7 @@ func TestLoadConfig(t *testing.T) {
 			expected: &Config{
 				ClientConfig:  defaultClientConfig(),
 				BackOffConfig: configretry.NewDefaultBackOffConfig(),
-				QueueSettings: newDefaultQueueConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueConfig(),
 				API: APIConfig{
 					Key:              "abc",
 					Site:             "datadoghq.com",