From 3fa8247360da0d04c7bf802be9b3089f20446e72 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 20 Jun 2025 13:10:43 +0100 Subject: [PATCH] Remove deprecated otel elasticsearch exporter config `*_dynamic_index` (#8592) As *_dynamic_index are all no-op and logs_index defaults to empty string from v0.122.0 elasticsearchexporter, clean up deprecated config usage in EA repo. (cherry picked from commit 3b4f9112edfbd01b3d3a39a11310d8f06019318f) --- ...asticsearchexporter-deprecated-config.yaml | 32 +++++++++++++++++++ .../edot-collector/kube-stack/values.yaml | 6 ---- .../samples/darwin/logs_metrics_traces.yml | 12 ------- .../pkg/otel/samples/darwin/platformlogs.yml | 6 ---- .../darwin/platformlogs_hostmetrics.yml | 12 ------- internal/pkg/otel/samples/linux/gateway.yml | 6 ---- .../samples/linux/logs_metrics_traces.yml | 12 ------- .../pkg/otel/samples/linux/platformlogs.yml | 6 ---- .../linux/platformlogs_hostmetrics.yml | 12 ------- internal/pkg/otel/translate/otelconfig.go | 4 +-- .../pkg/otel/translate/otelconfig_test.go | 4 --- 11 files changed, 33 insertions(+), 79 deletions(-) create mode 100644 changelog/fragments/1750349806-elasticsearchexporter-deprecated-config.yaml diff --git a/changelog/fragments/1750349806-elasticsearchexporter-deprecated-config.yaml b/changelog/fragments/1750349806-elasticsearchexporter-deprecated-config.yaml new file mode 100644 index 00000000000..0a43e3d0010 --- /dev/null +++ b/changelog/fragments/1750349806-elasticsearchexporter-deprecated-config.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# Change summary; a 80ish characters long description of the change. +summary: Remove deprecated otel elasticsearch exporter config `*_dynamic_index` from code and samples + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/8592 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/deploy/helm/edot-collector/kube-stack/values.yaml b/deploy/helm/edot-collector/kube-stack/values.yaml index af32383122f..218a2c89f09 100644 --- a/deploy/helm/edot-collector/kube-stack/values.yaml +++ b/deploy/helm/edot-collector/kube-stack/values.yaml @@ -610,12 +610,6 @@ collectors: endpoints: # List of Elasticsearch endpoints. - ${env:ELASTIC_ENDPOINT} api_key: ${env:ELASTIC_API_KEY} # API key for Elasticsearch authentication. - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true # Enable in order to skip the SSL certificate Check # tls: # insecure_skip_verify: true diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index ae042ff2ddc..d619b4492cf 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -99,24 +99,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true elasticsearch/otel: endpoints: [ "${env:ELASTIC_ENDPOINT}" ] api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index 78c69b17f59..7ece09d341d 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -58,12 +58,6 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index f57daea83a5..b121a6457da 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -90,23 +90,11 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true elasticsearch/otel: endpoints: [ "${env:ELASTIC_ENDPOINT}" ] api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/linux/gateway.yml b/internal/pkg/otel/samples/linux/gateway.yml index 5a954050dac..f4a69818252 100644 --- a/internal/pkg/otel/samples/linux/gateway.yml +++ b/internal/pkg/otel/samples/linux/gateway.yml @@ -49,12 +49,6 @@ exporters: endpoints: - ${ELASTIC_ENDPOINT} # Will be populated from environment variable api_key: ${ELASTIC_API_KEY} # Will be populated from environment variable - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true # tls: # insecure_skip_verify: true mapping: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index 9b0ae8bfe19..9a2aaa2cf13 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -106,24 +106,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true elasticsearch/otel: endpoints: [ "${env:ELASTIC_ENDPOINT}" ] api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml index 78c69b17f59..7ece09d341d 100644 --- a/internal/pkg/otel/samples/linux/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -58,12 +58,6 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml index 83e89bf7b72..749287987b0 100644 --- a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -97,23 +97,11 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true elasticsearch/otel: endpoints: [ "${env:ELASTIC_ENDPOINT}" ] api_key: ${env:ELASTIC_API_KEY} mapping: mode: otel - logs_dynamic_index: - enabled: true - metrics_dynamic_index: - enabled: true - traces_dynamic_index: - enabled: true service: extensions: [file_storage] diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 558c32222fe..69f15e9b65b 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -400,9 +400,7 @@ func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { if err != nil { return nil, err } - // we want to use dynamic indexing - esConfig["logs_index"] = "" // needs to be empty for logs_dynamic_index - esConfig["logs_dynamic_index"] = map[string]any{"enabled": true} + // dynamic indexing works by default // we also want to use dynamic log ids esConfig["logs_dynamic_id"] = map[string]any{"enabled": true} diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index e9a0b1527f1..02581b69e4f 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -222,7 +222,6 @@ func TestGetOtelConfig(t *testing.T) { "max_size": 1600, "min_size": 0, }, - "logs_index": "", "mapping": map[string]any{ "mode": "bodymap", }, @@ -235,9 +234,6 @@ func TestGetOtelConfig(t *testing.T) { "max_interval": 1 * time.Minute, "max_retries": 3, }, - "logs_dynamic_index": map[string]any{ - "enabled": true, - }, "logs_dynamic_id": map[string]any{ "enabled": true, },