diff --git a/.chloggen/elasticsearchexporter_mapping-mode-default-otel.yaml b/.chloggen/elasticsearchexporter_mapping-mode-default-otel.yaml new file mode 100644 index 0000000000000..5e57161de45be --- /dev/null +++ b/.chloggen/elasticsearchexporter_mapping-mode-default-otel.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Change default `mapping::mode` config to `otel` + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37241] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: The new default, OTel mapping mode, requires Elasticsearch 8.12+. To retain the old behavior, explicitly set `mapping::mode` to `none`. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index b70b4bcbd7feb..6df70c38405f3 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -158,7 +158,7 @@ The Elasticsearch exporter supports several document schemas and preprocessing behaviours, which may be configured through the following settings: - `mapping`: - - `mode` (default=none): The default mapping mode. Valid modes are: + - `mode` (default=otel): The default mapping mode. Valid modes are: - `none` - `ecs` - `otel` @@ -177,7 +177,12 @@ See below for a description of each mapping mode. #### OTel mapping mode -Requires Elasticsearch 8.12 or above. +The default and recommended "OTel-native" mapping mode. + +Requires Elasticsearch 8.12 or above[^1], works best with Elasticsearch 8.16 or above[^2]. + +[^1]: as it uses the undocumented `require_data_stream` bulk API parameter supported from Elasticsearch 8.12 +[^2]: Elasticsearch 8.16 contains a built-in `otel-data` plugin In `otel` mapping mode, the Elasticsearch Exporter stores documents in Elastic's preferred "OTel-native" schema. In this mapping mode, documents use the original attribute names and @@ -188,9 +193,7 @@ and `data_stream.namespace`. Instead of serializing these values under the `*att they are put at the root of the document, to conform with the conventions of the data stream naming scheme that maps these as `constant_keyword` fields. -`data_stream.dataset` will always be appended with `.otel`. It is recommended to use with -`*_dynamic_index::enabled: true` (e.g. `logs_dynamic_index::enabled`) to route documents to data stream -`${data_stream.type}-${data_stream.dataset}-${data_stream.namespace}`. +`data_stream.dataset` will always be appended with `.otel` if [dynamic data stream routing mode](#elasticsearch-document-routing) is active. Span events are stored in separate documents. They will be routed with `data_stream.type` set to `logs` if `traces_dynamic_index::enabled` is `true`. @@ -516,3 +519,16 @@ This gives the exporter the opportunity to group all related metrics into the sa 2. Otherwise, check your metrics pipeline setup for misconfiguration that causes an actual violation of the [single writer principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer). This means that the same metric with the same dimensions is sent from multiple sources, which is not allowed in the OTel metrics data model. + +### flush failed (400) illegal_argument_exception + +Symptom: bulk indexer logs an error that indicates "bulk indexer flush error" with bulk request returning HTTP 400 and an error type of `illegal_argument_exception`, similar to the following. + +``` +error elasticsearchexporter@v0.120.1/bulkindexer.go:343 bulk indexer flush error {"otelcol.component.id": "elasticsearch", "otelcol.component.kind": "Exporter", "otelcol.signal": "logs", "error": "flush failed (400): {\"error\":{\"type\":\"illegal_argument_exception\",\"caused_by\":{}}}"} +``` + +This may happen when you use [OTel mapping mode](#otel-mapping-mode) (the default mapping mode from v0.122.0, or explicitly by configuring `mapping::mode: otel`) sending to Elasticsearch version < 8.12. + +To resolve this, it is recommended to upgrade your Elasticsearch to 8.12+, ideally 8.16+. +Alternatively, try other mapping modes, but the document structure will be different. diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index e36f15ee86db3..e4bc453a916b0 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -109,7 +109,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", + Mode: "otel", AllowedModes: []string{"bodymap", "ecs", "none", "otel", "raw"}, }, LogstashFormat: LogstashFormatSettings{ @@ -184,7 +184,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", + Mode: "otel", AllowedModes: []string{"bodymap", "ecs", "none", "otel", "raw"}, }, LogstashFormat: LogstashFormatSettings{ @@ -259,7 +259,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", + Mode: "otel", AllowedModes: []string{"bodymap", "ecs", "none", "otel", "raw"}, }, LogstashFormat: LogstashFormatSettings{ diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 8b61a0537b0aa..9002952c6dc56 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -363,6 +363,7 @@ func TestExporterLogs(t *testing.T) { }) exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "none" cfg.LogsDynamicIndex.Enabled = true }) logs := newLogsWithAttributes( @@ -1734,6 +1735,7 @@ func TestExporterTraces(t *testing.T) { }) exporter := newTestTracesExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "none" cfg.TracesDynamicIndex.Enabled = true }) diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index a78e6d176717e..755894697827a 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -84,7 +84,7 @@ func createDefaultConfig() component.Config { }, }, Mapping: MappingsSettings{ - Mode: "none", + Mode: "otel", AllowedModes: slices.Sorted(maps.Keys(canonicalMappingModes)), }, LogstashFormat: LogstashFormatSettings{