Prom rw exporter, rw2 generate metrics from written headers (open-telemetry#40866)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
 
Implements metrics based on the written header in RW2 response as
discussed here

open-telemetry#40559 (comment)

Review please @ArthurSens @dashpole @ywwg.

<!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Partially implements open-telemetry#33661 (when merging PR please don't close the
tracing issue)


<!--Describe what testing was performed and which tests were added.-->
#### Testing
* [x] e2e ran with prometheus locally

![Screenshot 2025-06-23 at 18 38
39](https://github.com/user-attachments/assets/15687f8c-b036-483d-aeeb-c9689f0e0128)

Author: jmichalek132

.chloggen/jm-prom-rw-exporter-rw2-generate-metrics-from-written-headers.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: prometheusremotewriteexproter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: |
+  Adds additional metrics to the Prometheus Remote Write Exporter when RW2 enable. The new metrics are:
+  - `otelcol_exporter_prometheusremotewrite_written_samples`: Number of Prometheus Samples that were successfully written to the remote write endpoint.
+  - `otelcol_exporter_prometheusremotewrite_written_histograms`: Number of Prometheus Histograms that were successfully written to the remote write endpoint.
+  - `otelcol_exporter_prometheusremotewrite_written_exemplars`: Number of Prometheus Exemplars that were successfully written to the remote write endpoint.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33661]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/prometheusremotewriteexporter/documentation.md

@@ -109,3 +109,27 @@ Number of WAL writes that failed
 | Unit | Metric Type | Value Type | Monotonic |
 | ---- | ----------- | ---------- | --------- |
 | 1 | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_written_exemplars
+
+Number of Prometheus Exemplars that were successfully written to the remote write endpoint (only available when using remote write v2)
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {exemplar} | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_written_histograms
+
+Number of Prometheus Histograms that were successfully written to the remote write endpoint (only available when using remote write v2)
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {histogram} | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_written_samples
+
+Number of Prometheus Samples that were successfully written to the remote write endpoint (only available when using remote write v2)
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {sample} | Sum | Int | true |

exporter/prometheusremotewriteexporter/exporter.go

@@ -42,6 +42,9 @@ type prwTelemetry interface {
 	recordTranslatedTimeSeries(ctx context.Context, numTS int)
 	recordRemoteWriteSentBatch(ctx context.Context)
 	setNumberConsumer(ctx context.Context, n int64)
+	recordWrittenSamples(ctx context.Context, numSamples int64)
+	recordWrittenHistograms(ctx context.Context, numHistograms int64)
+	recordWrittenExemplars(ctx context.Context, numExemplars int64)
 }
 
 type prwTelemetryOtel struct {
@@ -65,6 +68,18 @@ func (p *prwTelemetryOtel) recordTranslatedTimeSeries(ctx context.Context, numTS
 	p.telemetryBuilder.ExporterPrometheusremotewriteTranslatedTimeSeries.Add(ctx, int64(numTS), metric.WithAttributes(p.otelAttrs...))
 }
 
+func (p *prwTelemetryOtel) recordWrittenSamples(ctx context.Context, numSamples int64) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWrittenSamples.Add(ctx, numSamples, metric.WithAttributes(p.otelAttrs...))
+}
+
+func (p *prwTelemetryOtel) recordWrittenHistograms(ctx context.Context, numHistograms int64) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWrittenHistograms.Add(ctx, numHistograms, metric.WithAttributes(p.otelAttrs...))
+}
+
+func (p *prwTelemetryOtel) recordWrittenExemplars(ctx context.Context, numExemplars int64) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWrittenExemplars.Add(ctx, numExemplars, metric.WithAttributes(p.otelAttrs...))
+}
+
 type buffer struct {
 	protobuf *proto.Buffer
 	snappy   []byte
@@ -421,13 +436,7 @@ func (prwe *prwExporter) execute(ctx context.Context, buf *buffer) error {
 		// implementation is not compliant with the specification. Reference:
 		// https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/#required-written-response-headers
 		if enableSendingRW2FeatureGate.IsEnabled() && prwe.RemoteWriteProtoMsg == config.RemoteWriteProtoMsgV2 {
-			samplesWritten := resp.Header.Get("X-Prometheus-Remote-Write-Samples-Written")
-			if samplesWritten == "" {
-				prwe.settings.Logger.Warn(
-					"X-Prometheus-Remote-Write-Samples-Written header is missing from the response, suggesting that the endpoint doesn't support RW2 and might be silently dropping data.",
-					zap.String("url", resp.Request.URL.String()),
-				)
-			}
+			prwe.handleWrittenHeaders(ctx, resp)
 		}
 
 		// 2xx status code is considered a success

exporter/prometheusremotewriteexporter/exporter_v2.go

@@ -6,6 +6,8 @@ package prometheusremotewriteexporter // import "github.com/open-telemetry/opent
 import (
 	"context"
 	"math"
+	"net/http"
+	"strconv"
 	"sync"
 
 	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
@@ -104,3 +106,41 @@ func (prwe *prwExporter) handleExportV2(ctx context.Context, symbolsTable writev
 
 	return prwe.exportV2(ctx, requests)
 }
+
+func (prwe *prwExporter) handleHeader(ctx context.Context, resp *http.Response, headerName string, metricType string, recordFunc func(context.Context, int64)) {
+	headerValue := resp.Header.Get(headerName)
+	if headerValue == "" {
+		prwe.settings.Logger.Warn(
+			headerName+" header is missing from the response, suggesting that the endpoint doesn't support RW2 and might be silently dropping data.",
+			zap.String("url", resp.Request.URL.String()),
+		)
+		return
+	}
+
+	value, err := strconv.ParseInt(headerValue, 10, 64)
+	if err != nil {
+		prwe.settings.Logger.Warn(
+			"Failed to convert "+headerName+" header to int64, not counting "+metricType+" written",
+			zap.String("url", resp.Request.URL.String()),
+		)
+		return
+	}
+	recordFunc(ctx, value)
+}
+
+func (prwe *prwExporter) handleWrittenHeaders(ctx context.Context, resp *http.Response) {
+	prwe.handleHeader(ctx, resp,
+		"X-Prometheus-Remote-Write-Samples-Written",
+		"samples",
+		prwe.telemetry.recordWrittenSamples)
+
+	prwe.handleHeader(ctx, resp,
+		"X-Prometheus-Remote-Write-Histograms-Written",
+		"histograms",
+		prwe.telemetry.recordWrittenHistograms)
+
+	prwe.handleHeader(ctx, resp,
+		"X-Prometheus-Remote-Write-Exemplars-Written",
+		"exemplars",
+		prwe.telemetry.recordWrittenExemplars)
+}

exporter/prometheusremotewriteexporter/internal/metadata/generated_telemetry.go

@@ -32,6 +32,27 @@ telemetry:
       sum:
         value_type: int
         monotonic: true
+    exporter_prometheusremotewrite_written_samples:
+      enabled: true
+      description: Number of Prometheus Samples that were successfully written to the remote write endpoint (only available when using remote write v2)
+      unit: "{sample}"
+      sum:
+        value_type: int
+        monotonic: true
+    exporter_prometheusremotewrite_written_histograms:
+      enabled: true
+      description: Number of Prometheus Histograms that were successfully written to the remote write endpoint (only available when using remote write v2)
+      unit: "{histogram}"
+      sum:
+        value_type: int
+        monotonic: true
+    exporter_prometheusremotewrite_written_exemplars:
+      enabled: true
+      description: Number of Prometheus Exemplars that were successfully written to the remote write endpoint (only available when using remote write v2)
+      unit: "{exemplar}"
+      sum:
+        value_type: int
+        monotonic: true
     exporter_prometheusremotewrite_consumers:
       enabled: true
       description: Number of configured workers to use to fan out the outgoing requests