Filter out auto-configured local exports when drains are configured

.changeset/fluffy-dragons-yell.md

@@ -0,0 +1,5 @@
+---
+"@vercel/otel": minor
+---
+
+Ignore auto-configuration based on the OTEL*EXPORTER_OTLP* env vars when trace drains are used. This avoids duplicate trace export.

.prettierrc

@@ -0,0 +1,3 @@
+{
+  "trailingComma": "all"
+}

packages/bridge-emulator/src/client/index.ts

@@ -4,13 +4,15 @@ import formidable from "formidable";
 export interface Bridge {
   port: number;
   fetches: Request[];
+  reportedSpans: object[];
   fetch: (input: string, init?: RequestInit) => Promise<Response>;
   reset: () => void;
   close: () => void;
 }
 
 interface BridgeOptions {
   serverPort: number;
+  traceDrains?: string[];
 }
 
 export async function start(opts: BridgeOptions): Promise<Bridge> {
@@ -34,6 +36,13 @@ interface StatusRequest {
   data: { status: string; [key: string]: unknown };
 }
 
+interface ReportSpansRequest {
+  cmd: "reportSpans";
+  testId: string;
+  runtime?: string;
+  data: object;
+}
+
 interface UnknownRequest {
   cmd: "unknown";
 }
@@ -42,17 +51,21 @@ type BridgeEmulatorRequest =
   | UnknownRequest
   | AckRequest
   | EchoRequest
-  | StatusRequest;
+  | StatusRequest
+  | ReportSpansRequest;
 
 class BridgeEmulatorServer implements Bridge {
   public port = -1;
   private serverPort: number;
   private server: Server | undefined;
   private waitingAck = new Map<string, Promise<unknown>>();
+  private traceDrains: string[] | undefined;
   public fetches: Request[] = [];
+  public reportedSpans: object[] = [];
 
-  constructor({ serverPort }: BridgeOptions) {
+  constructor({ serverPort, traceDrains }: BridgeOptions) {
     this.serverPort = serverPort;
+    this.traceDrains = traceDrains;
   }
 
   async connect(): Promise<void> {
@@ -144,6 +157,12 @@ class BridgeEmulatorServer implements Bridge {
         res.end();
         return;
       }
+      if (json.cmd === "reportSpans") {
+        res.writeHead(204, "OK", { "X-Server": "bridge" });
+        res.end();
+        this.reportedSpans.push(json.data);
+        return;
+      }
 
       res.writeHead(400, "Bad request", { "X-Server": "bridge" });
       res.end();
@@ -204,6 +223,9 @@ class BridgeEmulatorServer implements Bridge {
           "x-otel-test-id": testId,
           "x-otel-test-url": input,
           "x-otel-test-bridge-port": String(this.port),
+          ...(this.traceDrains
+            ? { "x-otel-test-trace-drains": this.traceDrains.join(",") }
+            : undefined),
         },
       });
       const resClone = res.clone();

packages/bridge-emulator/src/server/index.ts

@@ -69,6 +69,7 @@ export class BridgeEmulatorContextReader implements TextMapPropagator {
       "x-otel-test-id": testId,
       "x-otel-test-url": url,
       "x-otel-test-bridge-port": bridgePort,
+      "x-otel-test-trace-drains": traceDrainsCommaDelimited,
       ...headers
     } = allHeaders;
     if (testId && bridgePort) {
@@ -109,7 +110,22 @@ export class BridgeEmulatorContextReader implements TextMapPropagator {
           reportSpans: (data): void => {
             // eslint-disable-next-line no-console
             console.log("[BridgeEmulatorServer] reportSpans", data);
+            void fetch(`http://127.0.0.1:${bridgePort}`, {
+              method: "POST",
+              body: JSON.stringify({
+                cmd: "reportSpans",
+                testId,
+                runtime: process.env.NEXT_RUNTIME,
+                data: data ?? {},
+              }),
+              headers: { "content-type": "application/json" },
+              // @ts-expect-error - internal Next request.
+              next: { internal: true },
+            });
           },
+          ...(traceDrainsCommaDelimited
+            ? { traceDrains: traceDrainsCommaDelimited.split(",") }
+            : undefined),
         },
       };
     }

packages/otel/build.ts

@@ -7,7 +7,7 @@ const SOURCEMAP = true;
 
 const MAX_SIZES = {
   "dist/node/index.js": 300_000, // Increased from original 217KB limit
-  "dist/edge/index.js": 190_000, // Increased from original 185KB limit
+  "dist/edge/index.js": 191_000, // Increased from original 185KB limit
 };
 
 type ExternalPluginFactory = (external: string[]) => Plugin;

packages/otel/src/processor/filter-when-drained-span-processor.ts

@@ -0,0 +1,43 @@
+import type { Context } from "@opentelemetry/api";
+import type {
+  Span,
+  ReadableSpan,
+  SpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { diag } from "@opentelemetry/api";
+import { isDraining } from "../vercel-request-context/is-draining";
+
+let reported = false;
+
+/** @internal */
+export class FilterWhenDrainedSpanProcessor implements SpanProcessor {
+  constructor(private processor: SpanProcessor) {}
+
+  forceFlush(): Promise<void> {
+    return this.processor.forceFlush();
+  }
+
+  shutdown(): Promise<void> {
+    return this.processor.shutdown();
+  }
+
+  onStart(span: Span, parentContext: Context): void {
+    if (isDraining()) {
+      if (!reported) {
+        reported = true;
+        diag.debug(
+          "@vercel/otel: skipping automatic exporter due to configured trace drains",
+        );
+      }
+      return;
+    }
+    this.processor.onStart(span, parentContext);
+  }
+
+  onEnd(span: ReadableSpan): void {
+    if (isDraining()) {
+      return;
+    }
+    this.processor.onEnd(span);
+  }
+}

packages/otel/src/sdk.ts

@@ -57,6 +57,7 @@ import { W3CTraceContextPropagator } from "./propagators/w3c-tracecontext-propag
 import { VercelRuntimePropagator } from "./vercel-request-context/propagator";
 import { VercelRuntimeSpanExporter } from "./vercel-request-context/exporter";
 import { getStringFromEnv, getStringListFromEnv } from "./utils/env-parser";
+import { FilterWhenDrainedSpanProcessor } from "./processor/filter-when-drained-span-processor";
 
 interface Env {
   OTEL_SDK_DISABLED?: string;
@@ -474,7 +475,11 @@ function parseSpanProcessor(
                 ? new OTLPHttpProtoTraceExporter(config)
                 : new OTLPHttpJsonTraceExporter(config);
 
-            processors.push(new BatchSpanProcessor(exporter));
+            processors.push(
+              new FilterWhenDrainedSpanProcessor(
+                new BatchSpanProcessor(exporter),
+              ),
+            );
           }
 
           // Consider going throw `VERCEL_OTEL_ENDPOINTS` (otel collector) for OTLP.
@@ -484,7 +489,11 @@ function parseSpanProcessor(
             env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
             env.OTEL_EXPORTER_OTLP_ENDPOINT
           ) {
-            processors.push(new BatchSpanProcessor(parseTraceExporter(env)));
+            processors.push(
+              new FilterWhenDrainedSpanProcessor(
+                new BatchSpanProcessor(parseTraceExporter(env)),
+              ),
+            );
           }
 
           return processors;

packages/otel/src/vercel-request-context/api.ts

@@ -15,6 +15,7 @@ export interface VercelRequestContext {
   telemetry?: {
     reportSpans: (data: unknown) => void;
     rootSpanContext?: SpanContext;
+    traceDrains?: string[];
   };
   [key: symbol]: unknown;
 }

packages/otel/src/vercel-request-context/is-draining.ts

@@ -0,0 +1,6 @@
+import { getVercelRequestContext } from "./api";
+
+export function isDraining(): boolean {
+  const context = getVercelRequestContext();
+  return Boolean(context?.telemetry?.traceDrains?.length);
+}

tests/e2e/lib/with-bridge.ts

@@ -24,6 +24,7 @@ export interface DescribeProps {
 interface DescribeOptions {
   env?: Record<string, string>;
   collector?: OtelCollectorOptions;
+  bridge?: { traceDrains?: string[] };
 }
 
 export function describe(
@@ -42,7 +43,7 @@ export function describe(
     beforeAll(async () => {
       port = 30000 + Math.floor(Math.random() * 10000);
       collector = await startCollector(opts.collector);
-      bridge = await startBridge({ serverPort: port });
+      bridge = await startBridge({ ...opts.bridge, serverPort: port });
       if (!process.env.OTEL_LOG_LEVEL) {
         process.env.OTEL_LOG_LEVEL = "info";
       }

tests/e2e/test/vercel-deployment/drains.test.ts

@@ -0,0 +1,58 @@
+import { expect, it } from "vitest";
+import { describe } from "../../lib/with-bridge";
+
+describe(
+  "vercel deployment: drains",
+  {
+    bridge: {
+      traceDrains: ["traceful.dev"],
+    },
+  },
+  (props) => {
+    it("should NOT output traces", async () => {
+      const { collector, bridge } = props();
+
+      const execResp = await bridge.fetch("/slugs/baz");
+      expect(execResp.status).toBe(200);
+
+      // Wait to make sure no spans are reported.
+      await new Promise((resolve) => {
+        setTimeout(resolve, 1000);
+      });
+      expect(collector.getAllTraces()).toHaveLength(0);
+
+      // Instead the spans should be available via `reportSpans`.
+      expect(bridge.reportedSpans.length).toBeGreaterThan(0);
+
+      /* eslint-disable @typescript-eslint/no-explicit-any -- only testing that Rusty calls are being made */
+      /* eslint-disable @typescript-eslint/no-unsafe-member-access -- only testing that Rusty calls are being made */
+      /* eslint-disable @typescript-eslint/no-unsafe-assignment -- only testing that Rusty calls are being made */
+
+      let spanCount = 0;
+      let foundSpan: object | undefined;
+      for (const reportedSpans of bridge.reportedSpans as any[]) {
+        for (const resourceSpans of reportedSpans.resourceSpans) {
+          for (const scopeSpans of resourceSpans.scopeSpans) {
+            for (const span of scopeSpans.spans) {
+              spanCount++;
+              if (span.name === "GET /slugs/[slug]") {
+                foundSpan = span;
+                break;
+              }
+            }
+          }
+        }
+      }
+
+      expect(spanCount).toBeGreaterThan(2);
+      expect(foundSpan).toBeDefined();
+      expect(foundSpan).toMatchObject({
+        name: "GET /slugs/[slug]",
+      });
+
+      /* eslint-enable @typescript-eslint/no-explicit-any */
+      /* eslint-enable @typescript-eslint/no-unsafe-member-access */
+      /* eslint-enable @typescript-eslint/no-unsafe-assignment */
+    });
+  },
+);