feat(otlp-exporter): add timeout env var (#2738)

Co-authored-by: Gerhard Stöbich <deb2001-github@yahoo.de>
Co-authored-by: Rauno Viskus <Rauno56@users.noreply.github.com>
Co-authored-by: Valentin Marchaud <contact@vmarchaud.fr>
Co-authored-by: Daniel Dyla <dyladan@users.noreply.github.com>

Author: 5 people

experimental/CHANGELOG.md

@@ -63,6 +63,7 @@ All notable changes to experimental packages in this project will be documented
 * feat(proto): add @opentelemetry/otlp-transformer package with hand-rolled transformation #2746 @dyladan
 * feat(sdk-metrics-base): shutdown and forceflush on MeterProvider #2890 @legendecas
 * feat(sdk-metrics-base): return the same meter for identical input to getMeter #2901 @legendecas
+* feat(otlp-exporter): add [OTEL_EXPORTER_OTLP_TIMEOUT](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#configuration-options) env var to otlp exporters #2738 @svetlanabrennan
 * feat(sdk-metrics-base): hoist async instrument callback invocations #2822 @legendecas
 
 ### :bug: (Bug Fix)

experimental/packages/exporter-trace-otlp-grpc/README.md

@@ -108,6 +108,27 @@ provider.register();
 
 Note, that this will only work if TLS is also configured on the server.
 
+## Exporter Timeout Configuration
+
+The OTLPTraceExporter has a timeout configuration option which is the maximum time, in milliseconds, the OTLP exporter will wait for each batch export. The default value is 10000ms.
+
++ To override the default timeout duration, provide `timeoutMillis` to OTLPTraceExporter with `collectorOptions`:
+
+  ```js
+  const collectorOptions = {
+    timeoutMillis: 15000,
+    // url is optional and can be omitted - default is localhost:4317
+    url: '<collector-hostname>:<port>',
+    metadata, // // an optional grpc.Metadata object to be sent with each request
+  };
+    
+  const exporter = new OTLPTraceExporter(collectorOptions);
+  ```
+
+  > Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables.
+
+## Exporter Compression Configuration
+
 By default no compression will be used. To use compression, set it programmatically in `collectorOptions` or with environment variables. Supported compression options: `gzip` and `none`.
 
 ```js
@@ -126,13 +147,13 @@ const exporter = new OTLPTraceExporter(collectorOptions);
 
 ## Environment Variable Configuration
 
-Set compression with environment variables.
-
-```shell
-OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
-```
-
- > Compression set programatically in `collectorOptions` takes precedence over compression set with environment variables. `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` takes precedence and overrides `OTEL_EXPORTER_OTLP_COMPRESSION`.
+ | Environment variable | Description |
+  |----------------------|-------------|
+  | OTEL_EXPORTER_OTLP_TRACES_TIMEOUT | The maximum waiting time, in milliseconds, allowed to send each OTLP trace batch. Default is 10000. |
+  | OTEL_EXPORTER_OTLP_TIMEOUT | The maximum waiting time, in milliseconds, allowed to send each OTLP trace and metric batch. Default is 10000. |
+  | OTEL_EXPORTER_OTLP_TRACES_COMPRESSION | The compression type to use on OTLP trace requests. Options include gzip. By default no compression will be used. |
+  | OTEL_EXPORTER_OTLP_COMPRESSION | The compression type to use on OTLP trace, metric, and log requests. Options include gzip. By default no compression will be used. |
+ > The per-signal environment variables (`OTEL_EXPORTER_OTLP_TRACES_TIMEOUT`) takes precedence and non-per-signal environment variable (`OTEL_EXPORTER_OTLP_TIMEOUT`).
 
 ## Running opentelemetry-collector locally to see the traces
 
@@ -141,9 +162,9 @@ OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
 
 ## Useful links
 
-- For more information on OpenTelemetry, visit: <https://opentelemetry.io/>
-- For more about OpenTelemetry JavaScript: <https://github.com/open-telemetry/opentelemetry-js>
-- For help or feedback on this project, join us in [GitHub Discussions][discussions-url]
++ For more information on OpenTelemetry, visit: <https://opentelemetry.io/>
++ For more about OpenTelemetry JavaScript: <https://github.com/open-telemetry/opentelemetry-js>
++ For help or feedback on this project, join us in [GitHub Discussions][discussions-url]
 
 ## License
 

experimental/packages/exporter-trace-otlp-grpc/test/OTLPTraceExporter.test.ts

@@ -34,6 +34,7 @@ import {
   ensureResourceIsCorrect,
   mockedReadableSpan,
 } from './traceHelper';
+import * as core from '@opentelemetry/core';
 import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
 import { GrpcCompressionAlgorithm } from '@opentelemetry/otlp-grpc-exporter-base';
 import { IExportTraceServiceRequest, IResourceSpans } from '@opentelemetry/otlp-transformer';
@@ -197,7 +198,34 @@ const testCollectorExporter = (params: TestParams) =>
           ensureMetadataIsCorrect(reqMetadata, params?.metadata);
 
           done();
-        }, 200);
+        }, 500);
+      });
+      it('should log deadline exceeded error', done => {
+        const credentials = params.useTLS
+          ? grpc.credentials.createSsl(
+            fs.readFileSync('./test/certs/ca.crt'),
+            fs.readFileSync('./test/certs/client.key'),
+            fs.readFileSync('./test/certs/client.crt')
+          )
+          : undefined;
+
+        const collectorExporterWithTimeout = new OTLPTraceExporter({
+          url: 'grpcs://' + address,
+          credentials,
+          metadata: params.metadata,
+          timeoutMillis: 100,
+        });
+
+        const responseSpy = sinon.spy();
+        const spans = [Object.assign({}, mockedReadableSpan)];
+        collectorExporterWithTimeout.export(spans, responseSpy);
+
+        setTimeout(() => {
+          const result = responseSpy.args[0][0] as core.ExportResult;
+          assert.strictEqual(result.code, core.ExportResultCode.FAILED);
+          assert.strictEqual(responseSpy.args[0][0].error.details, 'Deadline exceeded');
+          done();
+        }, 300);
       });
     });
     describe('export - with gzip compression', () => {

experimental/packages/exporter-trace-otlp-http/README.md

@@ -107,16 +107,48 @@ OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=https://metric-service:4318/v1/metrics
 
 For more details, see [OpenTelemetry Specification on Protocol Exporter][opentelemetry-spec-protocol-exporter].
 
+## Exporter Timeout Configuration
+
+The OTLPTraceExporter has a timeout configuration option which is the maximum time, in milliseconds, the OTLP exporter will wait for each batch export. The default value is 10000ms.
+
+To override the default timeout duration, use the following options:
+
++ Set with environment variables:
+
+  | Environment variable | Description |
+  |----------------------|-------------|
+  | OTEL_EXPORTER_OTLP_TRACES_TIMEOUT | The maximum waiting time, in milliseconds, allowed to send each OTLP trace batch. Default is 10000. |
+  | OTEL_EXPORTER_OTLP_TIMEOUT | The maximum waiting time, in milliseconds, allowed to send each OTLP trace and metric batch. Default is 10000. |
+
+  > `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` takes precedence and overrides `OTEL_EXPORTER_OTLP_TIMEOUT`.
+
++ Provide `timeoutMillis` to OTLPTraceExporter with `collectorOptions`:
+
+  ```js
+  const collectorOptions = {
+    timeoutMillis: 15000,
+    url: '<opentelemetry-collector-url>', // url is optional and can be omitted - default is http://localhost:4318/v1/traces
+    headers: {
+      foo: 'bar'
+    }, // an optional object containing custom headers to be sent with each request will only work with http
+    concurrencyLimit: 10, // an optional limit on pending requests
+  };
+
+  const exporter = new OTLPTraceExporter(collectorOptions);
+  ```
+
+  > Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables.
+
 ## Running opentelemetry-collector locally to see the traces
 
 1. Go to `examples/otlp-exporter-node`
 2. Follow the instructions there to inspect traces.
 
 ## Useful links
 
-- For more information on OpenTelemetry, visit: <https://opentelemetry.io/>
-- For more about OpenTelemetry JavaScript: <https://github.com/open-telemetry/opentelemetry-js>
-- For help or feedback on this project, join us in [GitHub Discussions][discussions-url]
++ For more information on OpenTelemetry, visit: <https://opentelemetry.io/>
++ For more about OpenTelemetry JavaScript: <https://github.com/open-telemetry/opentelemetry-js>
++ For help or feedback on this project, join us in [GitHub Discussions][discussions-url]
 
 ## License
 

experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts

@@ -14,21 +14,21 @@
  * limitations under the License.
  */
 
+import * as core from '@opentelemetry/core';
 import { diag, DiagLogger, DiagLogLevel } from '@opentelemetry/api';
 import { ExportResultCode } from '@opentelemetry/core';
 import { ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import * as assert from 'assert';
 import * as sinon from 'sinon';
 import { OTLPTraceExporter } from '../../src/platform/browser/index';
-
 import {
   ensureSpanIsCorrect,
   ensureExportTraceServiceRequestIsSet,
   ensureWebResourceIsCorrect,
   ensureHeadersContain,
   mockedReadableSpan,
 } from '../traceHelper';
-import { OTLPExporterConfigBase } from '@opentelemetry/otlp-exporter-base';
+import { OTLPExporterConfigBase, OTLPExporterError } from '@opentelemetry/otlp-exporter-base';
 import { IExportTraceServiceRequest } from '@opentelemetry/otlp-transformer';
 
 describe('OTLPTraceExporter - web', () => {
@@ -176,7 +176,12 @@ describe('OTLPTraceExporter - web', () => {
 
     describe('when "sendBeacon" is NOT available', () => {
       let server: any;
+      let clock: sinon.SinonFakeTimers;
       beforeEach(() => {
+        // fakeTimers is used to replace the next setTimeout which is
+        // located in sendWithXhr function called by the export method
+        clock = sinon.useFakeTimers();
+
         (window.navigator as any).sendBeacon = false;
         collectorTraceExporter = new OTLPTraceExporter(
           collectorExporterConfig
@@ -191,7 +196,7 @@ describe('OTLPTraceExporter - web', () => {
         collectorTraceExporter.export(spans, () => {
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const request = server.requests[0];
           assert.strictEqual(request.method, 'POST');
           assert.strictEqual(request.url, 'http://foo.bar.com');
@@ -211,9 +216,9 @@ describe('OTLPTraceExporter - web', () => {
           ensureWebResourceIsCorrect(resource);
 
           assert.strictEqual(stubBeacon.callCount, 0);
-
           ensureExportTraceServiceRequestIsSet(json);
 
+          clock.restore();
           done();
         });
       });
@@ -236,15 +241,15 @@ describe('OTLPTraceExporter - web', () => {
         collectorTraceExporter.export(spans, () => {
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const request = server.requests[0];
           request.respond(200);
-
           const response: any = spyLoggerDebug.args[2][0];
           assert.strictEqual(response, 'xhr success');
           assert.strictEqual(spyLoggerError.args.length, 0);
-
           assert.strictEqual(stubBeacon.callCount, 0);
+
+          clock.restore();
           done();
         });
       });
@@ -256,21 +261,24 @@ describe('OTLPTraceExporter - web', () => {
           done();
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const request = server.requests[0];
           request.respond(400);
+          clock.restore();
+          done();
         });
       });
 
       it('should send custom headers', done => {
         collectorTraceExporter.export(spans, () => {
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const request = server.requests[0];
           request.respond(200);
 
           assert.strictEqual(stubBeacon.callCount, 0);
+          clock.restore();
           done();
         });
       });
@@ -373,7 +381,12 @@ describe('OTLPTraceExporter - web', () => {
     });
 
     describe('when "sendBeacon" is available', () => {
+      let clock: sinon.SinonFakeTimers;
       beforeEach(() => {
+        // fakeTimers is used to replace the next setTimeout which is
+        // located in sendWithXhr function called by the export method
+        clock = sinon.useFakeTimers();
+
         collectorTraceExporter = new OTLPTraceExporter(
           collectorExporterConfig
         );
@@ -382,20 +395,26 @@ describe('OTLPTraceExporter - web', () => {
         collectorTraceExporter.export(spans, () => {
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const [{ requestHeaders }] = server.requests;
 
           ensureHeadersContain(requestHeaders, customHeaders);
           assert.strictEqual(stubBeacon.callCount, 0);
           assert.strictEqual(stubOpen.callCount, 0);
 
+          clock.restore();
           done();
         });
       });
     });
 
     describe('when "sendBeacon" is NOT available', () => {
+      let clock: sinon.SinonFakeTimers;
       beforeEach(() => {
+        // fakeTimers is used to replace the next setTimeout which is
+        // located in sendWithXhr function called by the export method
+        clock = sinon.useFakeTimers();
+
         (window.navigator as any).sendBeacon = false;
         collectorTraceExporter = new OTLPTraceExporter(
           collectorExporterConfig
@@ -406,13 +425,30 @@ describe('OTLPTraceExporter - web', () => {
         collectorTraceExporter.export(spans, () => {
         });
 
-        setTimeout(() => {
+        queueMicrotask(() => {
           const [{ requestHeaders }] = server.requests;
 
           ensureHeadersContain(requestHeaders, customHeaders);
           assert.strictEqual(stubBeacon.callCount, 0);
           assert.strictEqual(stubOpen.callCount, 0);
 
+          clock.restore();
+          done();
+        });
+      });
+      it('should log the timeout request error message', done => {
+        const responseSpy = sinon.spy();
+        collectorTraceExporter.export(spans, responseSpy);
+        clock.tick(10000);
+        clock.restore();
+
+        setTimeout(() => {
+          const result = responseSpy.args[0][0] as core.ExportResult;
+          assert.strictEqual(result.code, core.ExportResultCode.FAILED);
+          const error = result.error as OTLPExporterError;
+          assert.ok(error !== undefined);
+          assert.strictEqual(error.message, 'Request Timeout');
+
           done();
         });
       });