Skip to content

Commit abfe059

Browse files
svetlanabrennanlegendecas
svetlanabrennan
and
legendecas
authored
feat(otlp-exporter-base): add retries (#3207)
* feat(otlp-exporter-base): add retries to sendWithHttp Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add tests and update abort logic Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): fix lint Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add retry test Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add retry to browser exporter and add tests Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): refactor Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add jitter Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): initialize reqIsDestroyed to false Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add throttle logic Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add retry to readme Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): add changelog Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): update throttle time function Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): refactor sec difference in throttle fun Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): fix lint Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): fix lint Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): fix lint Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): refactor retrieve throttle time func Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): fix lint Signed-off-by: Svetlana Brennan <[email protected]> * feat(otlp-exporter-base): move parseRetryAfterToMills to utils file Signed-off-by: Svetlana Brennan <[email protected]> --------- Signed-off-by: Svetlana Brennan <[email protected]> Co-authored-by: Chengzhong Wu <[email protected]>
1 parent 708afd0 commit abfe059

File tree

10 files changed

+419
-117
lines changed

10 files changed

+419
-117
lines changed

‎experimental/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ All notable changes to experimental packages in this project will be documented
7272
* deps: remove unused proto-loader dependencies and update grpc-js and proto-loader versions [#3337](https://github.com/open-telemetry/opentelemetry-js/pull/3337) @seemk
7373
* feat(metrics-exporters): configure temporality via environment variable [#3305](https://github.com/open-telemetry/opentelemetry-js/pull/3305) @pichlermarc
7474
* feat(console-metric-exporter): add temporality configuration [#3387](https://github.com/open-telemetry/opentelemetry-js/pull/3387) @pichlermarc
75+
* feat(otlp-exporter-base): add retries [#3207](https://github.com/open-telemetry/opentelemetry-js/pull/3207) @svetlanabrennan
7576

7677
### :bug: (Bug Fix)
7778

‎experimental/packages/exporter-trace-otlp-http/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,21 @@ To override the default timeout duration, use the following options:
143143

144144
> Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables.
145145
146+
## OTLP Exporter Retry
147+
148+
OTLP requires that transient errors be handled with a [retry strategy](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry).
149+
150+
This retry policy has the following configuration, which there is currently no way to customize.
151+
152+
+ `DEFAULT_EXPORT_MAX_ATTEMPTS`: The maximum number of attempts, including the original request. Defaults to 5.
153+
+ `DEFAULT_EXPORT_INITIAL_BACKOFF`: The initial backoff duration. Defaults to 1 second.
154+
+ `DEFAULT_EXPORT_MAX_BACKOFF`: The maximum backoff duration. Defaults to 5 seconds.
155+
+ `DEFAULT_EXPORT_BACKOFF_MULTIPLIER`: The backoff multiplier. Defaults to 1.5.
156+
157+
This retry policy first checks if the response has a `'Retry-After'` header. If there is a `'Retry-After'` header, the exporter will wait the amount specified in the `'Retry-After'` header before retrying. If there is no `'Retry-After'` header, the exporter will use an exponential backoff with jitter retry strategy.
158+
159+
> The exporter will retry exporting within the [exporter timeout configuration](#Exporter-Timeout-Configuration) time.
160+
146161
## Running opentelemetry-collector locally to see the traces
147162

148163
1. Go to `examples/otlp-exporter-node`

‎experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,3 +582,123 @@ describe('when configuring via environment', () => {
582582
envSource.OTEL_EXPORTER_OTLP_HEADERS = '';
583583
});
584584
});
585+
586+
describe('export with retry - real http request destroyed', () => {
587+
let server: any;
588+
let collectorTraceExporter: OTLPTraceExporter;
589+
let collectorExporterConfig: OTLPExporterConfigBase;
590+
let spans: ReadableSpan[];
591+
592+
beforeEach(() => {
593+
server = sinon.fakeServer.create({
594+
autoRespond: true,
595+
});
596+
collectorExporterConfig = {
597+
timeoutMillis: 1500,
598+
};
599+
});
600+
601+
afterEach(() => {
602+
server.restore();
603+
});
604+
605+
describe('when "sendBeacon" is NOT available', () => {
606+
beforeEach(() => {
607+
(window.navigator as any).sendBeacon = false;
608+
collectorTraceExporter = new OTLPTraceExporter(collectorExporterConfig);
609+
});
610+
it('should log the timeout request error message when retrying with exponential backoff with jitter', done => {
611+
spans = [];
612+
spans.push(Object.assign({}, mockedReadableSpan));
613+
614+
let retry = 0;
615+
server.respondWith(
616+
'http://localhost:4318/v1/traces',
617+
function (xhr: any) {
618+
retry++;
619+
xhr.respond(503);
620+
}
621+
);
622+
623+
collectorTraceExporter.export(spans, result => {
624+
assert.strictEqual(result.code, core.ExportResultCode.FAILED);
625+
const error = result.error as OTLPExporterError;
626+
assert.ok(error !== undefined);
627+
assert.strictEqual(error.message, 'Request Timeout');
628+
assert.strictEqual(retry, 1);
629+
done();
630+
});
631+
}).timeout(3000);
632+
633+
it('should log the timeout request error message when retry-after header is set to 3 seconds', done => {
634+
spans = [];
635+
spans.push(Object.assign({}, mockedReadableSpan));
636+
637+
let retry = 0;
638+
server.respondWith(
639+
'http://localhost:4318/v1/traces',
640+
function (xhr: any) {
641+
retry++;
642+
xhr.respond(503, { 'Retry-After': 3 });
643+
}
644+
);
645+
646+
collectorTraceExporter.export(spans, result => {
647+
assert.strictEqual(result.code, core.ExportResultCode.FAILED);
648+
const error = result.error as OTLPExporterError;
649+
assert.ok(error !== undefined);
650+
assert.strictEqual(error.message, 'Request Timeout');
651+
assert.strictEqual(retry, 1);
652+
done();
653+
});
654+
}).timeout(3000);
655+
it('should log the timeout request error message when retry-after header is a date', done => {
656+
spans = [];
657+
spans.push(Object.assign({}, mockedReadableSpan));
658+
659+
let retry = 0;
660+
server.respondWith(
661+
'http://localhost:4318/v1/traces',
662+
function (xhr: any) {
663+
retry++;
664+
const d = new Date();
665+
d.setSeconds(d.getSeconds() + 1);
666+
xhr.respond(503, { 'Retry-After': d });
667+
}
668+
);
669+
670+
collectorTraceExporter.export(spans, result => {
671+
assert.strictEqual(result.code, core.ExportResultCode.FAILED);
672+
const error = result.error as OTLPExporterError;
673+
assert.ok(error !== undefined);
674+
assert.strictEqual(error.message, 'Request Timeout');
675+
assert.strictEqual(retry, 2);
676+
done();
677+
});
678+
}).timeout(3000);
679+
it('should log the timeout request error message when retry-after header is a date with long delay', done => {
680+
spans = [];
681+
spans.push(Object.assign({}, mockedReadableSpan));
682+
683+
let retry = 0;
684+
server.respondWith(
685+
'http://localhost:4318/v1/traces',
686+
function (xhr: any) {
687+
retry++;
688+
const d = new Date();
689+
d.setSeconds(d.getSeconds() + 120);
690+
xhr.respond(503, { 'Retry-After': d });
691+
}
692+
);
693+
694+
collectorTraceExporter.export(spans, result => {
695+
assert.strictEqual(result.code, core.ExportResultCode.FAILED);
696+
const error = result.error as OTLPExporterError;
697+
assert.ok(error !== undefined);
698+
assert.strictEqual(error.message, 'Request Timeout');
699+
assert.strictEqual(retry, 1);
700+
done();
701+
});
702+
}).timeout(3000);
703+
});
704+
});

‎experimental/packages/exporter-trace-otlp-http/test/node/CollectorTraceExporter.test.ts‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -551,38 +551,3 @@ describe('export - real http request destroyed before response received', () =>
551551
}, 0);
552552
});
553553
});
554-
555-
describe('export - real http request destroyed after response received', () => {
556-
let collectorExporter: OTLPTraceExporter;
557-
let collectorExporterConfig: OTLPExporterNodeConfigBase;
558-
let spans: ReadableSpan[];
559-
560-
const server = http.createServer((_, res) => {
561-
res.write('writing something');
562-
});
563-
before(done => {
564-
server.listen(8081, done);
565-
});
566-
after(done => {
567-
server.close(done);
568-
});
569-
it('should log the timeout request error message', done => {
570-
collectorExporterConfig = {
571-
url: 'http://localhost:8081',
572-
timeoutMillis: 300,
573-
};
574-
collectorExporter = new OTLPTraceExporter(collectorExporterConfig);
575-
spans = [];
576-
spans.push(Object.assign({}, mockedReadableSpan));
577-
578-
setTimeout(() => {
579-
collectorExporter.export(spans, result => {
580-
assert.strictEqual(result.code, core.ExportResultCode.FAILED);
581-
const error = result.error as OTLPExporterError;
582-
assert.ok(error !== undefined);
583-
assert.strictEqual(error.message, 'Request Timeout');
584-
done();
585-
});
586-
}, 0);
587-
});
588-
});

‎experimental/packages/exporter-trace-otlp-proto/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@ To override the default timeout duration, use the following options:
7272

7373
> Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables.
7474
75+
## OTLP Exporter Retry
76+
77+
OTLP requires that transient errors be handled with a [retry strategy](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry).
78+
79+
This retry policy has the following configuration, which there is currently no way to customize.
80+
81+
+ `DEFAULT_EXPORT_MAX_ATTEMPTS`: The maximum number of attempts, including the original request. Defaults to 5.
82+
+ `DEFAULT_EXPORT_INITIAL_BACKOFF`: The initial backoff duration. Defaults to 1 second.
83+
+ `DEFAULT_EXPORT_MAX_BACKOFF`: The maximum backoff duration. Defaults to 5 seconds.
84+
+ `DEFAULT_EXPORT_BACKOFF_MULTIPLIER`: The backoff multiplier. Defaults to 1.5.
85+
86+
This retry policy first checks if the response has a `'Retry-After'` header. If there is a `'Retry-After'` header, the exporter will wait the amount specified in the `'Retry-After'` header before retrying. If there is no `'Retry-After'` header, the exporter will use an exponential backoff with jitter retry strategy.
87+
88+
> The exporter will retry exporting within the [exporter timeout configuration](#Exporter-Timeout-Configuration) time.
89+
7590
## Running opentelemetry-collector locally to see the traces
7691

7792
1. Go to examples/otlp-exporter-node

‎experimental/packages/otlp-exporter-base/src/platform/browser/util.ts‎

Lines changed: 92 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@
1515
*/
1616
import { diag } from '@opentelemetry/api';
1717
import { OTLPExporterError } from '../../types';
18+
import {
19+
DEFAULT_EXPORT_MAX_ATTEMPTS,
20+
DEFAULT_EXPORT_INITIAL_BACKOFF,
21+
DEFAULT_EXPORT_BACKOFF_MULTIPLIER,
22+
DEFAULT_EXPORT_MAX_BACKOFF,
23+
isExportRetryable,
24+
parseRetryAfterToMills,
25+
} from '../../util';
1826

1927
/**
2028
* Send metrics/spans using browser navigator.sendBeacon
@@ -57,47 +65,99 @@ export function sendWithXhr(
5765
onSuccess: () => void,
5866
onError: (error: OTLPExporterError) => void
5967
): void {
60-
let reqIsDestroyed: boolean;
68+
let retryTimer: ReturnType<typeof setTimeout>;
69+
let xhr: XMLHttpRequest;
70+
let reqIsDestroyed = false;
6171

6272
const exporterTimer = setTimeout(() => {
73+
clearTimeout(retryTimer);
6374
reqIsDestroyed = true;
64-
xhr.abort();
75+
76+
if (xhr.readyState === XMLHttpRequest.DONE) {
77+
const err = new OTLPExporterError('Request Timeout');
78+
onError(err);
79+
} else {
80+
xhr.abort();
81+
}
6582
}, exporterTimeout);
6683

67-
const xhr = new XMLHttpRequest();
68-
xhr.open('POST', url);
84+
const sendWithRetry = (
85+
retries = DEFAULT_EXPORT_MAX_ATTEMPTS,
86+
minDelay = DEFAULT_EXPORT_INITIAL_BACKOFF
87+
) => {
88+
xhr = new XMLHttpRequest();
89+
xhr.open('POST', url);
6990

70-
const defaultHeaders = {
71-
Accept: 'application/json',
72-
'Content-Type': 'application/json',
73-
};
91+
const defaultHeaders = {
92+
Accept: 'application/json',
93+
'Content-Type': 'application/json',
94+
};
7495

75-
Object.entries({
76-
...defaultHeaders,
77-
...headers,
78-
}).forEach(([k, v]) => {
79-
xhr.setRequestHeader(k, v);
80-
});
96+
Object.entries({
97+
...defaultHeaders,
98+
...headers,
99+
}).forEach(([k, v]) => {
100+
xhr.setRequestHeader(k, v);
101+
});
81102

82-
xhr.send(body);
103+
xhr.send(body);
83104

84-
xhr.onreadystatechange = () => {
85-
if (xhr.readyState === XMLHttpRequest.DONE) {
86-
if (xhr.status >= 200 && xhr.status <= 299) {
87-
clearTimeout(exporterTimer);
88-
diag.debug('xhr success', body);
89-
onSuccess();
90-
} else if (reqIsDestroyed) {
91-
const error = new OTLPExporterError('Request Timeout', xhr.status);
92-
onError(error);
93-
} else {
94-
const error = new OTLPExporterError(
95-
`Failed to export with XHR (status: ${xhr.status})`,
96-
xhr.status
97-
);
98-
clearTimeout(exporterTimer);
99-
onError(error);
105+
xhr.onreadystatechange = () => {
106+
if (xhr.readyState === XMLHttpRequest.DONE && reqIsDestroyed === false) {
107+
if (xhr.status >= 200 && xhr.status <= 299) {
108+
diag.debug('xhr success', body);
109+
onSuccess();
110+
clearTimeout(exporterTimer);
111+
clearTimeout(retryTimer);
112+
} else if (xhr.status && isExportRetryable(xhr.status) && retries > 0) {
113+
let retryTime: number;
114+
minDelay = DEFAULT_EXPORT_BACKOFF_MULTIPLIER * minDelay;
115+
116+
// retry after interval specified in Retry-After header
117+
if (xhr.getResponseHeader('Retry-After')) {
118+
retryTime = parseRetryAfterToMills(
119+
xhr.getResponseHeader('Retry-After')!
120+
);
121+
} else {
122+
// exponential backoff with jitter
123+
retryTime = Math.round(
124+
Math.random() * (DEFAULT_EXPORT_MAX_BACKOFF - minDelay) + minDelay
125+
);
126+
}
127+
128+
retryTimer = setTimeout(() => {
129+
sendWithRetry(retries - 1, minDelay);
130+
}, retryTime);
131+
} else {
132+
const error = new OTLPExporterError(
133+
`Failed to export with XHR (status: ${xhr.status})`,
134+
xhr.status
135+
);
136+
onError(error);
137+
clearTimeout(exporterTimer);
138+
clearTimeout(retryTimer);
139+
}
100140
}
101-
}
141+
};
142+
143+
xhr.onabort = () => {
144+
if (reqIsDestroyed) {
145+
const err = new OTLPExporterError('Request Timeout');
146+
onError(err);
147+
}
148+
clearTimeout(exporterTimer);
149+
clearTimeout(retryTimer);
150+
};
151+
152+
xhr.onerror = () => {
153+
if (reqIsDestroyed) {
154+
const err = new OTLPExporterError('Request Timeout');
155+
onError(err);
156+
}
157+
clearTimeout(exporterTimer);
158+
clearTimeout(retryTimer);
159+
};
102160
};
161+
162+
sendWithRetry();
103163
}

0 commit comments

Comments
 (0)