feat(tracing): support for truncation of span attribute values

jtmalinowski · jtmalinowski · commit 9f7c5f562a8e · 2020-10-28T18:17:50.000+01:00
diff --git a/packages/opentelemetry-core/src/common/attributes.ts b/packages/opentelemetry-core/src/common/attributes.ts
@@ -47,6 +47,47 @@ export function isAttributeValue(val: unknown): val is AttributeValue {
   return isValidPrimitiveAttributeValue(val);
 }
 
+export function truncateValueIfTooLong(
+  value: AttributeValue,
+  limit: number | null,
+  truncationWarningCallback = () => {}
+): AttributeValue {
+  if (limit === null) {
+    return value;
+  }
+
+  if (limit < 32) {
+    throw new Error('Value size limit cannot be lower than 32.');
+  }
+
+  if (typeof value === 'boolean' || typeof value === 'number') {
+    // these types can't exceed the attribute value size limit
+    return value;
+  }
+
+  if (Array.isArray(value)) {
+    // note: this is potentially incompatible with a given exporter
+    const serialized = JSON.stringify(value);
+
+    if (serialized.length > limit) {
+      return truncateValueIfTooLong(
+        serialized,
+        limit,
+        truncationWarningCallback
+      );
+    }
+
+    return value;
+  }
+
+  if (value.length > limit) {
+    truncationWarningCallback();
+    return value.substring(0, limit);
+  }
+
+  return value;
+}
+
 function isHomogeneousAttributeValueArray(arr: unknown[]): boolean {
   let type: string | undefined;
 
diff --git a/packages/opentelemetry-core/src/utils/environment.ts b/packages/opentelemetry-core/src/utils/environment.ts
@@ -25,10 +25,12 @@ export interface ENVIRONMENT {
   OTEL_LOG_LEVEL?: LogLevel;
   OTEL_NO_PATCH_MODULES?: string;
   OTEL_SAMPLING_PROBABILITY?: number;
+  OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT?: number | null;
 }
 
 const ENVIRONMENT_NUMBERS: Partial<keyof ENVIRONMENT>[] = [
   'OTEL_SAMPLING_PROBABILITY',
+  'OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT',
 ];
 
 /**
@@ -38,6 +40,7 @@ export const DEFAULT_ENVIRONMENT: Required<ENVIRONMENT> = {
   OTEL_NO_PATCH_MODULES: '',
   OTEL_LOG_LEVEL: LogLevel.INFO,
   OTEL_SAMPLING_PROBABILITY: 1,
+  OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT: null,
 };
 
 /**
@@ -116,6 +119,10 @@ export function parseEnvironment(values: ENVIRONMENT_MAP): ENVIRONMENT {
         setLogLevelFromEnv(key, environment, values);
         break;
 
+      case 'OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT':
+        parseNumber(key, environment, values, 32, Number.MAX_SAFE_INTEGER);
+        break;
+
       default:
         if (ENVIRONMENT_NUMBERS.indexOf(key) >= 0) {
           parseNumber(key, environment, values);
diff --git a/packages/opentelemetry-core/test/common/attributes.test.ts b/packages/opentelemetry-core/test/common/attributes.test.ts
@@ -15,9 +15,11 @@
  */
 
 import * as assert from 'assert';
+import * as sinon from 'sinon';
 import {
   isAttributeValue,
   sanitizeAttributes,
+  truncateValueIfTooLong,
 } from '../../src/common/attributes';
 
 describe('attributes', () => {
@@ -103,4 +105,62 @@ describe('attributes', () => {
       assert.strictEqual(attributes.arr[0], 'unmodified');
     });
   });
+  describe('#truncateValueIfTooLong', () => {
+    it('should not truncate any given value if the limit is not set', () => {
+      assert.strictEqual(truncateValueIfTooLong('a', null), 'a');
+      assert.strictEqual(truncateValueIfTooLong(1, null), 1);
+      assert.strictEqual(truncateValueIfTooLong(true, null), true);
+
+      const arrayRef: string[] = [];
+      assert.strictEqual(truncateValueIfTooLong(arrayRef, null), arrayRef);
+    });
+
+    it('passes numbers and bools through', () => {
+      assert.strictEqual(truncateValueIfTooLong(true, 32), true);
+      assert.strictEqual(truncateValueIfTooLong(false, 32), false);
+      assert.strictEqual(truncateValueIfTooLong(1, 32), 1);
+    });
+
+    it('truncates strings if they are longer than the limit', () => {
+      assert.strictEqual(
+        truncateValueIfTooLong('a'.repeat(100), 100),
+        'a'.repeat(100)
+      );
+      assert.strictEqual(
+        truncateValueIfTooLong('a'.repeat(101), 100),
+        'a'.repeat(100)
+      );
+    });
+
+    it('serializes and truncates arrays if they are longer than the limit', () => {
+      assert.strictEqual(
+        truncateValueIfTooLong(['a'.repeat(100)], 32),
+        '["' + 'a'.repeat(30)
+      );
+      assert.strictEqual(
+        truncateValueIfTooLong(
+          [...new Array(10).keys()].map(() => 1000),
+          32
+        ),
+        '[' + '1000,'.repeat(6) + '1'
+      );
+      assert.strictEqual(
+        truncateValueIfTooLong(
+          [...new Array(10).keys()].map(() => true),
+          32
+        ),
+        '[' + 'true,'.repeat(6) + 't'
+      );
+    });
+
+    it('executes callback if a value was truncated', () => {
+      const fakeCallback = sinon.spy();
+
+      truncateValueIfTooLong('a'.repeat(32), 32, fakeCallback);
+      assert.ok(!fakeCallback.called);
+
+      truncateValueIfTooLong('a'.repeat(33), 32, fakeCallback);
+      assert(fakeCallback.called);
+    });
+  });
 });
diff --git a/packages/opentelemetry-core/test/utils/environment.test.ts b/packages/opentelemetry-core/test/utils/environment.test.ts
@@ -77,11 +77,13 @@ describe('environment', () => {
         OTEL_NO_PATCH_MODULES: 'a,b,c',
         OTEL_LOG_LEVEL: 'ERROR',
         OTEL_SAMPLING_PROBABILITY: '0.5',
+        OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT: '32',
       });
       const env = getEnv();
       assert.strictEqual(env.OTEL_NO_PATCH_MODULES, 'a,b,c');
       assert.strictEqual(env.OTEL_LOG_LEVEL, LogLevel.ERROR);
       assert.strictEqual(env.OTEL_SAMPLING_PROBABILITY, 0.5);
+      assert.strictEqual(env.OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT, 32);
     });
 
     it('should parse OTEL_LOG_LEVEL despite casing', () => {
diff --git a/packages/opentelemetry-tracing/src/Span.ts b/packages/opentelemetry-tracing/src/Span.ts
@@ -16,12 +16,13 @@
 
 import * as api from '@opentelemetry/api';
 import {
-  isAttributeValue,
   hrTime,
   hrTimeDuration,
   InstrumentationLibrary,
+  isAttributeValue,
   isTimeInput,
   timeInputToHrTime,
+  truncateValueIfTooLong,
 } from '@opentelemetry/core';
 import { Resource } from '@opentelemetry/resources';
 import {
@@ -56,6 +57,7 @@ export class Span implements api.Span, ReadableSpan {
   endTime: api.HrTime = [0, 0];
   private _ended = false;
   private _duration: api.HrTime = [-1, -1];
+  private _hasTruncated = false;
   private readonly _logger: api.Logger;
   private readonly _spanProcessor: SpanProcessor;
   private readonly _traceParams: TraceParams;
@@ -112,7 +114,18 @@ export class Span implements api.Span, ReadableSpan {
         delete this.attributes[attributeKeyToDelete];
       }
     }
-    this.attributes[key] = value;
+
+    console.log('limit', this._traceParams.spanAttributeValueSizeLimit);
+    this.attributes[key] = truncateValueIfTooLong(
+      value,
+      this._traceParams.spanAttributeValueSizeLimit || null,
+      this._hasTruncated
+        ? undefined
+        : () => {
+            this._hasTruncated = true;
+            this._logger.warn(`Span attribute value truncated at key: ${key}.`);
+          }
+    );
     return this;
   }
 
diff --git a/packages/opentelemetry-tracing/src/Tracer.ts b/packages/opentelemetry-tracing/src/Tracer.ts
@@ -55,6 +55,15 @@ export class Tracer implements api.Tracer {
     this.resource = _tracerProvider.resource;
     this.instrumentationLibrary = instrumentationLibrary;
     this.logger = config.logger || new ConsoleLogger(config.logLevel);
+
+    const configuredAttributeLimit =
+      this._traceParams.spanAttributeValueSizeLimit || null;
+    if (configuredAttributeLimit !== null && configuredAttributeLimit < 32) {
+      this.logger.warn(
+        'OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT was set to a value lower than 32, which is not allowed, limit of 32 will be applied.'
+      );
+      this._traceParams.spanAttributeValueSizeLimit = 32;
+    }
   }
 
   /**
diff --git a/packages/opentelemetry-tracing/src/config.ts b/packages/opentelemetry-tracing/src/config.ts
@@ -36,6 +36,7 @@ export const DEFAULT_CONFIG = {
     numberOfAttributesPerSpan: DEFAULT_MAX_ATTRIBUTES_PER_SPAN,
     numberOfLinksPerSpan: DEFAULT_MAX_LINKS_PER_SPAN,
     numberOfEventsPerSpan: DEFAULT_MAX_EVENTS_PER_SPAN,
+    spanAttributeValueSizeLimit: getEnv().OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT,
   },
   gracefulShutdown: true,
 };
diff --git a/packages/opentelemetry-tracing/src/types.ts b/packages/opentelemetry-tracing/src/types.ts
@@ -74,6 +74,8 @@ export interface TraceParams {
   numberOfLinksPerSpan?: number;
   /** numberOfEventsPerSpan is number of message events per span */
   numberOfEventsPerSpan?: number;
+  /** this field defines maximum length of attribute value before it is truncated */
+  spanAttributeValueSizeLimit?: number | null;
 }
 
 /** Interface configuration for a buffer. */
diff --git a/packages/opentelemetry-tracing/src/utility.ts b/packages/opentelemetry-tracing/src/utility.ts
@@ -56,6 +56,9 @@ export function mergeConfig(userConfig: TracerConfig) {
       traceParams.numberOfEventsPerSpan || DEFAULT_MAX_EVENTS_PER_SPAN;
     target.traceParams.numberOfLinksPerSpan =
       traceParams.numberOfLinksPerSpan || DEFAULT_MAX_LINKS_PER_SPAN;
+    target.traceParams.spanAttributeValueSizeLimit =
+      target.traceParams.spanAttributeValueSizeLimit ||
+      getEnv().OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT;
   }
   return target;
 }
diff --git a/packages/opentelemetry-tracing/test/BasicTracerProvider.test.ts b/packages/opentelemetry-tracing/test/BasicTracerProvider.test.ts
@@ -78,6 +78,7 @@ describe('BasicTracerProvider', () => {
         numberOfAttributesPerSpan: 32,
         numberOfEventsPerSpan: 128,
         numberOfLinksPerSpan: 32,
+        spanAttributeValueSizeLimit: null,
       });
     });
 
@@ -91,6 +92,7 @@ describe('BasicTracerProvider', () => {
         numberOfAttributesPerSpan: 100,
         numberOfEventsPerSpan: 128,
         numberOfLinksPerSpan: 32,
+        spanAttributeValueSizeLimit: null,
       });
     });
 
@@ -104,6 +106,7 @@ describe('BasicTracerProvider', () => {
         numberOfAttributesPerSpan: 32,
         numberOfEventsPerSpan: 300,
         numberOfLinksPerSpan: 32,
+        spanAttributeValueSizeLimit: null,
       });
     });
 
@@ -117,6 +120,21 @@ describe('BasicTracerProvider', () => {
         numberOfAttributesPerSpan: 32,
         numberOfEventsPerSpan: 128,
         numberOfLinksPerSpan: 10,
+        spanAttributeValueSizeLimit: null,
+      });
+    });
+
+    it('should construct an instance with customized spanAttributeValueSizeLimit trace params', () => {
+      const tracer = new BasicTracerProvider({
+        traceParams: {
+          spanAttributeValueSizeLimit: 100,
+        },
+      }).getTracer('default');
+      assert.deepStrictEqual(tracer.getActiveTraceParams(), {
+        numberOfAttributesPerSpan: 32,
+        numberOfEventsPerSpan: 128,
+        numberOfLinksPerSpan: 32,
+        spanAttributeValueSizeLimit: 100,
       });
     });
 
diff --git a/packages/opentelemetry-tracing/test/Span.test.ts b/packages/opentelemetry-tracing/test/Span.test.ts
@@ -31,6 +31,7 @@ import {
 } from '@opentelemetry/core';
 import { ExceptionAttribute } from '@opentelemetry/semantic-conventions';
 import * as assert from 'assert';
+import sinon = require('sinon');
 import { BasicTracerProvider, Span, SpanProcessor } from '../src';
 
 const performanceTimeOrigin = hrTime();
@@ -255,6 +256,86 @@ describe('Span', () => {
     assert.strictEqual(span.attributes['foo149'], 'bar149');
   });
 
+  it('should truncate attribute values exceeding length limit', () => {
+    const tracerWithLimit = new BasicTracerProvider({
+      logger: new NoopLogger(),
+      traceParams: {
+        spanAttributeValueSizeLimit: 100,
+      },
+    }).getTracer('default');
+
+    const spanWithLimit = new Span(
+      tracerWithLimit,
+      name,
+      spanContext,
+      SpanKind.CLIENT
+    );
+    const spanWithoutLimit = new Span(
+      tracer,
+      name,
+      spanContext,
+      SpanKind.CLIENT
+    );
+
+    spanWithLimit.setAttribute('attr under limit', 'a'.repeat(100));
+    assert.strictEqual(
+      spanWithLimit.attributes['attr under limit'],
+      'a'.repeat(100)
+    );
+    spanWithoutLimit.setAttribute('attr under limit', 'a'.repeat(100));
+    assert.strictEqual(
+      spanWithoutLimit.attributes['attr under limit'],
+      'a'.repeat(100)
+    );
+
+    spanWithLimit.setAttribute('attr over limit', 'b'.repeat(101));
+    assert.strictEqual(
+      spanWithLimit.attributes['attr over limit'],
+      'b'.repeat(100)
+    );
+    spanWithoutLimit.setAttribute('attr over limit', 'b'.repeat(101));
+    assert.strictEqual(
+      spanWithoutLimit.attributes['attr over limit'],
+      'b'.repeat(101)
+    );
+  });
+
+  it('should warn once when truncating attribute values exceeding length limit', () => {
+    const logger = new NoopLogger();
+    const loggerWarnSpy = sinon.spy(logger, 'warn');
+
+    const tracerWithLimit = new BasicTracerProvider({
+      logger,
+      traceParams: {
+        spanAttributeValueSizeLimit: 100,
+      },
+    }).getTracer('default');
+
+    const spanWithLimit = new Span(
+      tracerWithLimit,
+      name,
+      spanContext,
+      SpanKind.CLIENT
+    );
+
+    spanWithLimit.setAttribute('longAttr', 'b'.repeat(100));
+    assert(!loggerWarnSpy.called);
+
+    spanWithLimit.setAttribute('longAttr', 'b'.repeat(101));
+    assert(
+      loggerWarnSpy.withArgs('Span attribute value truncated at key: longAttr.')
+        .calledOnce
+    );
+
+    spanWithLimit.setAttribute('longAttr', 'c'.repeat(102));
+    assert(
+      loggerWarnSpy.withArgs('Span attribute value truncated at key: longAttr.')
+        .calledOnce
+    );
+
+    assert.strictEqual(spanWithLimit.attributes.longAttr, 'c'.repeat(100));
+  });
+
   it('should set an error status', () => {
     const span = new Span(tracer, name, spanContext, SpanKind.CLIENT);
     span.setStatus({

Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,8 @@ export interface TraceParams {`
`74`	`74`	`numberOfLinksPerSpan?: number;`
`75`	`75`	`/** numberOfEventsPerSpan is number of message events per span */`
`76`	`76`	`numberOfEventsPerSpan?: number;`
	`77`	`+ /** this field defines maximum length of attribute value before it is truncated */`
	`78`	`+ spanAttributeValueSizeLimit?: number \| null;`
`77`	`79`	`}`
`78`	`80`
`79`	`81`	`/** Interface configuration for a buffer. */`
Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,9 @@ export function mergeConfig(userConfig: TracerConfig) {`
`56`	`56`	`traceParams.numberOfEventsPerSpan \|\| DEFAULT_MAX_EVENTS_PER_SPAN;`
`57`	`57`	`target.traceParams.numberOfLinksPerSpan =`
`58`	`58`	`traceParams.numberOfLinksPerSpan \|\| DEFAULT_MAX_LINKS_PER_SPAN;`
	`59`	`+ target.traceParams.spanAttributeValueSizeLimit =`
	`60`	`+ target.traceParams.spanAttributeValueSizeLimit \|\|`
	`61`	`+ getEnv().OTEL_SPAN_ATTRIBUTE_VALUE_SIZE_LIMIT;`
`59`	`62`	`}`
`60`	`63`	`return target;`
`61`	`64`	`}`