From a587a192416a57ecb28fa8aa4bc96d11daa4ca6a Mon Sep 17 00:00:00 2001 From: Raphael Koh <72152843+kohrapha@users.noreply.github.com> Date: Wed, 21 Oct 2020 18:25:28 -0400 Subject: [PATCH 1/5] Restructure buildCWMetric logic (#1) * Restructure code to remove duplicated logic * Update format * Improve function and variable names * Extract logic for dimension creation and add test * Implement minor fixes * Remove changes to go.sum * Implement tests for getCWMetrics * Implement tests for buildCWMetric --- exporter/awsemfexporter/metric_translator.go | 267 +++---- .../awsemfexporter/metric_translator_test.go | 742 +++++++++++++++++- 2 files changed, 850 insertions(+), 159 deletions(-) diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index bfeaca03f52d8..f5f26758c86cd 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -76,9 +76,50 @@ type CWMetricStats struct { Sum float64 } +// Wrapper interface for: +// - pdata.IntDataPointSlice +// - pdata.DoubleDataPointSlice +// - pdata.IntHistogramDataPointSlice +// - pdata.DoubleHistogramDataPointSlice +type DataPoints interface { + Len() int + At(int) DataPoint +} + +// Wrapper interface for: +// - pdata.IntDataPoint +// - pdata.DoubleDataPoint +// - pdata.IntHistogramDataPoint +// - pdata.DoubleHistogramDataPoint +type DataPoint interface { + IsNil() bool + LabelsMap() pdata.StringMap +} + +// Define wrapper interfaces such that At(i) returns a `DataPoint` +type IntDataPointSlice struct { + pdata.IntDataPointSlice +} +type DoubleDataPointSlice struct { + pdata.DoubleDataPointSlice +} +type DoubleHistogramDataPointSlice struct { + pdata.DoubleHistogramDataPointSlice +} + +func (dps IntDataPointSlice) At(i int) DataPoint { + return dps.IntDataPointSlice.At(i) +} +func (dps DoubleDataPointSlice) At(i int) DataPoint { + return dps.DoubleDataPointSlice.At(i) +} +func (dps DoubleHistogramDataPointSlice) At(i int) DataPoint { + return dps.DoubleHistogramDataPointSlice.At(i) +} + // TranslateOtToCWMetric converts OT metrics to CloudWatch Metric format func TranslateOtToCWMetric(rm *pdata.ResourceMetrics, dimensionRollupOption string, namespace string) ([]*CWMetrics, int) { - var cwMetricLists []*CWMetrics + var cwMetricList []*CWMetrics totalDroppedMetrics := 0 var instrumentationLibName string @@ -117,11 +158,11 @@ func TranslateOtToCWMetric(rm *pdata.ResourceMetrics, dimensionRollupOption stri totalDroppedMetrics++ continue } - cwMetricList := getMeasurements(&metric, namespace, instrumentationLibName, dimensionRollupOption) - cwMetricLists = append(cwMetricLists, cwMetricList...) + cwMetrics := getCWMetrics(&metric, namespace, instrumentationLibName, dimensionRollupOption) + cwMetricList = append(cwMetricList, cwMetrics...) } } - return cwMetricLists, totalDroppedMetrics + return cwMetricList, totalDroppedMetrics } func TranslateCWMetricToEMF(cwMetricLists []*CWMetrics) []*LogEvent { @@ -150,217 +191,127 @@ func TranslateCWMetricToEMF(cwMetricLists []*CWMetrics) []*LogEvent { return ples } -func getMeasurements(metric *pdata.Metric, namespace string, instrumentationLibName string, dimensionRollupOption string) []*CWMetrics { +// Translates OTLP Metric to list of CW Metrics +func getCWMetrics(metric *pdata.Metric, namespace string, instrumentationLibName string, dimensionRollupOption string) []*CWMetrics { var result []*CWMetrics + var dps DataPoints // metric measure data from OT metricMeasure := make(map[string]string) - // metric measure slice could include multiple metric measures - metricSlice := []map[string]string{} metricMeasure["Name"] = metric.Name() metricMeasure["Unit"] = metric.Unit() - metricSlice = append(metricSlice, metricMeasure) + // metric measure slice could include multiple metric measures + metricSlice := []map[string]string{metricMeasure} + // Retrieve data points switch metric.DataType() { case pdata.MetricDataTypeIntGauge: - dps := metric.IntGauge().DataPoints() - if dps.Len() == 0 { - return result - } - for m := 0; m < dps.Len(); m++ { - dp := dps.At(m) - if dp.IsNil() { - continue - } - cwMetric := buildCWMetricFromDP(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) - if cwMetric != nil { - result = append(result, cwMetric) - } - } + dps = IntDataPointSlice{metric.IntGauge().DataPoints()} case pdata.MetricDataTypeDoubleGauge: - dps := metric.DoubleGauge().DataPoints() - if dps.Len() == 0 { - return result - } - for m := 0; m < dps.Len(); m++ { - dp := dps.At(m) - if dp.IsNil() { - continue - } - cwMetric := buildCWMetricFromDP(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) - if cwMetric != nil { - result = append(result, cwMetric) - } - } + dps = DoubleDataPointSlice{metric.DoubleGauge().DataPoints()} case pdata.MetricDataTypeIntSum: - dps := metric.IntSum().DataPoints() - if dps.Len() == 0 { - return result - } - for m := 0; m < dps.Len(); m++ { - dp := dps.At(m) - if dp.IsNil() { - continue - } - cwMetric := buildCWMetricFromDP(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) - if cwMetric != nil { - result = append(result, cwMetric) - } - } + dps = IntDataPointSlice{metric.IntSum().DataPoints()} case pdata.MetricDataTypeDoubleSum: - dps := metric.DoubleSum().DataPoints() - if dps.Len() == 0 { - return result - } - for m := 0; m < dps.Len(); m++ { - dp := dps.At(m) - if dp.IsNil() { - continue - } - cwMetric := buildCWMetricFromDP(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) - if cwMetric != nil { - result = append(result, cwMetric) - } - } + dps = DoubleDataPointSlice{metric.DoubleSum().DataPoints()} case pdata.MetricDataTypeDoubleHistogram: - dps := metric.DoubleHistogram().DataPoints() - if dps.Len() == 0 { - return result + dps = DoubleHistogramDataPointSlice{metric.DoubleHistogram().DataPoints()} + } + + if dps.Len() == 0 { + return result + } + for m := 0; m < dps.Len(); m++ { + dp := dps.At(m) + if dp.IsNil() { + continue } - for m := 0; m < dps.Len(); m++ { - dp := dps.At(m) - if dp.IsNil() { - continue - } - cwMetric := buildCWMetricFromHistogram(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) - if cwMetric != nil { - result = append(result, cwMetric) - } + cwMetric := buildCWMetric(dp, metric, namespace, metricSlice, instrumentationLibName, dimensionRollupOption) + if cwMetric != nil { + result = append(result, cwMetric) } } return result } -func buildCWMetricFromDP(dp interface{}, pmd *pdata.Metric, namespace string, metricSlice []map[string]string, instrumentationLibName string, dimensionRollupOption string) *CWMetrics { - // fields contains metric and dimensions key/value pairs - fieldsPairs := make(map[string]interface{}) - var dimensionArray [][]string - // Dimensions Slice - var dimensionSlice []string - var dimensionKV pdata.StringMap - switch metric := dp.(type) { - case pdata.IntDataPoint: - dimensionKV = metric.LabelsMap() - case pdata.DoubleDataPoint: - dimensionKV = metric.LabelsMap() - } - - dimensionKV.ForEach(func(k string, v pdata.StringValue) { - fieldsPairs[k] = v.Value() - dimensionSlice = append(dimensionSlice, k) - }) - // add OTel instrumentation lib name as an additional dimension if it is defined - if instrumentationLibName != noInstrumentationLibraryName { - fieldsPairs[OTellibDimensionKey] = instrumentationLibName - dimensionArray = append(dimensionArray, append(dimensionSlice, OTellibDimensionKey)) - } else { - dimensionArray = append(dimensionArray, dimensionSlice) +// Build CWMetric from DataPoint +func buildCWMetric(dp DataPoint, pmd *pdata.Metric, namespace string, metricSlice []map[string]string, instrumentationLibName string, dimensionRollupOption string) *CWMetrics { + dimensions, fields := createDimensions(dp, instrumentationLibName, dimensionRollupOption) + cwMeasurement := &CwMeasurement{ + Namespace: namespace, + Dimensions: dimensions, + Metrics: metricSlice, } - + metricList := []CwMeasurement{*cwMeasurement} timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + // Extract metric var metricVal interface{} switch metric := dp.(type) { case pdata.IntDataPoint: - // Put a fake but identical metric value here in order to add metric name into fieldsPairs + // Put a fake but identical metric value here in order to add metric name into fields // since calculateRate() needs metric name as one of metric identifiers - fieldsPairs[pmd.Name()] = int64(FakeMetricValue) + fields[pmd.Name()] = int64(FakeMetricValue) metricVal = metric.Value() if needsCalculateRate(pmd) { - metricVal = calculateRate(fieldsPairs, metric.Value(), timestamp) + metricVal = calculateRate(fields, metric.Value(), timestamp) } case pdata.DoubleDataPoint: - fieldsPairs[pmd.Name()] = float64(FakeMetricValue) + fields[pmd.Name()] = float64(FakeMetricValue) metricVal = metric.Value() if needsCalculateRate(pmd) { - metricVal = calculateRate(fieldsPairs, metric.Value(), timestamp) + metricVal = calculateRate(fields, metric.Value(), timestamp) + } + case pdata.DoubleHistogramDataPoint: + bucketBounds := metric.ExplicitBounds() + metricVal = &CWMetricStats{ + Min: bucketBounds[0], + Max: bucketBounds[len(bucketBounds)-1], + Count: metric.Count(), + Sum: metric.Sum(), } } if metricVal == nil { return nil } - fieldsPairs[pmd.Name()] = metricVal + fields[pmd.Name()] = metricVal - // EMF dimension attr takes list of list on dimensions. Including single/zero dimension rollup - rollupDimensionArray := dimensionRollup(dimensionRollupOption, dimensionSlice, instrumentationLibName) - if len(rollupDimensionArray) > 0 { - dimensionArray = append(dimensionArray, rollupDimensionArray...) - } - - cwMeasurement := &CwMeasurement{ - Namespace: namespace, - Dimensions: dimensionArray, - Metrics: metricSlice, - } - metricList := make([]CwMeasurement, 1) - metricList[0] = *cwMeasurement cwMetric := &CWMetrics{ Measurements: metricList, Timestamp: timestamp, - Fields: fieldsPairs, + Fields: fields, } return cwMetric } -func buildCWMetricFromHistogram(metric pdata.DoubleHistogramDataPoint, pmd *pdata.Metric, namespace string, metricSlice []map[string]string, instrumentationLibName string, dimensionRollupOption string) *CWMetrics { +// Create dimensions from DataPoint labels, where dimensions is a 2D array of dimension names, +// and initialize fields with dimension key/value pairs +func createDimensions(dp DataPoint, instrumentationLibName string, dimensionRollupOption string) (dimensions [][]string, fields map[string]interface{}) { // fields contains metric and dimensions key/value pairs - fieldsPairs := make(map[string]interface{}) - var dimensionArray [][]string - // Dimensions Slice - var dimensionSlice []string - dimensionKV := metric.LabelsMap() + fields = make(map[string]interface{}) + dimensionKV := dp.LabelsMap() + dimensionSlice := make([]string, dimensionKV.Len(), dimensionKV.Len()+1) + idx := 0 dimensionKV.ForEach(func(k string, v pdata.StringValue) { - fieldsPairs[k] = v.Value() - dimensionSlice = append(dimensionSlice, k) + fields[k] = v.Value() + dimensionSlice[idx] = k + idx++ }) - // add OTel instrumentation lib name as an additional dimension if it is defined + // Add OTel instrumentation lib name as an additional dimension if it is defined if instrumentationLibName != noInstrumentationLibraryName { - fieldsPairs[OTellibDimensionKey] = instrumentationLibName - dimensionArray = append(dimensionArray, append(dimensionSlice, OTellibDimensionKey)) + fields[OTellibDimensionKey] = instrumentationLibName + dimensions = append(dimensions, append(dimensionSlice, OTellibDimensionKey)) } else { - dimensionArray = append(dimensionArray, dimensionSlice) - } - - timestamp := time.Now().UnixNano() / int64(time.Millisecond) - - bucketBounds := metric.ExplicitBounds() - metricStats := &CWMetricStats{ - Min: bucketBounds[0], - Max: bucketBounds[len(bucketBounds)-1], - Count: metric.Count(), - Sum: metric.Sum(), + dimensions = append(dimensions, dimensionSlice) } - fieldsPairs[pmd.Name()] = metricStats // EMF dimension attr takes list of list on dimensions. Including single/zero dimension rollup rollupDimensionArray := dimensionRollup(dimensionRollupOption, dimensionSlice, instrumentationLibName) if len(rollupDimensionArray) > 0 { - dimensionArray = append(dimensionArray, rollupDimensionArray...) + dimensions = append(dimensions, rollupDimensionArray...) } - cwMeasurement := &CwMeasurement{ - Namespace: namespace, - Dimensions: dimensionArray, - Metrics: metricSlice, - } - metricList := make([]CwMeasurement, 1) - metricList[0] = *cwMeasurement - cwMetric := &CWMetrics{ - Measurements: metricList, - Timestamp: timestamp, - Fields: fieldsPairs, - } - return cwMetric + return } // rate is calculated by valDelta / timeDelta diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 8a4b2f189d3d8..74171dceb4a42 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -17,6 +17,7 @@ package awsemfexporter import ( "io/ioutil" "sort" + "strings" "testing" "time" @@ -245,7 +246,746 @@ func TestTranslateCWMetricToEMF(t *testing.T) { assert.Equal(t, readFromFile("testdata/testTranslateCWMetricToEMF.json"), *inputLogEvent[0].InputLogEvent.Message, "Expect to be equal") } -func TestGetMeasurements(t *testing.T) { +func TestGetCWMetrics(t *testing.T) { + namespace := "Namespace" + OTelLib := "OTelLib" + instrumentationLibName := "InstrLibName" + + testCases := []struct { + testName string + metric *metricspb.Metric + expected []*CWMetrics + }{ + { + "Int gauge", + &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "foo", + Type: metricspb.MetricDescriptor_GAUGE_INT64, + Unit: "Count", + LabelKeys: []*metricspb.LabelKey{ + {Key: "label1"}, + {Key: "label2"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "value1", HasValue: true}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_Int64Value{ + Int64Value: 1, + }, + }, + }, + }, + { + LabelValues: []*metricspb.LabelValue{ + {HasValue: false}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_Int64Value{ + Int64Value: 3, + }, + }, + }, + }, + }, + }, + []*CWMetrics{ + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label1", "label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": int64(1), + "label1": "value1", + "label2": "value2", + }, + }, + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": int64(3), + "label2": "value2", + }, + }, + }, + }, + { + "Double gauge", + &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "foo", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "Count", + LabelKeys: []*metricspb.LabelKey{ + {Key: "label1"}, + {Key: "label2"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "value1", HasValue: true}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.1, + }, + }, + }, + }, + { + LabelValues: []*metricspb.LabelValue{ + {HasValue: false}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.3, + }, + }, + }, + }, + }, + }, + []*CWMetrics{ + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label1", "label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0.1, + "label1": "value1", + "label2": "value2", + }, + }, + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0.3, + "label2": "value2", + }, + }, + }, + }, + { + "Int sum", + &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "foo", + Type: metricspb.MetricDescriptor_CUMULATIVE_INT64, + Unit: "Count", + LabelKeys: []*metricspb.LabelKey{ + {Key: "label1"}, + {Key: "label2"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "value1", HasValue: true}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_Int64Value{ + Int64Value: 1, + }, + }, + }, + }, + { + LabelValues: []*metricspb.LabelValue{ + {HasValue: false}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_Int64Value{ + Int64Value: 3, + }, + }, + }, + }, + }, + }, + []*CWMetrics{ + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label1", "label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0, + "label1": "value1", + "label2": "value2", + }, + }, + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0, + "label2": "value2", + }, + }, + }, + }, + { + "Double sum", + &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "foo", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + Unit: "Count", + LabelKeys: []*metricspb.LabelKey{ + {Key: "label1"}, + {Key: "label2"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "value1", HasValue: true}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.1, + }, + }, + }, + }, + { + LabelValues: []*metricspb.LabelValue{ + {HasValue: false}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.3, + }, + }, + }, + }, + }, + }, + []*CWMetrics{ + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label1", "label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0, + "label1": "value1", + "label2": "value2", + }, + }, + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Count"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": 0, + "label2": "value2", + }, + }, + }, + }, + { + "Double histogram", + &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "foo", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + Unit: "Seconds", + LabelKeys: []*metricspb.LabelKey{ + {Key: "label1"}, + {Key: "label2"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "value1", HasValue: true}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Sum: 15.0, + Count: 5, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10}, + }, + }, + }, + Buckets: []*metricspb.DistributionValue_Bucket{ + { + Count: 0, + }, + { + Count: 4, + }, + { + Count: 1, + }, + }, + }, + }, + }, + }, + }, + { + LabelValues: []*metricspb.LabelValue{ + {HasValue: false}, + {Value: "value2", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Sum: 35.0, + Count: 18, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10}, + }, + }, + }, + Buckets: []*metricspb.DistributionValue_Bucket{ + { + Count: 5, + }, + { + Count: 6, + }, + { + Count: 7, + }, + }, + }, + }, + }, + }, + }, + }, + }, + []*CWMetrics{ + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label1", "label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Seconds"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": &CWMetricStats{ + Min: 0, + Max: 10, + Sum: 15.0, + Count: 5, + }, + "label1": "value1", + "label2": "value2", + }, + }, + { + Measurements: []CwMeasurement{ + { + Namespace: namespace, + Dimensions: [][]string{ + {"label2", OTelLib}, + }, + Metrics: []map[string]string{ + {"Name": "foo", "Unit": "Seconds"}, + }, + }, + }, + Fields: map[string]interface{}{ + OTelLib: instrumentationLibName, + "foo": &CWMetricStats{ + Min: 0, + Max: 10, + Sum: 35.0, + Count: 18, + }, + "label2": "value2", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + oc := consumerdata.MetricsData{ + Node: &commonpb.Node{}, + Resource: &resourcepb.Resource{ + Labels: map[string]string{ + conventions.AttributeServiceName: "myServiceName", + conventions.AttributeServiceNamespace: "myServiceNS", + }, + }, + Metrics: []*metricspb.Metric{tc.metric}, + } + + // Retrieve *pdata.Metric + rms := internaldata.OCToMetrics(oc).ResourceMetrics() + assert.Equal(t, 1, rms.Len()) + ilms := rms.At(0).InstrumentationLibraryMetrics() + assert.Equal(t, 1, ilms.Len()) + metrics := ilms.At(0).Metrics() + assert.Equal(t, 1, metrics.Len()) + metric := metrics.At(0) + + cwMetrics := getCWMetrics(&metric, namespace, instrumentationLibName, "") + assert.Equal(t, len(tc.expected), len(cwMetrics)) + + for i, expected := range tc.expected { + cwMetric := cwMetrics[i] + assert.Equal(t, len(expected.Measurements), len(cwMetric.Measurements)) + assert.Equal(t, expected.Measurements, cwMetric.Measurements) + assert.Equal(t, len(expected.Fields), len(cwMetric.Fields)) + assert.Equal(t, expected.Fields, cwMetric.Fields) + } + }) + } +} + +func TestBuildCWMetric(t *testing.T) { + namespace := "Namespace" + instrLibName := "InstrLibName" + OTelLib := "OTelLib" + metricSlice := []map[string]string{ + map[string]string{ + "Name": "foo", + "Unit": "", + }, + } + metric := pdata.NewMetric() + metric.InitEmpty() + metric.SetName("foo") + + t.Run("Int gauge", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeIntGauge) + dp := pdata.NewIntDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(map[string]string{ + "label1": "value1", + }) + dp.SetValue(int64(-17)) + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + + assert.NotNil(t, cwMetric) + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: [][]string{{"label1", OTelLib}}, + Metrics: metricSlice, + } + assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + expectedFields := map[string]interface{}{ + OTelLib: instrLibName, + "foo": int64(-17), + "label1": "value1", + } + assert.Equal(t, expectedFields, cwMetric.Fields) + }) + + t.Run("Double gauge", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeDoubleGauge) + dp := pdata.NewDoubleDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(map[string]string{ + "label1": "value1", + }) + dp.SetValue(0.3) + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + + assert.NotNil(t, cwMetric) + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: [][]string{{"label1", OTelLib}}, + Metrics: metricSlice, + } + assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + expectedFields := map[string]interface{}{ + OTelLib: instrLibName, + "foo": 0.3, + "label1": "value1", + } + assert.Equal(t, expectedFields, cwMetric.Fields) + }) + + t.Run("Int sum", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeIntSum) + metric.IntSum().InitEmpty() + metric.IntSum().SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + dp := pdata.NewIntDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(map[string]string{ + "label1": "value1", + }) + dp.SetValue(int64(-17)) + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + + assert.NotNil(t, cwMetric) + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: [][]string{{"label1", OTelLib}}, + Metrics: metricSlice, + } + assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + expectedFields := map[string]interface{}{ + OTelLib: instrLibName, + "foo": 0, + "label1": "value1", + } + assert.Equal(t, expectedFields, cwMetric.Fields) + }) + + t.Run("Double sum", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeDoubleSum) + metric.DoubleSum().InitEmpty() + metric.DoubleSum().SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + dp := pdata.NewDoubleDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(map[string]string{ + "label1": "value1", + }) + dp.SetValue(0.3) + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + + assert.NotNil(t, cwMetric) + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: [][]string{{"label1", OTelLib}}, + Metrics: metricSlice, + } + assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + expectedFields := map[string]interface{}{ + OTelLib: instrLibName, + "foo": 0, + "label1": "value1", + } + assert.Equal(t, expectedFields, cwMetric.Fields) + }) + + t.Run("Double histogram", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeDoubleHistogram) + dp := pdata.NewDoubleHistogramDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(map[string]string{ + "label1": "value1", + }) + dp.SetCount(uint64(17)) + dp.SetSum(float64(17.13)) + dp.SetBucketCounts([]uint64{1, 2, 3}) + dp.SetExplicitBounds([]float64{1, 2, 3}) + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + + assert.NotNil(t, cwMetric) + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: [][]string{{"label1", OTelLib}}, + Metrics: metricSlice, + } + assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + expectedFields := map[string]interface{}{ + OTelLib: instrLibName, + "foo": &CWMetricStats{ + Min: 1, + Max: 3, + Sum: 17.13, + Count: 17, + }, + "label1": "value1", + } + assert.Equal(t, expectedFields, cwMetric.Fields) + }) + + t.Run("Invalid datapoint type", func(t *testing.T) { + metric.SetDataType(pdata.MetricDataTypeIntGauge) + dp := pdata.NewIntHistogramDataPoint() + dp.InitEmpty() + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + assert.Nil(t, cwMetric) + }) +} + +func TestCreateDimensions(t *testing.T) { + OTelLib := "OTelLib" + testCases := []struct { + testName string + labels map[string]string + dims [][]string + }{ + { + "single label", + map[string]string{"a": "foo"}, + [][]string{ + {"a", OTelLib}, + {OTelLib}, + }, + }, + { + "multiple labels", + map[string]string{"a": "foo", "b": "bar"}, + [][]string{ + {"a", "b", OTelLib}, + {OTelLib}, + {OTelLib, "a"}, + {OTelLib, "b"}, + }, + }, + { + "no labels", + map[string]string{}, + [][]string{ + {OTelLib}, + }, + }, + } + + sliceSorter := func(slice [][]string) func(a, b int) bool { + stringified := make([]string, len(slice)) + for i, v := range slice { + stringified[i] = strings.Join(v, ",") + } + return func(i, j int) bool { + return stringified[i] > stringified[j] + } + } + + for _, tc := range testCases { + dp := pdata.NewIntDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(tc.labels) + dimensions, fields := createDimensions(dp, OTelLib, ZeroAndSingleDimensionRollup) + + // Sort slice for equality check + sort.Slice(tc.dims, sliceSorter(tc.dims)) + sort.Slice(dimensions, sliceSorter(dimensions)) + + assert.Equal(t, tc.dims, dimensions) + + expectedFields := make(map[string]interface{}) + for k, v := range tc.labels { + expectedFields[k] = v + } + expectedFields[OTellibDimensionKey] = OTelLib + + assert.Equal(t, expectedFields, fields) + } } From f3ad7e98185404803b5b295f4a0e20d3236675d2 Mon Sep 17 00:00:00 2001 From: Raphael Koh Date: Thu, 22 Oct 2020 13:35:18 -0400 Subject: [PATCH 2/5] Format metric_translator_test.go --- .../awsemfexporter/metric_translator_test.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 74171dceb4a42..a47c8a5bec2df 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -779,9 +779,9 @@ func TestBuildCWMetric(t *testing.T) { assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) expectedMeasurement := CwMeasurement{ - Namespace: namespace, + Namespace: namespace, Dimensions: [][]string{{"label1", OTelLib}}, - Metrics: metricSlice, + Metrics: metricSlice, } assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ @@ -806,9 +806,9 @@ func TestBuildCWMetric(t *testing.T) { assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) expectedMeasurement := CwMeasurement{ - Namespace: namespace, + Namespace: namespace, Dimensions: [][]string{{"label1", OTelLib}}, - Metrics: metricSlice, + Metrics: metricSlice, } assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ @@ -835,9 +835,9 @@ func TestBuildCWMetric(t *testing.T) { assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) expectedMeasurement := CwMeasurement{ - Namespace: namespace, + Namespace: namespace, Dimensions: [][]string{{"label1", OTelLib}}, - Metrics: metricSlice, + Metrics: metricSlice, } assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ @@ -864,9 +864,9 @@ func TestBuildCWMetric(t *testing.T) { assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) expectedMeasurement := CwMeasurement{ - Namespace: namespace, + Namespace: namespace, Dimensions: [][]string{{"label1", OTelLib}}, - Metrics: metricSlice, + Metrics: metricSlice, } assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ @@ -894,14 +894,14 @@ func TestBuildCWMetric(t *testing.T) { assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) expectedMeasurement := CwMeasurement{ - Namespace: namespace, + Namespace: namespace, Dimensions: [][]string{{"label1", OTelLib}}, - Metrics: metricSlice, + Metrics: metricSlice, } assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ - OTelLib: instrLibName, - "foo": &CWMetricStats{ + OTelLib: instrLibName, + "foo": &CWMetricStats{ Min: 1, Max: 3, Sum: 17.13, @@ -911,7 +911,7 @@ func TestBuildCWMetric(t *testing.T) { } assert.Equal(t, expectedFields, cwMetric.Fields) }) - + t.Run("Invalid datapoint type", func(t *testing.T) { metric.SetDataType(pdata.MetricDataTypeIntGauge) dp := pdata.NewIntHistogramDataPoint() From 820f516825fee00f07da7935c49771fb25abf0bb Mon Sep 17 00:00:00 2001 From: Raphael Koh Date: Thu, 22 Oct 2020 13:37:09 -0400 Subject: [PATCH 3/5] Run with gofmt -s --- exporter/awsemfexporter/metric_translator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index a47c8a5bec2df..30117adb1a6d0 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -756,7 +756,7 @@ func TestBuildCWMetric(t *testing.T) { instrLibName := "InstrLibName" OTelLib := "OTelLib" metricSlice := []map[string]string{ - map[string]string{ + { "Name": "foo", "Unit": "", }, From 4360a095868e935087fba010107eac1a4da527b3 Mon Sep 17 00:00:00 2001 From: Raphael Koh Date: Thu, 22 Oct 2020 14:05:58 -0400 Subject: [PATCH 4/5] Disregard ordering of dimensions in test case --- .../awsemfexporter/metric_translator_test.go | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 30117adb1a6d0..9cf61e3e262ee 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -959,6 +959,7 @@ func TestCreateDimensions(t *testing.T) { sliceSorter := func(slice [][]string) func(a, b int) bool { stringified := make([]string, len(slice)) for i, v := range slice { + sort.Strings(v) stringified[i] = strings.Join(v, ",") } return func(i, j int) bool { @@ -967,24 +968,26 @@ func TestCreateDimensions(t *testing.T) { } for _, tc := range testCases { - dp := pdata.NewIntDataPoint() - dp.InitEmpty() - dp.LabelsMap().InitFromMap(tc.labels) - dimensions, fields := createDimensions(dp, OTelLib, ZeroAndSingleDimensionRollup) + t.Run(tc.testName, func(t *testing.T) { + dp := pdata.NewIntDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(tc.labels) + dimensions, fields := createDimensions(dp, OTelLib, ZeroAndSingleDimensionRollup) - // Sort slice for equality check - sort.Slice(tc.dims, sliceSorter(tc.dims)) - sort.Slice(dimensions, sliceSorter(dimensions)) + // Sort slice for equality check + sort.Slice(tc.dims, sliceSorter(tc.dims)) + sort.Slice(dimensions, sliceSorter(dimensions)) - assert.Equal(t, tc.dims, dimensions) + assert.Equal(t, tc.dims, dimensions) - expectedFields := make(map[string]interface{}) - for k, v := range tc.labels { - expectedFields[k] = v - } - expectedFields[OTellibDimensionKey] = OTelLib + expectedFields := make(map[string]interface{}) + for k, v := range tc.labels { + expectedFields[k] = v + } + expectedFields[OTellibDimensionKey] = OTelLib - assert.Equal(t, expectedFields, fields) + assert.Equal(t, expectedFields, fields) + }) } } From 10fb898bcc6d87c5341cf6a518937c994a595f6d Mon Sep 17 00:00:00 2001 From: Raphael Koh Date: Thu, 22 Oct 2020 14:34:47 -0400 Subject: [PATCH 5/5] Perform dimension equality checking as a helper function --- .../awsemfexporter/metric_translator_test.go | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 9cf61e3e262ee..c1ad4ca9ec8c6 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -33,6 +33,25 @@ import ( "go.opentelemetry.io/collector/translator/internaldata" ) +// Asserts whether dimension sets are equal (i.e. has same sets of dimensions) +func assertDimsEqual(t *testing.T, expected, actual [][]string) { + // Convert to string for easier sorting + expectedStringified := make([]string, len(expected)) + actualStringified := make([]string, len(actual)) + for i, v := range expected { + sort.Strings(v) + expectedStringified[i] = strings.Join(v, ",") + } + for i, v := range actual { + sort.Strings(v) + actualStringified[i] = strings.Join(v, ",") + } + // Sort across dimension sets for equality checking + sort.Strings(expectedStringified) + sort.Strings(actualStringified) + assert.Equal(t, expectedStringified, actualStringified) +} + func TestTranslateOtToCWMetricWithInstrLibrary(t *testing.T) { md := createMetricTestData() @@ -956,17 +975,6 @@ func TestCreateDimensions(t *testing.T) { }, } - sliceSorter := func(slice [][]string) func(a, b int) bool { - stringified := make([]string, len(slice)) - for i, v := range slice { - sort.Strings(v) - stringified[i] = strings.Join(v, ",") - } - return func(i, j int) bool { - return stringified[i] > stringified[j] - } - } - for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { dp := pdata.NewIntDataPoint() @@ -974,11 +982,7 @@ func TestCreateDimensions(t *testing.T) { dp.LabelsMap().InitFromMap(tc.labels) dimensions, fields := createDimensions(dp, OTelLib, ZeroAndSingleDimensionRollup) - // Sort slice for equality check - sort.Slice(tc.dims, sliceSorter(tc.dims)) - sort.Slice(dimensions, sliceSorter(dimensions)) - - assert.Equal(t, tc.dims, dimensions) + assertDimsEqual(t, tc.dims, dimensions) expectedFields := make(map[string]interface{}) for k, v := range tc.labels {