awsecscontainermetrics: update CpuUtilized metric calculation (#1283)

We had our internal discussion and we decided to report `CpuUtilized` metric in percentage. We studied our new calculation and verified with domain expert.

[Update] As the reviewer's suggestion, will create a separate PR for the following one.
Earlier we decided to add task and container metadata as resource attributes. However, CloudWatch `awsemf` exporter reads dimensions only from metric labels. As this exporter is major target for `awsecscontainermetrics` receiver, we need to add these metadata as metric labels. I discussed this in our OTel Metric SIG meeting. We also discussed internally and agreed to add them as metric labels as well.


This change will address both of these issues.

**Link to tracking Issue:**
#1282

Author: hossain-rayhan

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/accumulator.go

@@ -40,6 +40,10 @@ func (acc *metricDataAccumulator) getMetricsData(containerStatsMap map[string]Co
 		containerMetrics.MemoryReserved = *containerMetadata.Limits.Memory
 		containerMetrics.CPUReserved = *containerMetadata.Limits.CPU
 
+		if containerMetrics.CPUReserved > 0 {
+			containerMetrics.CPUUtilized = (containerMetrics.CPUUtilized / containerMetrics.CPUReserved)
+		}
+
 		containerResource := containerResource(containerMetadata)
 		for k, v := range taskResource.Labels {
 			containerResource.Labels[k] = v
@@ -65,6 +69,12 @@ func (acc *metricDataAccumulator) getMetricsData(containerStatsMap map[string]Co
 		taskMetrics.CPUReserved = *metadata.Limits.CPU
 	}
 
+	// taskMetrics.CPUReserved cannot be zero. In ECS, user needs to set CPU limit
+	// at least in one place (either in task level or in container level). If the
+	// task level CPULimit is not present, we calculate it from the summation of
+	// all container CPU limits.
+	taskMetrics.CPUUtilized = ((taskMetrics.CPUUsageInVCPU / taskMetrics.CPUReserved) * 100)
+
 	acc.accumulate(
 		taskResource,
 		convertToOCMetrics(TaskPrefix, taskMetrics, nil, nil, timestamp),

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/constant.go

@@ -48,6 +48,7 @@ const (
 	AttributeCPUOnlines         = "cpu.onlines"
 	AttributeCPUReserved        = "cpu.reserved"
 	AttributeCPUUtilized        = "cpu.utilized"
+	AttributeCPUUsageInVCPU     = "cpu.usage.vcpu"
 
 	AttributeNetworkRateRx = "network.rate.rx"
 	AttributeNetworkRateTx = "network.rate.tx"
@@ -70,4 +71,5 @@ const (
 	UnitBytesPerSec = "Bytes/Sec"
 	UnitCount       = "Count"
 	UnitVCpu        = "vCPU"
+	UnitPercent     = "Percent"
 )

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/docker_stats.go

@@ -14,16 +14,21 @@
 
 package awsecscontainermetrics
 
+import "time"
+
 // ContainerStats defines the structure for container stats
 type ContainerStats struct {
-	Name string `json:"name"`
-	ID   string `json:"id"`
+	Name         string    `json:"name"`
+	ID           string    `json:"id"`
+	Read         time.Time `json:"read"`
+	PreviousRead time.Time `json:"preread"`
 
 	Memory      MemoryStats             `json:"memory_stats,omitempty"`
 	Disk        DiskStats               `json:"blkio_stats,omitempty"`
 	Network     map[string]NetworkStats `json:"networks,omitempty"`
 	NetworkRate NetworkRateStats        `json:"network_rate_stats,omitempty"`
 	CPU         CPUStats                `json:"cpu_stats,omitempty"`
+	PreviousCPU CPUStats                `json:"precpu_stats,omitempty"`
 }
 
 // MemoryStats defines the memory stats

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/ecs_metrics.go

@@ -30,6 +30,7 @@ type ECSMetrics struct {
 	NumOfCPUCores        uint64
 	CPUReserved          float64
 	CPUUtilized          float64
+	CPUUsageInVCPU       float64
 
 	NetworkRateRxBytesPerSecond float64
 	NetworkRateTxBytesPerSecond float64

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/metrics_helper.go

@@ -17,11 +17,15 @@ package awsecscontainermetrics
 // getContainerMetrics generate ECS Container metrics from Container stats
 func getContainerMetrics(stats ContainerStats) ECSMetrics {
 	memoryUtilizedInMb := (*stats.Memory.Usage - stats.Memory.Stats["cache"]) / BytesInMiB
-
 	numOfCores := (uint64)(len(stats.CPU.CPUUsage.PerCPUUsage))
+	timeDiffSinceLastRead := (float64)(stats.Read.Sub(stats.PreviousRead).Nanoseconds())
 
-	// TODO: match with ECS Agent calculation and modify if needed
-	cpuUtilized := (float64)(*stats.CPU.CPUUsage.TotalUsage / numOfCores / 1024)
+	cpuUsageInVCpu := float64(0)
+	if timeDiffSinceLastRead > 0 {
+		cpuDelta := (float64)(*stats.CPU.CPUUsage.TotalUsage - *stats.PreviousCPU.CPUUsage.TotalUsage)
+		cpuUsageInVCpu = cpuDelta / timeDiffSinceLastRead
+	}
+	cpuUtilized := cpuUsageInVCpu * 100
 
 	netStatArray := getNetworkStats(stats.Network)
 
@@ -41,6 +45,7 @@ func getContainerMetrics(stats ContainerStats) ECSMetrics {
 	m.NumOfCPUCores = numOfCores
 	m.CPUOnlineCpus = *stats.CPU.OnlineCpus
 	m.SystemCPUUsage = *stats.CPU.SystemCPUUsage
+	m.CPUUsageInVCPU = cpuUsageInVCpu
 	m.CPUUtilized = cpuUtilized
 
 	if stats.NetworkRate == (NetworkRateStats{}) {
@@ -119,8 +124,8 @@ func aggregateTaskMetrics(taskMetrics *ECSMetrics, conMetrics ECSMetrics) {
 	taskMetrics.NumOfCPUCores += conMetrics.NumOfCPUCores
 	taskMetrics.CPUOnlineCpus += conMetrics.CPUOnlineCpus
 	taskMetrics.SystemCPUUsage += conMetrics.SystemCPUUsage
-	taskMetrics.CPUUtilized += conMetrics.CPUUtilized
 	taskMetrics.CPUReserved += conMetrics.CPUReserved
+	taskMetrics.CPUUsageInVCPU += conMetrics.CPUUsageInVCPU
 
 	taskMetrics.NetworkRateRxBytesPerSecond += conMetrics.NetworkRateRxBytesPerSecond
 	taskMetrics.NetworkRateTxBytesPerSecond += conMetrics.NetworkRateTxBytesPerSecond

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/metrics_helper_test.go

@@ -16,6 +16,7 @@ package awsecscontainermetrics
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
@@ -77,14 +78,33 @@ func TestGetContainerAndTaskMetrics(t *testing.T) {
 		CPUUtilized:    &v,
 		CPUReserved:    &v,
 	}
+
+	previousCPUUsage := CPUUsage{
+		TotalUsage:        &v,
+		UsageInKernelmode: &v,
+		UsageInUserMode:   &v,
+		PerCPUUsage:       percpu,
+	}
+
+	previousCPUStats := CPUStats{
+		CPUUsage:       previousCPUUsage,
+		OnlineCpus:     &v,
+		SystemCPUUsage: &v,
+		CPUUtilized:    &v,
+		CPUReserved:    &v,
+	}
+
 	containerStats := ContainerStats{
-		Name:        "test",
-		ID:          "001",
-		Memory:      mem,
-		Disk:        disk,
-		Network:     net,
-		NetworkRate: netRate,
-		CPU:         cpuStats,
+		Name:         "test",
+		ID:           "001",
+		Read:         time.Now(),
+		PreviousRead: time.Now().Add(-10 * time.Second),
+		Memory:       mem,
+		Disk:         disk,
+		Network:      net,
+		NetworkRate:  netRate,
+		CPU:          cpuStats,
+		PreviousCPU:  previousCPUStats,
 	}
 
 	containerMetrics := getContainerMetrics(containerStats)

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/translator.go

@@ -34,8 +34,9 @@ func convertToOCMetrics(prefix string, m ECSMetrics, labelKeys []*metricspb.Labe
 		intGauge(prefix+AttributeCPUCores, UnitCount, &m.NumOfCPUCores, labelKeys, labelValues),
 		intGauge(prefix+AttributeCPUOnlines, UnitCount, &m.CPUOnlineCpus, labelKeys, labelValues),
 		intCumulative(prefix+AttributeCPUSystemUsage, UnitNanoSecond, &m.SystemCPUUsage, labelKeys, labelValues),
-		doubleGauge(prefix+AttributeCPUUtilized, UnitVCpu, &m.CPUUtilized, labelKeys, labelValues),
+		doubleGauge(prefix+AttributeCPUUtilized, UnitPercent, &m.CPUUtilized, labelKeys, labelValues),
 		doubleGauge(prefix+AttributeCPUReserved, UnitVCpu, &m.CPUReserved, labelKeys, labelValues),
+		doubleGauge(prefix+AttributeCPUUsageInVCPU, UnitVCpu, &m.CPUUsageInVCPU, labelKeys, labelValues),
 
 		doubleGauge(prefix+AttributeNetworkRateRx, UnitBytesPerSec, &m.NetworkRateRxBytesPerSecond, labelKeys, labelValues),
 		doubleGauge(prefix+AttributeNetworkRateTx, UnitBytesPerSec, &m.NetworkRateTxBytesPerSecond, labelKeys, labelValues),

receiver/awsecscontainermetricsreceiver/awsecscontainermetrics/translator_test.go

@@ -39,7 +39,7 @@ func TestConvertToOTMetrics(t *testing.T) {
 	}
 
 	metrics := convertToOCMetrics("container.", m, labelKeys, labelValues, timestamp)
-	require.EqualValues(t, 25, len(metrics))
+	require.EqualValues(t, 26, len(metrics))
 }
 
 func TestIntGauge(t *testing.T) {