Add support for System.Diagnostics.Metrics in dotnet-monitor's collector

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MetricSourceConfiguration.cs

@@ -13,39 +13,83 @@
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
+    [Flags]
+    public enum MetricType
+    {
+        EventCounter = 0x1,
+        Meter = 0x2,
+        All = 0xFF
+    }
+
+    public sealed class MetricEventPipeProvider
+    {
+        public string Provider { get; set; }
+
+        public float? IntervalSeconds { get; set; }
+
+        public MetricType Type { get; set; } = MetricType.All;
+    }
+
     public sealed class MetricSourceConfiguration : MonitoringSourceConfiguration
     {
         private readonly IList<EventPipeProvider> _eventPipeProviders;
+        public string SessionId { get; private set; }
 
-        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> customProviderNames)
+        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> eventCounterProviderNames)
+            : this(metricIntervalSeconds, CreateProviders(eventCounterProviderNames?.Any() == true ? eventCounterProviderNames : DefaultMetricProviders))
         {
-            RequestRundown = false;
-            if (customProviderNames == null)
-            {
-                throw new ArgumentNullException(nameof(customProviderNames));
-            }
-            MetricIntervalSeconds = metricIntervalSeconds.ToString(CultureInfo.InvariantCulture);
+        }
 
-            IEnumerable<string> providers = null;
-            if (customProviderNames.Any())
+        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<MetricEventPipeProvider> providers, int maxHistograms = 20, int maxTimeSeries = 1000)
+        {
+            if (providers == null)
             {
-                providers = customProviderNames;
+                throw new ArgumentNullException(nameof(providers));
             }
-            else
+
+            RequestRundown = false;
+
+            _eventPipeProviders = providers.Where(provider => provider.Type.HasFlag(MetricType.EventCounter))
+                .Select((MetricEventPipeProvider provider) => new EventPipeProvider(provider.Provider,
+                    EventLevel.Informational,
+                    (long)ClrTraceEventParser.Keywords.None,
+                    new Dictionary<string, string>()
+                    {
+                        {
+                            "EventCounterIntervalSec", (provider.IntervalSeconds ?? metricIntervalSeconds).ToString(CultureInfo.InvariantCulture)
+                        }
+                    })).ToList();
+
+            IEnumerable<MetricEventPipeProvider> meterProviders = providers.Where(provider => provider.Type.HasFlag(MetricType.Meter));
+
+            if (meterProviders.Any())
             {
-                providers = new[] { SystemRuntimeEventSourceName, MicrosoftAspNetCoreHostingEventSourceName, GrpcAspNetCoreServer };
-            }
+                const long TimeSeriesValues = 0x2;
+                string metrics = string.Join(',', meterProviders.Select(p => p.Provider));
+
+                SessionId = Guid.NewGuid().ToString();
 
-            _eventPipeProviders = providers.Select((string provider) => new EventPipeProvider(provider,
-               EventLevel.Informational,
-               (long)ClrTraceEventParser.Keywords.None,
-               new Dictionary<string, string>()
-               {
-                    { "EventCounterIntervalSec", MetricIntervalSeconds }
-               })).ToList();
+                EventPipeProvider metricsEventSourceProvider =
+                    new EventPipeProvider("System.Diagnostics.Metrics", EventLevel.Informational, TimeSeriesValues,
+                        new Dictionary<string, string>()
+                        {
+                            { "SessionId", SessionId },
+                            { "Metrics", metrics },
+                            { "RefreshInterval", metricIntervalSeconds.ToString(CultureInfo.InvariantCulture) },
+                            { "MaxTimeSeries", maxTimeSeries.ToString() },
+                            { "MaxHistograms", maxHistograms.ToString() }
+                        }
+                    );
+
+                _eventPipeProviders = _eventPipeProviders.Append(metricsEventSourceProvider).ToArray();
+            }
         }
 
-        private string MetricIntervalSeconds { get; }
+        private static IEnumerable<MetricEventPipeProvider> CreateProviders(IEnumerable<string> providers) =>
+            providers.Select(provider => new MetricEventPipeProvider {
+                Provider = provider,
+                Type = MetricType.EventCounter
+            });
 
         public override IList<EventPipeProvider> GetProviders() => _eventPipeProviders;
     }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterFilter.cs

@@ -38,12 +38,20 @@ public void AddFilter(string providerName, string[] counters)
 
         public IEnumerable<string> GetProviders() => _enabledCounters.Keys;
 
+        public int IntervalSeconds => _intervalMilliseconds / 1000;
+
         public bool IsIncluded(string providerName, string counterName, int intervalMilliseconds)
         {
             if (_intervalMilliseconds != intervalMilliseconds)
             {
                 return false;
             }
+
+            return IsIncluded(providerName, counterName);
+        }
+
+        public bool IsIncluded(string providerName, string counterName)
+        {
             if (_enabledCounters.Count == 0)
             {
                 return true;

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs

@@ -5,17 +5,15 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Linq;
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
+    /// <summary>
+    /// TODO This is currently a duplication of the src\Tools\dotnet-counters\CounterPayload.cs stack. The two will be unified in a separate change.
+    /// </summary>
     internal class CounterPayload : ICounterPayload
     {
-#if NETSTANDARD
-        private static readonly IReadOnlyDictionary<string, string> Empty = new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0));
-#else
-        private static readonly IReadOnlyDictionary<string, string> Empty = System.Collections.Immutable.ImmutableDictionary<string, string>.Empty;
-#endif
-
         public CounterPayload(DateTime timestamp,
             string provider,
             string name,
@@ -24,7 +22,7 @@ public CounterPayload(DateTime timestamp,
             double value,
             CounterType counterType,
             float interval,
-            Dictionary<string, string> metadata)
+            string metadata)
         {
             Timestamp = timestamp;
             Name = name;
@@ -34,14 +32,27 @@ public CounterPayload(DateTime timestamp,
             CounterType = counterType;
             Provider = provider;
             Interval = interval;
-            Metadata = metadata ?? Empty;
+            Metadata = metadata;
+            EventType = EventType.Gauge;
+        }
+
+        // Copied from dotnet-counters
+        public CounterPayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, DateTime timestamp, string type, EventType eventType)
+        {
+            Provider = providerName;
+            Name = name;
+            Metadata = metadata;
+            Value = value;
+            Timestamp = timestamp;
+            CounterType = (CounterType)Enum.Parse(typeof(CounterType), type);
+            EventType = eventType;
         }
 
         public string Namespace { get; }
 
         public string Name { get; }
 
-        public string DisplayName { get; }
+        public string DisplayName { get; protected set; }
 
         public string Unit { get; }
 
@@ -55,6 +66,81 @@ public CounterPayload(DateTime timestamp,
 
         public string Provider { get; }
 
-        public IReadOnlyDictionary<string, string> Metadata { get; }
+        public string Metadata { get; }
+
+        public EventType EventType { get; set; }
+    }
+
+    internal class GaugePayload : CounterPayload
+    {
+        public GaugePayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Metric", EventType.Gauge)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+        }
+    }
+
+    internal class CounterEndedPayload : CounterPayload
+    {
+        public CounterEndedPayload(string providerName, string name, string displayName, DateTime timestamp)
+            : base(providerName, name, displayName, string.Empty, null, 0.0, timestamp, "Metric", EventType.CounterEnded)
+        {
+
+        }
+    }
+
+    internal class RatePayload : CounterPayload
+    {
+        public RatePayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, double intervalSecs, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Rate", EventType.Rate)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            string unitsName = string.IsNullOrEmpty(displayUnits) ? "Count" : displayUnits;
+            string intervalName = intervalSecs.ToString() + " sec";
+            DisplayName = $"{counterName} ({unitsName} / {intervalName})";
+        }
+    }
+
+    internal class PercentilePayload : CounterPayload
+    {
+        public PercentilePayload(string providerName, string name, string displayName, string displayUnits, string metadata, IEnumerable<(double quantile, double value)> quantiles, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, 0.0, timestamp, "Metric", EventType.Histogram)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+            Quantiles = quantiles.Select(v => new Quantile(v.quantile, v.value)).ToArray();
+        }
+
+        public Quantile[] Quantiles { get; }
+    }
+
+    internal record struct Quantile(double Percentage, double Value);
+
+    internal class ErrorPayload : CounterPayload
+    {
+        public ErrorPayload(string errorMessage) : this(errorMessage, DateTime.UtcNow) 
+        {
+        }
+
+        public ErrorPayload(string errorMessage, DateTime timestamp) :
+            base(string.Empty, string.Empty, string.Empty, string.Empty, null, 0.0, timestamp, "Metric", EventType.Error)
+        {
+            ErrorMessage = errorMessage;
+        }
+
+        public string ErrorMessage { get; private set; }
+    }
+
+    internal enum EventType : int
+    {
+        Rate,
+        Gauge,
+        Histogram,
+        Error,
+        CounterEnded
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/EventCounterPipeline.cs → src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPipeline.cs

@@ -6,18 +6,20 @@
 using Microsoft.Diagnostics.Tracing;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
-    internal class EventCounterPipeline : EventSourcePipeline<EventPipeCounterPipelineSettings>
+    internal class CounterPipeline : EventSourcePipeline<CounterPipelineSettings>
     {
         private readonly IEnumerable<ICountersLogger> _loggers;
         private readonly CounterFilter _filter;
+        private string _sessionId;
 
-        public EventCounterPipeline(DiagnosticsClient client,
-            EventPipeCounterPipelineSettings settings,
+        public CounterPipeline(DiagnosticsClient client,
+            CounterPipelineSettings settings,
             IEnumerable<ICountersLogger> loggers) : base(client, settings)
         {
             _loggers = loggers ?? throw new ArgumentNullException(nameof(loggers));
@@ -38,18 +40,28 @@ public EventCounterPipeline(DiagnosticsClient client,
 
         protected override MonitoringSourceConfiguration CreateConfiguration()
         {
-            return new MetricSourceConfiguration(Settings.CounterIntervalSeconds, _filter.GetProviders());
+            var config = new MetricSourceConfiguration(Settings.CounterIntervalSeconds, Settings.CounterGroups.Select((EventPipeCounterGroup counterGroup) => new MetricEventPipeProvider
+                {
+                    Provider = counterGroup.ProviderName,
+                    IntervalSeconds = counterGroup.IntervalSeconds,
+                    Type = (MetricType)counterGroup.Type
+                }),
+                Settings.MaxHistograms, Settings.MaxTimeSeries);
+
+            _sessionId = config.SessionId;
+
+            return config;
         }
 
         protected override async Task OnEventSourceAvailable(EventPipeEventSource eventSource, Func<Task> stopSessionAsync, CancellationToken token)
         {
-            ExecuteCounterLoggerAction((metricLogger) => metricLogger.PipelineStarted());
+            await ExecuteCounterLoggerActionAsync((metricLogger) => metricLogger.PipelineStarted(token));
 
             eventSource.Dynamic.All += traceEvent =>
             {
                 try
                 {
-                    if (traceEvent.TryGetCounterPayload(_filter, out ICounterPayload counterPayload))
+                    if (traceEvent.TryGetCounterPayload(_filter, _sessionId, out ICounterPayload counterPayload))
                     {
                         ExecuteCounterLoggerAction((metricLogger) => metricLogger.Log(counterPayload));
                     }
@@ -67,7 +79,21 @@ protected override async Task OnEventSourceAvailable(EventPipeEventSource eventS
 
             await sourceCompletedTaskSource.Task;
 
-            ExecuteCounterLoggerAction((metricLogger) => metricLogger.PipelineStopped());
+            await ExecuteCounterLoggerActionAsync((metricLogger) => metricLogger.PipelineStopped(token));
+        }
+
+        private async Task ExecuteCounterLoggerActionAsync(Func<ICountersLogger, Task> action)
+        {
+            foreach (ICountersLogger logger in _loggers)
+            {
+                try
+                {
+                    await action(logger);
+                }
+                catch (ObjectDisposedException)
+                {
+                }
+            }
         }
 
         private void ExecuteCounterLoggerAction(Action<ICountersLogger> action)

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/EventPipeCounterPipelineSettings.cs → src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPipelineSettings.cs

@@ -8,18 +8,35 @@
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
-    internal class EventPipeCounterPipelineSettings : EventSourcePipelineSettings
+    internal class CounterPipelineSettings : EventSourcePipelineSettings
     {
         public EventPipeCounterGroup[] CounterGroups { get; set; }
 
         //Do not use TimeSpan here since we may need to synchronize this pipeline interval
         //with a different session and want to make sure the values are identical.
         public float CounterIntervalSeconds { get; set; }
+
+        public int MaxHistograms { get; set; }
+
+        public int MaxTimeSeries { get; set; }
+    }
+
+    [Flags]
+    internal enum CounterGroupType
+    {
+        EventCounter = 0x1,
+        Meter = 0x2,
+        All = 0xFF
     }
 
     internal class EventPipeCounterGroup
     {
         public string ProviderName { get; set; }
+
         public string[] CounterNames { get; set; }
+
+        public CounterGroupType Type { get; set; } = CounterGroupType.All;
+
+        public float? IntervalSeconds { get; set; }
     }
 }