Fix MetricsEventSource tests

Last week I added a change to the error handling behavior when MetricEventSource
is enabled by multiple listeners and didn't properly update the tests.
1. The major issue was timout failures which were being caused because there is
a bug in EventListener where it doesn't notify EventSources that the source has
been disabled when the listener is disposed. This in turn caused every tests after
the first to reject the new EventListener because the EventSource believed it was
still in use by the first EventListener.
(EventListener bug is tracked #56378)
2. A secondary issue is that I didn't update the test which was explicitly verifying
the behavior where the EventSource emits an error in response to having two
listeners and I had changed the product behavior there.

Fixes:
1. I worked around the EventListener bug by explicitly calling DisableEvents().
I also updated OnEventWritten to log the MultipleSessionsNotSupportedError
so that any future error that is similar has more obvious diagnostic logging.
2. I updated the test with new expectation that the 1st listener continues normal
operation and the 2nd listener is the one that gets rejected.

I also did a little refactoring and added a 2nd tests of multiple listeners to confirm
it works if the first one is disabled before the 2nd one starts.

Author: noahfalk

src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs

@@ -376,7 +376,7 @@ public void EventSourcePublishesMissingDataPoints()
 
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))]
         [OuterLoop("Slow and has lots of console spew")]
-        public void EventSourcePublishesEndEventsOnNewListener()
+        public void EventSourceRejectsNewListener()
         {
             using Meter meter = new Meter("TestMeter7");
             Counter<int> c = meter.CreateCounter<int>("counter1", "hat", "Fooz!!");
@@ -395,11 +395,13 @@ public void EventSourcePublishesEndEventsOnNewListener()
                 listener.WaitForCollectionStop(s_waitForEventTimeout, 2);
                 c.Add(12);
                 h.Record(26);
-                listener.WaitForCollectionStop(s_waitForEventTimeout, 3);
 
-                // some alternate listener starts listening
+                // some alternate listener attempts to listen in the middle
                 using MetricsEventListener listener2 = new MetricsEventListener(_output, MetricsEventListener.TimeSeriesValues, IntervalSecs, "ADifferentMeter");
-                listener.WaitForEndInstrumentReporting(s_waitForEventTimeout, 4);
+                listener2.WaitForMultipleSessionsNotSupportedError(s_waitForEventTimeout);
+
+
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 3);
                 events = listener.Events.ToArray();
             }
 
@@ -409,7 +411,6 @@ public void EventSourcePublishesEndEventsOnNewListener()
             AssertGaugeEventsPresent(events, meter.Name, og.Name, "", og.Unit, "9", "18", "27");
             AssertHistogramEventsPresent(events, meter.Name, h.Name, "", h.Unit, "0.5=19;0.95=19;0.99=19", "0.5=26;0.95=26;0.99=26");
             AssertCollectStartStopEventsPresent(events, IntervalSecs, 3);
-            AssertEndInstrumentReportingEventsPresent(events, c, oc, og, h);
         }
 
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))]
@@ -646,6 +647,63 @@ public void EventSourceHandlesObservableCallbackException()
             AssertCollectStartStopEventsPresent(events, IntervalSecs, 3);
         }
 
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))]
+        [OuterLoop("Slow and has lots of console spew")]
+        public void EventSourceWorksWithSequentialListeners()
+        {
+            using Meter meter = new Meter("TestMeter16");
+            Counter<int> c = meter.CreateCounter<int>("counter1");
+            int counterState = 3;
+            ObservableCounter<int> oc = meter.CreateObservableCounter<int>("observableCounter1", () => { counterState += 7; return counterState; });
+            int gaugeState = 0;
+            ObservableGauge<int> og = meter.CreateObservableGauge<int>("observableGauge1", () => { gaugeState += 9; return gaugeState; });
+            Histogram<int> h = meter.CreateHistogram<int>("histogram1");
+
+            EventWrittenEventArgs[] events;
+            using (MetricsEventListener listener = new MetricsEventListener(_output, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter16"))
+            {
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 1);
+                c.Add(5);
+                h.Record(19);
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 2);
+                c.Add(12);
+                h.Record(26);
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 3);
+                events = listener.Events.ToArray();
+            }
+
+            AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h);
+            AssertInitialEnumerationCompleteEventPresent(events);
+            AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", "5", "12");
+            AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", "", "7");
+            AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "9", "18");
+            AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", "0.5=19;0.95=19;0.99=19", "0.5=26;0.95=26;0.99=26");
+            AssertCollectStartStopEventsPresent(events, IntervalSecs, 3);
+
+            // Now create a new listener and do everything a 2nd time. Because the listener above has been disposed the source should be
+            // free to accept a new connection.
+            events = null;
+            using (MetricsEventListener listener = new MetricsEventListener(_output, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter16"))
+            {
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 1);
+                c.Add(5);
+                h.Record(19);
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 2);
+                c.Add(12);
+                h.Record(26);
+                listener.WaitForCollectionStop(s_waitForEventTimeout, 3);
+                events = listener.Events.ToArray();
+            }
+
+            AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h);
+            AssertInitialEnumerationCompleteEventPresent(events);
+            AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", "5", "12");
+            AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", "", "7");
+            AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "36", "45");
+            AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", "0.5=19;0.95=19;0.99=19", "0.5=26;0.95=26;0.99=26");
+            AssertCollectStartStopEventsPresent(events, IntervalSecs, 3);
+        }
+
         private void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments)
         {
             var beginReportEvents = events.Where(e => e.EventName == "BeginInstrumentReporting").Select(e =>
@@ -961,10 +1019,20 @@ protected override void OnEventSourceCreated(EventSource eventSource)
             }
         }
 
+        public override void Dispose()
+        {
+            if (_source != null)
+            {
+                // workaround for https://github.com/dotnet/runtime/issues/56378
+                DisableEvents(_source);
+            }
+            base.Dispose();
+        }
+
         protected override void OnEventWritten(EventWrittenEventArgs eventData)
         {
             string sessionId = eventData.Payload[0].ToString();
-            if(sessionId != "" && sessionId != _sessionId)
+            if (eventData.EventName != "MultipleSessionsNotSupportedError" && sessionId != "" && sessionId != _sessionId)
             {
                 return;
             }
@@ -988,68 +1056,36 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData)
             _autoResetEvent.Set();
         }
 
-        public void WaitForCollectionStop(TimeSpan timeout, int numStops)
-        {
-            DateTime startTime = DateTime.Now;
-            DateTime stopTime = startTime + timeout;
-            int initialStopCount = GetCountCollectionStops();
-            while (true)
-            {
-                if (GetCountCollectionStops() >= numStops)
-                {
-                    return;
-                }
-                TimeSpan remainingTime = stopTime - DateTime.Now;
-                if (remainingTime.TotalMilliseconds < 0 || !_autoResetEvent.WaitOne(remainingTime))
-                {
-                    int currentStopCount = GetCountCollectionStops();
-                    throw new TimeoutException("Timed out waiting for a StopCollection event. " +
-                        $"StartTime={startTime} stopTime={stopTime} initialStopCount={initialStopCount} currentStopCount={currentStopCount} targetStopCount={numStops}");
-                }
-            }
-        }
+        public void WaitForCollectionStop(TimeSpan timeout, int numEvents) => WaitForEvent(timeout, numEvents, "CollectionStop");
+
+        public void WaitForEndInstrumentReporting(TimeSpan timeout, int numEvents) => WaitForEvent(timeout, numEvents, "EndInstrumentReporting");
+
+        public void WaitForEnumerationComplete(TimeSpan timeout) => WaitForEvent(timeout, 1, "InitialInstrumentEnumerationComplete");
 
-        public void WaitForEndInstrumentReporting(TimeSpan timeout, int numEvents)
+        public void WaitForMultipleSessionsNotSupportedError(TimeSpan timeout) => WaitForEvent(timeout, 1, "MultipleSessionsNotSupportedError");
+
+        void WaitForEvent(TimeSpan timeout, int numEvents, string eventName)
         {
             DateTime startTime = DateTime.Now;
             DateTime stopTime = startTime + timeout;
-            int initialEventCount = GetCountEndInstrumentReporting();
+            int initialEventCount = GetCountEvents(eventName);
             while (true)
             {
-                if (GetCountEndInstrumentReporting() >= numEvents)
+                if (GetCountEvents(eventName) >= numEvents)
                 {
                     return;
                 }
                 TimeSpan remainingTime = stopTime - DateTime.Now;
                 if (remainingTime.TotalMilliseconds < 0 || !_autoResetEvent.WaitOne(remainingTime))
                 {
-                    int currentEventCount = GetCountEndInstrumentReporting();
-                    throw new TimeoutException("Timed out waiting for a EndInstrumentReporting event. " +
+                    int currentEventCount = GetCountEvents(eventName);
+                    throw new TimeoutException($"Timed out waiting for a {eventName} event. " +
                         $"StartTime={startTime} stopTime={stopTime} initialEventCount={initialEventCount} currentEventCount={currentEventCount} targetEventCount={numEvents}");
                 }
             }
         }
 
-        public void WaitForEnumerationComplete(TimeSpan timeout)
-        {
-            DateTime startTime = DateTime.Now;
-            DateTime stopTime = startTime + timeout;
-            int initialEventCount = GetCountEnumerationComplete();
-            while (true)
-            {
-                if (GetCountEnumerationComplete() >= 1)
-                {
-                    return;
-                }
-                TimeSpan remainingTime = stopTime - DateTime.Now;
-                if (remainingTime.TotalMilliseconds < 0 || !_autoResetEvent.WaitOne(remainingTime))
-                {
-                    int currentEventCount = GetCountEnumerationComplete();
-                    throw new TimeoutException("Timed out waiting for a EndInstrumentReporting event. " +
-                        $"StartTime={startTime} stopTime={stopTime} initialEventCount={initialEventCount} currentEventCount={currentEventCount}");
-                }
-            }
-        }
+
 
         private void AssertOnError()
         {
@@ -1064,29 +1100,12 @@ private void AssertOnError()
             }
         }
 
-        private int GetCountCollectionStops()
+        private int GetCountEvents(string eventName)
         {
             lock (this)
             {
                 AssertOnError();
-                return Events.Where(e => e.EventName == "CollectionStop").Count();
-            }
-        }
-
-        private int GetCountEndInstrumentReporting()
-        {
-            lock (this)
-            {
-                AssertOnError();
-                return Events.Where(e => e.EventName == "EndInstrumentReporting").Count();
-            }
-        }
-
-        private int GetCountEnumerationComplete()
-        {
-            lock (this)
-            {
-                return Events.Where(e => e.EventName == "InitialInstrumentEnumerationComplete").Count();
+                return Events.Where(e => e.EventName == eventName).Count();
             }
         }
     }