Update Quality and Safety evaluators that use context to record the context used in the metrics they produce

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatConversationEvaluator.cs

@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -35,7 +34,7 @@ public abstract class ChatConversationEvaluator : IEvaluator
     protected virtual string? SystemPrompt => null;
 
     /// <inheritdoc/>
-    public async ValueTask<EvaluationResult> EvaluateAsync(
+    public virtual async ValueTask<EvaluationResult> EvaluateAsync(
         IEnumerable<ChatMessage> messages,
         ChatResponse modelResponse,
         ChatConfiguration? chatConfiguration = null,

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/EquivalenceEvaluator.cs

@@ -49,6 +49,28 @@ public sealed class EquivalenceEvaluator : SingleNumericMetricEvaluator
     /// <inheritdoc/>
     protected override bool IgnoresHistory => true;
 
+    /// <inheritdoc/>
+    public override async ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        EvaluationResult result =
+            await base.EvaluateAsync(
+                messages,
+                modelResponse,
+                chatConfiguration,
+                additionalContext,
+                cancellationToken).ConfigureAwait(false);
+
+        EquivalenceEvaluatorContext context = GetRelevantContext(additionalContext);
+        result.AddOrUpdateContextInAllMetrics("Ground Truth", context.GetContents());
+
+        return result;
+    }
+
     /// <inheritdoc/>
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
@@ -66,18 +88,8 @@ userRequest is not null
                 ? await RenderAsync(userRequest, cancellationToken).ConfigureAwait(false)
                 : string.Empty;
 
-        string groundTruth;
-
-        if (additionalContext?.OfType<EquivalenceEvaluatorContext>().FirstOrDefault()
-                is EquivalenceEvaluatorContext context)
-        {
-            groundTruth = context.GroundTruth;
-        }
-        else
-        {
-            throw new InvalidOperationException(
-                $"A value of type '{nameof(EquivalenceEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
-        }
+        EquivalenceEvaluatorContext context = GetRelevantContext(additionalContext);
+        string groundTruth = context.GroundTruth;
 
         string prompt =
             $$"""
@@ -149,4 +161,16 @@ alleviating stress and augmenting general mood.
 
         return prompt;
     }
+
+    private static EquivalenceEvaluatorContext GetRelevantContext(IEnumerable<EvaluationContext>? additionalContext)
+    {
+        if (additionalContext?.OfType<EquivalenceEvaluatorContext>().FirstOrDefault()
+                is EquivalenceEvaluatorContext context)
+        {
+            return context;
+        }
+
+        throw new InvalidOperationException(
+            $"A value of type '{nameof(EquivalenceEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
+    }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/GroundednessEvaluator.cs

@@ -49,6 +49,30 @@ public sealed class GroundednessEvaluator : SingleNumericMetricEvaluator
     /// <inheritdoc/>
     protected override bool IgnoresHistory => false;
 
+    /// <inheritdoc/>
+    public override async ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        EvaluationResult result =
+            await base.EvaluateAsync(
+                messages,
+                modelResponse,
+                chatConfiguration,
+                additionalContext,
+                cancellationToken).ConfigureAwait(false);
+
+        if (GetRelevantContext(additionalContext) is GroundednessEvaluatorContext context)
+        {
+            result.AddOrUpdateContextInAllMetrics("Grounding Context", context.GetContents());
+        }
+
+        return result;
+    }
+
     /// <inheritdoc/>
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
@@ -68,8 +92,7 @@ userRequest is not null
 
         var builder = new StringBuilder();
 
-        if (additionalContext?.OfType<GroundednessEvaluatorContext>().FirstOrDefault()
-                is GroundednessEvaluatorContext context)
+        if (GetRelevantContext(additionalContext) is GroundednessEvaluatorContext context)
         {
             _ = builder.Append(context.GroundingContext);
             _ = builder.AppendLine();
@@ -162,4 +185,15 @@ is not French.
 
         return prompt;
     }
+
+    private static GroundednessEvaluatorContext? GetRelevantContext(IEnumerable<EvaluationContext>? additionalContext)
+    {
+        if (additionalContext?.OfType<GroundednessEvaluatorContext>().FirstOrDefault()
+                is GroundednessEvaluatorContext context)
+        {
+            return context;
+        }
+
+        return null;
+    }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/GroundednessProEvaluator.cs

@@ -63,22 +63,30 @@ await EvaluateContentSafetyAsync(
                 contentSafetyServicePayloadFormat: ContentSafetyServicePayloadFormat.QuestionAnswer.ToString(),
                 cancellationToken: cancellationToken).ConfigureAwait(false);
 
+        GroundednessProEvaluatorContext context = GetRelevantContext(additionalContext);
+        result.AddOrUpdateContextInAllMetrics("Grounding Context", context.GetContents());
+
         return result;
     }
 
     /// <inheritdoc/>
     protected override IReadOnlyList<EvaluationContext>? FilterAdditionalContext(
         IEnumerable<EvaluationContext>? additionalContext)
+    {
+        GroundednessProEvaluatorContext context = GetRelevantContext(additionalContext);
+        return [context];
+    }
+
+    private static GroundednessProEvaluatorContext GetRelevantContext(
+        IEnumerable<EvaluationContext>? additionalContext)
     {
         if (additionalContext?.OfType<GroundednessProEvaluatorContext>().FirstOrDefault()
                 is GroundednessProEvaluatorContext context)
         {
-            return [context];
-        }
-        else
-        {
-            throw new InvalidOperationException(
-                $"A value of type '{nameof(GroundednessProEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
+            return context;
         }
+
+        throw new InvalidOperationException(
+            $"A value of type '{nameof(GroundednessProEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
     }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/UngroundedAttributesEvaluator.cs

@@ -67,22 +67,30 @@ await EvaluateContentSafetyAsync(
                 contentSafetyServicePayloadFormat: ContentSafetyServicePayloadFormat.QueryResponse.ToString(),
                 cancellationToken: cancellationToken).ConfigureAwait(false);
 
+        UngroundedAttributesEvaluatorContext context = GetRelevantContext(additionalContext);
+        result.AddOrUpdateContextInAllMetrics("Grounding Context", context.GetContents());
+
         return result;
     }
 
     /// <inheritdoc/>
     protected override IReadOnlyList<EvaluationContext>? FilterAdditionalContext(
         IEnumerable<EvaluationContext>? additionalContext)
+    {
+        UngroundedAttributesEvaluatorContext context = GetRelevantContext(additionalContext);
+        return [context];
+    }
+
+    private static UngroundedAttributesEvaluatorContext GetRelevantContext(
+        IEnumerable<EvaluationContext>? additionalContext)
     {
         if (additionalContext?.OfType<UngroundedAttributesEvaluatorContext>().FirstOrDefault()
                 is UngroundedAttributesEvaluatorContext context)
         {
-            return [context];
-        }
-        else
-        {
-            throw new InvalidOperationException(
-                $"A value of type '{nameof(UngroundedAttributesEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
+            return context;
         }
+
+        throw new InvalidOperationException(
+            $"A value of type '{nameof(UngroundedAttributesEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection.");
     }
 }