Remove the abstraction for token counting from the main evaluation API

Introduce a stand alone extension method (that is decoupled from the rest of the evaluation API) that can be used for token counting instead.

This change is being made because there is still some uncertainty around what a general purpose token counting abstraction (that supports all kinds of future models, and all kinds of input modalities) should look like at the moment. We do not want to bake in an API that only supports text based inputs for the models and use cases that are prevalent today, since it would be a breaking change to change this API once we release a stable version of the evaluation APIs.

We can always reintroduce the token counting support if and when there is more clarity on what a general purpose token counting abstraction should look like, or if and when a general purpose token counting abstraction is introduced in a lower layer (Microsoft.Extensions.AI) in the future.

In the meanwhile, the extension method, even though less than ideal / perfect, should still allow callers to trim down the evaluated conversation history in cases where a token limit needs to be enforced.

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatConversationEvaluator.cs

@@ -57,99 +57,6 @@ public virtual async ValueTask<EvaluationResult> EvaluateAsync(
 
         (ChatMessage? userRequest, List<ChatMessage> history) = GetUserRequestAndHistory(messages);
 
-        int inputTokenLimit = 0;
-        int ignoredMessagesCount = 0;
-
-        if (chatConfiguration.TokenCounter is not null)
-        {
-            IEvaluationTokenCounter tokenCounter = chatConfiguration.TokenCounter;
-            inputTokenLimit = tokenCounter.InputTokenLimit;
-            int tokenBudget = inputTokenLimit;
-
-            void OnTokenBudgetExceeded()
-            {
-                EvaluationDiagnostic tokenBudgetExceeded =
-                    EvaluationDiagnostic.Error(
-                        $"Evaluation failed because the specified limit of {inputTokenLimit} input tokens was exceeded.");
-
-                result.AddDiagnosticsToAllMetrics(tokenBudgetExceeded);
-            }
-
-            if (!string.IsNullOrWhiteSpace(SystemPrompt))
-            {
-                tokenBudget -= tokenCounter.CountTokens(SystemPrompt!);
-                if (tokenBudget < 0)
-                {
-                    OnTokenBudgetExceeded();
-                    return result;
-                }
-            }
-
-            string baseEvaluationPrompt =
-                await RenderEvaluationPromptAsync(
-                    userRequest,
-                    modelResponse,
-                    includedHistory: [],
-                    additionalContext,
-                    cancellationToken).ConfigureAwait(false);
-
-            tokenBudget -= tokenCounter.CountTokens(baseEvaluationPrompt);
-            if (tokenBudget < 0)
-            {
-                OnTokenBudgetExceeded();
-                return result;
-            }
-
-            if (history.Count > 0 && !IgnoresHistory)
-            {
-                if (history.Count == 1)
-                {
-                    (bool canRender, tokenBudget) =
-                        await CanRenderAsync(
-                            history[0],
-                            tokenBudget,
-                            chatConfiguration,
-                            cancellationToken).ConfigureAwait(false);
-
-                    if (!canRender)
-                    {
-                        ignoredMessagesCount = 1;
-                        history = [];
-                    }
-                }
-                else
-                {
-                    int totalMessagesCount = history.Count;
-                    int includedMessagesCount = 0;
-
-                    history.Reverse();
-
-                    foreach (ChatMessage message in history)
-                    {
-                        cancellationToken.ThrowIfCancellationRequested();
-
-                        (bool canRender, tokenBudget) =
-                            await CanRenderAsync(
-                                message,
-                                tokenBudget,
-                                chatConfiguration,
-                                cancellationToken).ConfigureAwait(false);
-
-                        if (!canRender)
-                        {
-                            ignoredMessagesCount = totalMessagesCount - includedMessagesCount;
-                            history.RemoveRange(index: includedMessagesCount, count: ignoredMessagesCount);
-                            break;
-                        }
-
-                        includedMessagesCount++;
-                    }
-
-                    history.Reverse();
-                }
-            }
-        }
-
         var evaluationMessages = new List<ChatMessage>();
         if (!string.IsNullOrWhiteSpace(SystemPrompt))
         {
@@ -172,84 +79,9 @@ await PerformEvaluationAsync(
             result,
             cancellationToken).ConfigureAwait(false);
 
-        if (inputTokenLimit > 0 && ignoredMessagesCount > 0)
-        {
-#pragma warning disable S103 // Lines should not be too long
-            result.AddDiagnosticsToAllMetrics(
-                EvaluationDiagnostic.Warning(
-                    $"The evaluation may be inconclusive because the oldest {ignoredMessagesCount} messages in the supplied conversation history were ignored in order to stay under the specified limit of {inputTokenLimit} input tokens."));
-#pragma warning restore S103
-        }
-
         return result;
     }
 
-    /// <summary>
-    /// Determines if there is sufficient <paramref name="tokenBudget"/> remaining to render the
-    /// supplied <paramref name="message"/> as part of the evaluation prompt that this <see cref="IEvaluator"/> uses.
-    /// </summary>
-    /// <param name="message">
-    /// A message that is part of the conversation history for the response being evaluated and that is to be rendered
-    /// as part of the evaluation prompt.
-    /// </param>
-    /// <param name="tokenBudget">
-    /// The number of tokens available for the rendering additional content as part of the evaluation prompt.
-    /// </param>
-    /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
-    /// </param>
-    /// <param name="cancellationToken">A <see cref="CancellationToken"/> that can cancel the operation.</param>
-    /// <returns>
-    /// A tuple containing a <see langword="bool"/> indicating whether there is sufficient
-    /// <paramref name="tokenBudget"/> remaining to render the supplied <paramref name="message"/> as part of the
-    /// evaluation prompt, and an <see langword="int"/> containing the remaining token budget that would be available
-    /// once this <paramref name="message"/> is rendered.
-    /// </returns>
-    protected virtual ValueTask<(bool canRender, int remainingTokenBudget)> CanRenderAsync(
-        ChatMessage message,
-        int tokenBudget,
-        ChatConfiguration chatConfiguration,
-        CancellationToken cancellationToken)
-    {
-        _ = Throw.IfNull(message);
-        _ = Throw.IfNull(chatConfiguration);
-
-        IEvaluationTokenCounter? tokenCounter = chatConfiguration.TokenCounter;
-        if (tokenCounter is null)
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget));
-        }
-
-        string? author = message.AuthorName;
-        string role = message.Role.Value;
-        string content = message.Text ?? string.Empty;
-
-        int tokenCount =
-            string.IsNullOrWhiteSpace(author)
-                ? tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens("] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n")
-                : tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(author!) +
-                    tokenCounter.CountTokens(" (") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens(")] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n");
-
-        if (tokenCount > tokenBudget)
-        {
-            return new ValueTask<(bool, int)>((false, tokenBudget));
-        }
-        else
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget - tokenCount));
-        }
-    }
-
     /// <summary>
     /// Renders the supplied <paramref name="response"/> to a string that can be included as part of the evaluation
     /// prompt that this <see cref="IEvaluator"/> uses.
@@ -351,8 +183,8 @@ protected abstract ValueTask<string> RenderEvaluationPromptAsync(
     /// <see cref="EvaluationMetric"/>s in the supplied <paramref name="result"/>.
     /// </summary>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that should be used if one or
+    /// more composed <see cref="IEvaluator"/>s use an AI model to perform evaluation.
     /// </param>
     /// <param name="evaluationMessages">
     /// The set of messages that are to be sent to the supplied <see cref="ChatConfiguration.ChatClient"/> to perform

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs

@@ -29,10 +29,9 @@ public static class AzureStorageReportingConfiguration
     /// survive in the cache before they are considered expired and evicted.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in the
-    /// returned <see cref="ReportingConfiguration"/>. Can be omitted if none of the included
-    /// <paramref name="evaluators"/> are AI-based.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
+    /// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="enableResponseCaching">
     /// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs

@@ -30,9 +30,8 @@ public sealed class ReportingConfiguration
     public IResultStore ResultStore { get; }
 
     /// <summary>
-    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <see cref="Evaluators"/> included in this
-    /// <see cref="ReportingConfiguration"/>.
+    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <see cref="Evaluators"/> included in this <see cref="ReportingConfiguration"/>.
     /// </summary>
     public ChatConfiguration? ChatConfiguration { get; }
 
@@ -103,10 +102,9 @@ public sealed class ReportingConfiguration
     /// The <see cref="IResultStore"/> that should be used to persist the <see cref="ScenarioRunResult"/>s.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in this
-    /// <see cref="ReportingConfiguration"/>. Can be omitted if none of the included <paramref name="evaluators"/> are
-    /// AI-based.
+    /// A <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <paramref name="evaluators"/> included in this <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="responseCacheProvider">
     /// The <see cref="IResponseCacheProvider"/> that should be used to cache AI responses. If omitted, AI responses
@@ -246,7 +244,7 @@ await ResponseCacheProvider.GetCacheAsync(
             }
 #pragma warning restore CA2000
 
-            chatConfiguration = new ChatConfiguration(chatClient, chatConfiguration.TokenCounter);
+            chatConfiguration = new ChatConfiguration(chatClient);
         }
 
         return new ScenarioRun(

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs

@@ -80,9 +80,9 @@ public sealed class ScenarioRun : IAsyncDisposable
     public string ExecutionName { get; }
 
     /// <summary>
-    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <see cref="IEvaluator"/>s that are invoked as
-    /// part of the evaluation of this <see cref="ScenarioRun"/>.
+    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <see cref="IEvaluator"/>s that are invoked as part of the evaluation of this
+    /// <see cref="ScenarioRun"/>.
     /// </summary>
     public ChatConfiguration? ChatConfiguration { get; }
 

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs

@@ -25,10 +25,9 @@ public static class DiskBasedReportingConfiguration
     /// The set of <see cref="IEvaluator"/>s that should be invoked to evaluate AI responses.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in the
-    /// returned <see cref="ReportingConfiguration"/>. Can be omitted if none of the included
-    /// <paramref name="evaluators"/> are AI-based.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
+    /// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="enableResponseCaching">
     /// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs

@@ -46,7 +46,7 @@ public static ChatConfiguration ToChatConfiguration(
                 originalChatClient: originalChatConfiguration?.ChatClient);
 #pragma warning restore CA2000
 
-        return new ChatConfiguration(newChatClient, originalChatConfiguration?.TokenCounter);
+        return new ChatConfiguration(newChatClient);
     }
 
     /// <summary>

src/Libraries/Microsoft.Extensions.AI.Evaluation/ChatConfiguration.cs

@@ -9,27 +9,13 @@
 namespace Microsoft.Extensions.AI.Evaluation;
 
 /// <summary>
-/// Specifies the <see cref="IChatClient"/> and the <see cref="IEvaluationTokenCounter"/> that should be used when
-/// evaluation is performed using an AI model.
+/// Specifies the <see cref="IChatClient"/> that should be used when evaluation is performed using an AI model.
 /// </summary>
 /// <param name="chatClient">An <see cref="IChatClient"/> that can be used to communicate with an AI model.</param>
-/// <param name="tokenCounter">
-/// An <see cref="IEvaluationTokenCounter"/> that can be used to counts tokens present in evaluation prompts, or
-/// <see langword="null"/> if the AI model / deployment being used does not impose an input token limit.
-/// </param>
-public sealed class ChatConfiguration(IChatClient chatClient, IEvaluationTokenCounter? tokenCounter = null)
+public sealed class ChatConfiguration(IChatClient chatClient)
 {
     /// <summary>
     /// Gets an <see cref="IChatClient"/> that can be used to communicate with an AI model.
     /// </summary>
     public IChatClient ChatClient { get; } = chatClient;
-
-    /// <summary>
-    /// Gets an <see cref="IEvaluationTokenCounter"/> that can be used to counts tokens present in evaluation prompts.
-    /// </summary>
-    /// <remarks>
-    /// <see cref="TokenCounter"/> can be set to <see langword="null"/> if the AI model / deployment being used does
-    /// not impose an input token limit.
-    /// </remarks>
-    public IEvaluationTokenCounter? TokenCounter { get; } = tokenCounter;
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation/ChatMessageExtensions.cs

@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Tokenizers;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI.Evaluation;
+
+/// <summary>
+/// Extension methods for <see cref="ChatMessage"/>.
+/// </summary>
+public static class ChatMessageExtensions
+{
+    /// <summary>
+    /// Uses the supplied <paramref name="tokenizer"/> to count the number of tokens present in the supplied
+    /// <paramref name="conversation"/>, and discards the oldest messages (from the beginning of the collection), if
+    /// necessary, to ensure that the remaining messages fit within the specified <paramref name="tokenBudget"/>.
+    /// </summary>
+    /// <param name="conversation">
+    /// A collection of <see cref="ChatMessage"/>s representing an LLM conversation history, with the oldest messages
+    /// appearing towards the beginning of the collection, and the newest ones appearing towards the end.
+    /// </param>
+    /// <param name="tokenizer">
+    /// The <see cref="Tokenizer"/> to be used to count the number of tokens present in each message in the supplied
+    /// <paramref name="conversation"/>.
+    /// </param>
+    /// <param name="tokenBudget">The overall budget for the number of tokens available.</param>
+    /// <returns>
+    /// A <see cref="Tuple{T1, T2}"/> that contains the collection of messages that were retained after trimming down
+    /// the supplied <paramref name="conversation"/> to fit within the specified <paramref name="tokenBudget"/>, as
+    /// well as an <see langword="int"/> count identifying the remaining number of tokens available after this trimming
+    /// operation.
+    /// </returns>
+    public static (IEnumerable<ChatMessage> trimmedConversation, int remainingTokenBudget) Trim(
+        this IEnumerable<ChatMessage> conversation,
+        Tokenizer tokenizer,
+        int tokenBudget)
+    {
+        _ = Throw.IfNull(conversation);
+        _ = Throw.IfNull(tokenizer);
+        _ = Throw.IfLessThan(tokenBudget, min: 0);
+
+        var trimmedConversation = new List<ChatMessage>();
+        int remainingTokenBudget = tokenBudget;
+
+        foreach (ChatMessage message in conversation.Reverse())
+        {
+            int tokenCount = tokenizer.CountTokens(message.Text);
+            int newTokenBudget = remainingTokenBudget - tokenCount;
+
+            if (newTokenBudget < 0)
+            {
+                break;
+            }
+            else
+            {
+                remainingTokenBudget = newTokenBudget;
+                trimmedConversation.Add(message);
+            }
+        }
+
+        return (trimmedConversation, remainingTokenBudget);
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation/CompositeEvaluator.cs

@@ -88,9 +88,8 @@ public CompositeEvaluator(IEnumerable<IEvaluator> evaluators)
     /// </param>
     /// <param name="modelResponse">The response that is to be evaluated.</param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that should be used if one or more composed <see cref="IEvaluator"/>s use
-    /// an AI model to perform evaluation.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that should be used if one or
+    /// more composed <see cref="IEvaluator"/>s use an AI model to perform evaluation.
     /// </param>
     /// <param name="additionalContext">
     /// Additional contextual information (beyond that which is available in <paramref name="messages"/>) that composed