Skip to content

Commit f6bd93c

Browse files
Shyam Namboodiripadjoperezr
authored andcommitted
Merged PR 49951: [AI Evaluation] Cherry pick commits for 9.5 preview release
#### AI description (iteration 1) #### PR Classification This PR cherry picks commits for the 9.5 preview release and introduces API improvements, interface renaming, and caching logic updates. #### PR Summary The PR streamlines caching in AI Evaluation by removing legacy CacheOptions and updating related interfaces while also enhancing JSON converters and chat extension methods. Key changes include: - **`src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs`**: Removed CacheOptions logic, added a configurable TTL parameter, and refactored cache methods. - **Interface Renaming**: Renamed `IResponseCacheProvider` to `IEvaluationResponseCacheProvider` and `IResultStore` to `IEvaluationResultStore` across core code and associated tests. - **`src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs`**: Modified the `AddTurnDetails` extension to accept multiple chat turns. - **JSON Serialization**: Updated converters in `AzureStorageJsonUtilities.cs` and `JsonUtilities.cs` to use newer types (e.g., `CamelCaseEnumConverter`, `TimeSpanConverter`, and added `EvaluationContextConverter`). - **`src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs`**: Reordered and renamed enum values to better reflect the intended rating semantics. <!-- GitOpsUserAgent=GitOps.Apps.Server.pullrequestcopilot -->
2 parents a3d136b + 2d442e1 commit f6bd93c

File tree

41 files changed

+289
-451
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+289
-451
lines changed

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ internal sealed class CleanCacheCommand(ILogger logger)
1818
{
1919
internal async Task<int> InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default)
2020
{
21-
IResponseCacheProvider cacheProvider;
21+
IEvaluationResponseCacheProvider cacheProvider;
2222

2323
if (storageRootDir is not null)
2424
{

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ internal async Task<int> InvokeAsync(
2323
int lastN,
2424
CancellationToken cancellationToken = default)
2525
{
26-
IResultStore resultStore;
26+
IEvaluationResultStore resultStore;
2727

2828
if (storageRootDir is not null)
2929
{

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ internal async Task<int> InvokeAsync(
2828
Format format,
2929
CancellationToken cancellationToken = default)
3030
{
31-
IResultStore resultStore;
31+
IEvaluationResultStore resultStore;
3232

3333
if (storageRootDir is not null)
3434
{

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
<PropertyGroup>
44
<Description>A command line dotnet tool for generating reports and managing evaluation data.</Description>
55
<OutputType>Exe</OutputType>
6-
<!-- Building only one TFM due to bug: https://github.com/dotnet/sdk/issues/47696
7-
Once this is fixed, we can go back to building multiple. -->
8-
<TargetFrameworks>$(MinimumSupportedTfmForPackaging)</TargetFrameworks>
6+
<TargetFrameworks>$(NetCoreTargetFrameworks)</TargetFrameworks>
97
<RootNamespace>Microsoft.Extensions.AI.Evaluation.Console</RootNamespace>
108
<!-- EA0000: Use source generated logging methods for improved performance. -->
119
<NoWarn>$(NoWarn);EA0000</NoWarn>
@@ -22,6 +20,15 @@
2220
<MinMutationScore>0</MinMutationScore>
2321
</PropertyGroup>
2422

23+
<!--
24+
Disable parallel build to work around https://github.com/dotnet/sdk/issues/47696. The problem has been fixed in
25+
https://github.com/dotnet/sdk/pull/47788, however the fix has not yet been back ported to the dotnet 9 SDK. We can
26+
remove this workaround once the fix is available in the dotnet 9 SDK.
27+
-->
28+
<PropertyGroup>
29+
<BuildInParallel>false</BuildInParallel>
30+
</PropertyGroup>
31+
2532
<ItemGroup>
2633
<PackageReference Include="Azure.Identity" />
2734
<PackageReference Include="Azure.Storage.Files.DataLake" />

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs

Lines changed: 0 additions & 11 deletions
This file was deleted.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Diagnostics.CodeAnalysis;
45
using System.Text.Encodings.Web;
56
using System.Text.Json;
67
using System.Text.Json.Serialization;
@@ -11,7 +12,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization;
1112

1213
internal static partial class AzureStorageJsonUtilities
1314
{
14-
[System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")]
15+
[SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")]
1516
internal static class Default
1617
{
1718
private static JsonSerializerOptions? _options;
@@ -24,6 +25,7 @@ internal static class Compact
2425
{
2526
private static JsonSerializerOptions? _options;
2627
internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false);
28+
internal static JsonTypeInfo<CacheEntry> CacheEntryTypeInfo => Options.GetTypeInfo<CacheEntry>();
2729
internal static JsonTypeInfo<ScenarioRunResult> ScenarioRunResultTypeInfo => Options.GetTypeInfo<ScenarioRunResult>();
2830
}
2931

@@ -45,14 +47,14 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden
4547
[JsonSerializable(typeof(CacheEntry))]
4648
[JsonSourceGenerationOptions(
4749
Converters = [
48-
typeof(AzureStorageCamelCaseEnumConverter<EvaluationDiagnosticSeverity>),
49-
typeof(AzureStorageCamelCaseEnumConverter<EvaluationRating>),
50-
typeof(AzureStorageTimeSpanConverter)
50+
typeof(CamelCaseEnumConverter<EvaluationDiagnosticSeverity>),
51+
typeof(CamelCaseEnumConverter<EvaluationRating>),
52+
typeof(TimeSpanConverter),
53+
typeof(EvaluationContextConverter)
5154
],
5255
WriteIndented = true,
5356
IgnoreReadOnlyProperties = false,
5457
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
5558
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
5659
private sealed partial class JsonContext : JsonSerializerContext;
57-
5860
}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs

Lines changed: 0 additions & 17 deletions
This file was deleted.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@
1717
<MinMutationScore>0</MinMutationScore>
1818
</PropertyGroup>
1919

20+
<ItemGroup>
21+
<Compile Include="..\Microsoft.Extensions.AI.Evaluation.Reporting\CSharp\JsonSerialization\CamelCaseEnumConverter.cs" Link="JsonSerialization\CamelCaseEnumConverter.cs" />
22+
<Compile Include="..\Microsoft.Extensions.AI.Evaluation.Reporting\CSharp\JsonSerialization\EvaluationContextConverter.cs" Link="JsonSerialization\EvaluationContextConverter.cs" />
23+
<Compile Include="..\Microsoft.Extensions.AI.Evaluation.Reporting\CSharp\JsonSerialization\TimeSpanConverter.cs" Link="JsonSerialization\TimeSpanConverter.cs" />
24+
</ItemGroup>
25+
2026
<ItemGroup>
2127
<PackageReference Include="Azure.Storage.Files.DataLake" />
2228
</ItemGroup>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@ public static class AzureStorageReportingConfiguration
2424
/// <param name="evaluators">
2525
/// The set of <see cref="IEvaluator"/>s that should be invoked to evaluate AI responses.
2626
/// </param>
27-
/// <param name="timeToLiveForCacheEntries">
28-
/// An optional <see cref="TimeSpan"/> that specifies the maximum amount of time that cached AI responses should
29-
/// survive in the cache before they are considered expired and evicted.
30-
/// </param>
3127
/// <param name="chatConfiguration">
3228
/// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
3329
/// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
@@ -36,6 +32,10 @@ public static class AzureStorageReportingConfiguration
3632
/// <param name="enableResponseCaching">
3733
/// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.
3834
/// </param>
35+
/// <param name="timeToLiveForCacheEntries">
36+
/// An optional <see cref="TimeSpan"/> that specifies the maximum amount of time that cached AI responses should
37+
/// survive in the cache before they are considered expired and evicted.
38+
/// </param>
3939
/// <param name="cachingKeys">
4040
/// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI
4141
/// responses. See <see cref="ReportingConfiguration.CachingKeys"/> for more information about this concept.
@@ -63,21 +63,21 @@ public static class AzureStorageReportingConfiguration
6363
public static ReportingConfiguration Create(
6464
DataLakeDirectoryClient client,
6565
IEnumerable<IEvaluator> evaluators,
66-
TimeSpan? timeToLiveForCacheEntries = null,
6766
ChatConfiguration? chatConfiguration = null,
6867
bool enableResponseCaching = true,
68+
TimeSpan? timeToLiveForCacheEntries = null,
6969
IEnumerable<string>? cachingKeys = null,
7070
string executionName = Defaults.DefaultExecutionName,
7171
Func<EvaluationMetric, EvaluationMetricInterpretation?>? evaluationMetricInterpreter = null,
7272
IEnumerable<string>? tags = null)
7373
#pragma warning restore S107
7474
{
75-
IResponseCacheProvider? responseCacheProvider =
75+
IEvaluationResponseCacheProvider? responseCacheProvider =
7676
chatConfiguration is not null && enableResponseCaching
7777
? new AzureStorageResponseCacheProvider(client, timeToLiveForCacheEntries)
7878
: null;
7979

80-
IResultStore resultStore = new AzureStorageResultStore(client);
80+
IEvaluationResultStore resultStore = new AzureStorageResultStore(client);
8181

8282
return new ReportingConfiguration(
8383
evaluators,

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ internal sealed partial class AzureStorageResponseCache(
4040
private const string EntryAndContentsFilesNotFound = "Cache entry file {0} and contents file {1} were not found.";
4141

4242
private readonly string _iterationPath = $"cache/{scenarioName}/{iterationName}";
43+
private readonly Func<DateTime> _provideDateTime = provideDateTime;
4344
private readonly TimeSpan _timeToLiveForCacheEntries =
4445
timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries;
45-
private readonly Func<DateTime> _provideDateTime = provideDateTime;
4646

4747
public byte[]? Get(string key)
4848
{

0 commit comments

Comments
 (0)