Add output content moderation, using Azure AI Content Safety

Directory.Build.props

@@ -2,7 +2,7 @@
 <Project>
     <PropertyGroup>
         <!-- Central version prefix - applies to all nuget packages. -->
-        <Version>0.79.0</Version>
+        <Version>0.80.0</Version>
 
         <!-- C# lang version, https://learn.microsoft.com/dotnet/csharp/whats-new -->
         <LangVersion>12</LangVersion>

Directory.Packages.props

@@ -5,10 +5,11 @@
   <ItemGroup>
     <PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.2.1" />
     <PackageVersion Include="AWSSDK.S3" Version="3.7.404.6" />
+    <PackageVersion Include="Azure.AI.ContentSafety" Version="1.0.0" />
     <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0" />
     <PackageVersion Include="Azure.Core" Version="1.44.1" />
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.2" />
-    <PackageVersion Include="Azure.Identity" Version="1.12.1" />
+    <PackageVersion Include="Azure.Identity" Version="1.13.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.22.2" />
     <PackageVersion Include="Azure.Storage.Queues" Version="12.20.1" />
@@ -37,8 +38,8 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
     <PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.4.0" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.79.241014.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.79.241014.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.79.241014.2" />
+    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.79.241014.2" />
     <PackageVersion Include="MongoDB.Driver.GridFS" Version="2.29.0" />
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
@@ -49,7 +50,7 @@
     <PackageVersion Include="RabbitMQ.Client" Version="6.8.1" />
     <PackageVersion Include="NRedisStack" Version="0.13.0" />
     <PackageVersion Include="ReadLine" Version="2.0.1" />
-    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.8.1" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.9.0" />
     <PackageVersion Include="System.IO.Packaging" Version="8.0.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.1" />

KernelMemory.sln

@@ -329,6 +329,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Onnx", "extensions\ONNX\Onn
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RabbitMQ.TestApplication", "extensions\RabbitMQ\RabbitMQ.TestApplication\RabbitMQ.TestApplication.csproj", "{82670921-FDCD-4672-84BD-4353F5AC24A0}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureAIContentSafety", "extensions\AzureAIContentSafety\AzureAIContentSafety\AzureAIContentSafety.csproj", "{58E65B3F-EFF0-401A-AC76-A49835AE0220}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -607,6 +609,10 @@ Global
 		{82670921-FDCD-4672-84BD-4353F5AC24A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{82670921-FDCD-4672-84BD-4353F5AC24A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{82670921-FDCD-4672-84BD-4353F5AC24A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{58E65B3F-EFF0-401A-AC76-A49835AE0220}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{58E65B3F-EFF0-401A-AC76-A49835AE0220}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{58E65B3F-EFF0-401A-AC76-A49835AE0220}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{58E65B3F-EFF0-401A-AC76-A49835AE0220}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -704,6 +710,7 @@ Global
 		{7BBD348E-CDD9-4462-B8C9-47613C5EC682} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
 		{345DEF9B-6EE1-49DF-B46A-25E38CE9B151} = {155DA079-E267-49AF-973A-D1D44681970F}
 		{82670921-FDCD-4672-84BD-4353F5AC24A0} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
+		{58E65B3F-EFF0-401A-AC76-A49835AE0220} = {155DA079-E267-49AF-973A-D1D44681970F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

examples/210-KM-without-builder/Program.cs

@@ -19,6 +19,7 @@
 using Microsoft.KernelMemory.MemoryStorage;
 using Microsoft.KernelMemory.Pipeline;
 using Microsoft.KernelMemory.Prompts;
+using Microsoft.KernelMemory.Safety.AzureAIContentSafety;
 using Microsoft.KernelMemory.Search;
 
 /// <summary>
@@ -43,6 +44,7 @@ public static async Task Main()
         var azureOpenAIEmbeddingConfig = new AzureOpenAIConfig();
         var azureOpenAITextConfig = new AzureOpenAIConfig();
         var azureAIDocIntelConfig = new AzureAIDocIntelConfig();
+        var azureAIContentSafetyModerationConfig = new AzureAIContentSafetyConfig();
         var textPartitioningOptions = new TextPartitioningOptions();
         var msExcelDecoderConfig = new MsExcelDecoderConfig();
         var msPowerPointDecoderConfig = new MsPowerPointDecoderConfig();
@@ -63,7 +65,8 @@ public static async Task Main()
             .BindSection("KernelMemory:Services:AzureAISearch", azureAISearchConfig)
             .BindSection("KernelMemory:Services:AzureOpenAIEmbedding", azureOpenAIEmbeddingConfig)
             .BindSection("KernelMemory:Services:AzureOpenAIText", azureOpenAITextConfig)
-            .BindSection("KernelMemory:Services:AzureAIDocIntel", azureAIDocIntelConfig);
+            .BindSection("KernelMemory:Services:AzureAIDocIntel", azureAIDocIntelConfig)
+            .BindSection("KernelMemory:Services:AzureAIContentSafety", azureAIContentSafetyModerationConfig);
         // =================================================================
 
         // Logger
@@ -79,6 +82,7 @@ public static async Task Main()
         var embeddingGenerator = new AzureOpenAITextEmbeddingGenerator(azureOpenAIEmbeddingConfig, tokenizer, loggerFactory, embeddingGeneratorHttpClient);
         var textGeneratorHttpClient = new HttpClient();
         var textGenerator = new AzureOpenAITextGenerator(azureOpenAITextConfig, tokenizer, loggerFactory, textGeneratorHttpClient);
+        var contentModeration = new AzureAIContentSafetyModeration(azureAIContentSafetyModerationConfig, loggerFactory);
 
         // Storage
         var documentStorage = new AzureBlobsStorage(azureBlobsConfig, mimeTypeDetection, loggerFactory);
@@ -116,7 +120,7 @@ public static async Task Main()
         orchestrator.AddHandler(new DeleteDocumentHandler("private_delete_document", documentStorage, memoryDbs, loggerFactory));
 
         // Create memory instance
-        var searchClient = new SearchClient(memoryDb, textGenerator, searchClientConfig, promptProvider, loggerFactory);
+        var searchClient = new SearchClient(memoryDb, textGenerator, searchClientConfig, promptProvider, contentModeration, loggerFactory);
         var memory = new MemoryServerless(orchestrator, searchClient, kernelMemoryConfig);
 
         // End-to-end test

examples/210-KM-without-builder/appsettings.json

@@ -111,6 +111,9 @@
     "TextGeneratorType": "",
     // Name of the index to use when none is specified
     "DefaultIndexName": "default",
+    // Which service to use for content moderation. Optional.
+    // Currently supported: AzureAIContentSafety
+    "ContentModerationType": "AzureAIContentSafety",
     // Data ingestion pipelines configuration.
     "DataIngestion": {
       // - InProcess: in process .NET orchestrator, synchronous/no queues
@@ -217,6 +220,9 @@
         "StopSequences": []
         // Modify the likelihood of specified tokens appearing in the completion.
         //"TokenSelectionBiases": { }
+        // Whether to check is the generated answers are safe.
+        // A content moderation service must be present in the system.
+        "UseModerationService": true
       }
     },
     "Services": {

extensions/AzureAIContentSafety/AzureAIContentSafety/AzureAIContentSafety.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <RollForward>LatestMajor</RollForward>
+        <AssemblyName>Microsoft.KernelMemory.Safety.AzureAIContentSafety</AssemblyName>
+        <RootNamespace>Microsoft.KernelMemory.Safety.AzureAIContentSafety</RootNamespace>
+        <NoWarn>$(NoWarn);KMEXP02;CA1024;CA1724;</NoWarn>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\..\service\Abstractions\Abstractions.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Azure.AI.ContentSafety" />
+        <PackageReference Include="Azure.Core" />
+        <PackageReference Include="Azure.Identity" />
+    </ItemGroup>
+
+    <PropertyGroup>
+        <IsPackable>true</IsPackable>
+        <PackageId>Microsoft.KernelMemory.Safety.AzureAIContentSafety</PackageId>
+        <Product>Azure AI Content Safety for Kernel Memory</Product>
+        <Description>Add Azure AI Content Safety to Kernel Memory to moderate user content.</Description>
+        <PackageTags>Moderation, Safety, Memory, RAG, Kernel Memory, Semantic Memory, Episodic Memory, Declarative Memory, AI, Artificial Intelligence, Embeddings, Vector DB, Vector Search, Memory DB, ETL</PackageTags>
+        <DocumentationFile>bin/$(Configuration)/$(TargetFramework)/$(AssemblyName).xml</DocumentationFile>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <None Include="..\README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
+    </ItemGroup>
+
+</Project>

extensions/AzureAIContentSafety/AzureAIContentSafety/AzureAIContentSafetyConfig.cs

@@ -0,0 +1,63 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+using Azure.Core;
+
+namespace Microsoft.KernelMemory.Safety.AzureAIContentSafety;
+
+public class AzureAIContentSafetyConfig
+{
+    private TokenCredential? _tokenCredential;
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public enum AuthTypes
+    {
+        Unknown = -1,
+        AzureIdentity,
+        APIKey,
+        ManualTokenCredential,
+    }
+
+    public AuthTypes Auth { get; set; } = AuthTypes.Unknown;
+    public string Endpoint { get; set; } = string.Empty;
+    public string APIKey { get; set; } = string.Empty;
+    public double GlobalSafetyThreshold { get; set; } = 0.0;
+    public List<string> IgnoredWords { get; set; } = new();
+
+    public void SetCredential(TokenCredential credential)
+    {
+        this.Auth = AuthTypes.ManualTokenCredential;
+        this._tokenCredential = credential;
+    }
+
+    public TokenCredential GetTokenCredential()
+    {
+        return this._tokenCredential
+               ?? throw new ConfigurationException($"Azure AI Search: {nameof(this._tokenCredential)} not defined");
+    }
+
+    public void Validate()
+    {
+        if (this.Auth == AuthTypes.Unknown)
+        {
+            throw new ConfigurationException($"Azure AI Content Safety: {nameof(this.Auth)} (authentication type) is not defined");
+        }
+
+        if (this.Auth == AuthTypes.APIKey && string.IsNullOrWhiteSpace(this.APIKey))
+        {
+            throw new ConfigurationException($"Azure AI Content Safety: {nameof(this.APIKey)} is empty");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.Endpoint))
+        {
+            throw new ConfigurationException($"Azure AI Content Safety: {nameof(this.Endpoint)} is empty");
+        }
+
+        if (!this.Endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new ConfigurationException($"Azure AI Content Safety: {nameof(this.Endpoint)} must start with https://");
+        }
+    }
+}

extensions/AzureAIContentSafety/AzureAIContentSafety/AzureAIContentSafetyModeration.cs

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.AI.ContentSafety;
+using Azure.Identity;
+using Microsoft.Extensions.Logging;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.Diagnostics;
+using Microsoft.KernelMemory.DocumentStorage;
+
+namespace Microsoft.KernelMemory.Safety.AzureAIContentSafety;
+
+[Experimental("KMEXP05")]
+public class AzureAIContentSafetyModeration : IContentModeration
+{
+    private const string Replacement = "****";
+
+    private readonly ContentSafetyClient _client;
+    private readonly ILogger<AzureAIContentSafetyModeration> _log;
+    private readonly double _globalSafetyThreshold;
+    private readonly List<string> _ignoredWords;
+
+    public AzureAIContentSafetyModeration(
+        AzureAIContentSafetyConfig config,
+        ILoggerFactory? loggerFactory = null)
+    {
+        config.Validate();
+        this._log = (loggerFactory ?? DefaultLogger.Factory).CreateLogger<AzureAIContentSafetyModeration>();
+        this._globalSafetyThreshold = config.GlobalSafetyThreshold;
+        this._ignoredWords = config.IgnoredWords;
+
+        switch (config.Auth)
+        {
+            case AzureAIContentSafetyConfig.AuthTypes.AzureIdentity:
+                this._client = new ContentSafetyClient(new Uri(config.Endpoint), new DefaultAzureCredential());
+                break;
+
+            case AzureAIContentSafetyConfig.AuthTypes.APIKey:
+                this._client = new ContentSafetyClient(new Uri(config.Endpoint), new AzureKeyCredential(config.APIKey));
+                break;
+
+            case AzureAIContentSafetyConfig.AuthTypes.ManualTokenCredential:
+                this._client = new ContentSafetyClient(new Uri(config.Endpoint), config.GetTokenCredential());
+                break;
+
+            default:
+                this._log.LogCritical("Azure AI Search authentication type '{0}' undefined or not supported", config.Auth);
+                throw new DocumentStorageException($"Azure AI Search authentication type '{config.Auth}' undefined or not supported");
+        }
+    }
+
+    /// <inheritdoc/>
+    public Task<bool> IsSafeAsync(string? text, CancellationToken cancellationToken)
+    {
+        return this.IsSafeAsync(text, this._globalSafetyThreshold, cancellationToken);
+    }
+
+    /// <inheritdoc/>
+    public async Task<bool> IsSafeAsync(string? text, double threshold, CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrWhiteSpace(text)) { return true; }
+
+        text = this.RemoveIgnoredWords(text);
+
+        Response<AnalyzeTextResult>? result = await this._client.AnalyzeTextAsync(text, cancellationToken).ConfigureAwait(false);
+
+        bool isSafe = result.HasValue && result.Value.CategoriesAnalysis.All(x => x.Severity <= threshold);
+
+        if (!isSafe)
+        {
+            IEnumerable<string> report = result.HasValue ? result.Value.CategoriesAnalysis.Select(x => $"{x.Category}: {x.Severity}") : Array.Empty<string>();
+            this._log.LogWarning("Unsafe content detected, report: {0}", string.Join("; ", report));
+            this._log.LogSensitive("Unsafe content: {0}", text);
+        }
+
+        return isSafe;
+    }
+
+    private string RemoveIgnoredWords(string text)
+    {
+        foreach (var word in this._ignoredWords)
+        {
+            text = ReplaceWholeWordIgnoreCase(text, word, Replacement);
+        }
+
+        return text;
+    }
+
+    private static string ReplaceWholeWordIgnoreCase(string text, string oldValue, string newValue)
+    {
+        // \b ensures word boundaries, Regex.Escape escapes any special characters in oldValue
+        return Regex.Replace(text, $@"\b{Regex.Escape(oldValue)}\b", newValue, RegexOptions.IgnoreCase);
+    }
+}

extensions/AzureAIContentSafety/AzureAIContentSafety/DependencyInjection.cs

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.Safety.AzureAIContentSafety;
+
+#pragma warning disable IDE0130 // reduce number of "using" statements
+// ReSharper disable once CheckNamespace - reduce number of "using" statements
+namespace Microsoft.KernelMemory;
+
+/// <summary>
+/// Kernel Memory builder extensions
+/// </summary>
+[Experimental("KMEXP05")]
+public static partial class KernelMemoryBuilderExtensions
+{
+    public static IKernelMemoryBuilder WithAzureAIContentSafetyModeration(this IKernelMemoryBuilder builder, AzureAIContentSafetyConfig config)
+    {
+        builder.Services.AddAzureAIContentSafetyModeration(config);
+        return builder;
+    }
+
+    public static IKernelMemoryBuilder WithAzureAIContentSafetyModeration(this IKernelMemoryBuilder builder, string endpoint, string apiKey)
+    {
+        builder.Services.AddAzureAIContentSafetyModeration(endpoint, apiKey);
+        return builder;
+    }
+}
+
+/// <summary>
+/// .NET IServiceCollection dependency injection extensions.
+/// </summary>
+[Experimental("KMEXP05")]
+public static partial class DependencyInjection
+{
+    public static IServiceCollection AddAzureAIContentSafetyModeration(this IServiceCollection services, AzureAIContentSafetyConfig config)
+    {
+        config.Validate();
+        return services
+            .AddSingleton<AzureAIContentSafetyConfig>(config)
+            .AddSingleton<IContentModeration, AzureAIContentSafetyModeration>();
+    }
+
+    public static IServiceCollection AddAzureAIContentSafetyModeration(this IServiceCollection services, string endpoint, string apiKey)
+    {
+        var config = new AzureAIContentSafetyConfig { Endpoint = endpoint, APIKey = apiKey, Auth = AzureAIContentSafetyConfig.AuthTypes.APIKey };
+        config.Validate();
+        return services.AddAzureAIContentSafetyModeration(config);
+    }
+}

extensions/AzureAIContentSafety/README.md

@@ -0,0 +1,7 @@
+# Kernel Memory with Azure AI Content Safety
+
+[![Nuget package](https://img.shields.io/nuget/v/Microsoft.KernelMemory.Safety.AzureAIContentSafety)](https://www.nuget.org/packages/Microsoft.KernelMemory.Safety.AzureAIContentSafety/)
+[![Discord](https://img.shields.io/discord/1063152441819942922?label=Discord&logo=discord&logoColor=white&color=d82679)](https://aka.ms/KMdiscord)
+
+This project contains the [Azure AI Content Safety](https://azure.microsoft.com/products/ai-services/ai-content-safety)
+adapter allowing to use the Azure service in Kernel Memory to moderate content.