Image generation tool

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs

@@ -22,7 +22,7 @@ namespace Microsoft.Extensions.AI;
 /// </para>
 /// <para>
 /// The relationship between <see cref="ChatResponse"/> and <see cref="ChatResponseUpdate"/> is
-/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync"/> and
+/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync(IAsyncEnumerable{ChatResponseUpdate}, System.Threading.CancellationToken)"/> and
 /// <see cref="ChatResponse.ToChatResponseUpdates"/>, which enable bidirectional conversions
 /// between the two. Note, however, that the provided conversions may be lossy, for example if multiple
 /// updates all have different <see cref="RawRepresentation"/> objects whereas there's only one slot for

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Provides options for configuring how <see cref="ChatResponseUpdate"/> instances are coalesced
+/// when converting them to <see cref="ChatMessage"/> instances.
+/// </summary>
+[Experimental("EXTAI0001")]
+public class ChatResponseUpdateCoalescingOptions
+{
+    /// <summary>
+    /// Gets or sets a value indicating whether to replace existing <see cref="DataContent"/> items
+    /// when a new <see cref="DataContent"/> item with the same <see cref="DataContent.Name"/> is encountered.
+    /// </summary>
+    /// <value>
+    /// <see langword="true"/> to replace existing <see cref="DataContent"/> items with the same name;
+    /// <see langword="false"/> to keep all <see cref="DataContent"/> items. The default is <see langword="false"/>.
+    /// </value>
+    /// <remarks>
+    /// When this property is <see langword="true"/>, if a <see cref="DataContent"/> item is being added
+    /// and there's already a <see cref="DataContent"/> item in the content list with the same
+    /// <see cref="DataContent.Name"/>, the existing item will be replaced with the new one.
+    /// This is useful for scenarios where updated data should override previous data with the same identifier.
+    /// </remarks>
+    public bool ReplaceDataContentWithSameName { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents a hosted tool that can be specified to an AI service to enable it to perform image generation.</summary>
+/// <remarks>
+/// This tool does not itself implement image generation. It is a marker that can be used to inform a service
+/// that the service is allowed to perform image generation if the service is capable of doing so.
+/// </remarks>
+[Experimental("MEAI001")]
+public class ImageGenerationTool : AITool
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ImageGenerationTool"/> class with the specified options.
+    /// </summary>
+    /// <param name="options">The options to configure the image generation request. If <paramref name="options"/> is <see langword="null"/>, default options will be used.</param>
+    public ImageGenerationTool(ImageGenerationOptions? options = null)
+        : base()
+    {
+        AdditionalProperties = new AdditionalPropertiesDictionary(new Dictionary<string, object?>
+        {
+            [nameof(ImageGenerationOptions)] = options
+        });
+    }
+
+    /// <inheritdoc />
+    public override IReadOnlyDictionary<string, object?> AdditionalProperties { get; }
+}

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs

@@ -14,7 +14,9 @@ namespace Microsoft.Extensions.AI;
     WriteIndented = true)]
 [JsonSerializable(typeof(OpenAIClientExtensions.ToolJson))]
 [JsonSerializable(typeof(IDictionary<string, object?>))]
+[JsonSerializable(typeof(IDictionary<string, string?>))]
 [JsonSerializable(typeof(string[]))]
 [JsonSerializable(typeof(IEnumerable<string>))]
 [JsonSerializable(typeof(JsonElement))]
+[JsonSerializable(typeof(int))]
 internal sealed partial class OpenAIJsonContext : JsonSerializerContext;

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs

@@ -4,14 +4,17 @@
 using System;
 using System.ClientModel.Primitives;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Text.Json;
+using System.Text.Json.Serialization.Metadata;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Shared.Diagnostics;
+using OpenAI.Images;
 using OpenAI.Responses;
 
 #pragma warning disable S907 // "goto" statement should not be used
@@ -163,7 +166,15 @@ internal static IEnumerable<ChatMessage> ToChatMessages(IEnumerable<ResponseItem
                     break;
 
                 default:
-                    message.Contents.Add(new() { RawRepresentation = outputItem });
+                    if (outputItem.GetType().Name == "InternalImageGenToolCallItemResource")
+                    {
+                        message.Contents.Add(GetContentFromImageGen(outputItem));
+                    }
+                    else
+                    {
+                        message.Contents.Add(new() { RawRepresentation = outputItem });
+                    }
+
                     break;
             }
         }
@@ -174,6 +185,118 @@ internal static IEnumerable<ChatMessage> ToChatMessages(IEnumerable<ResponseItem
         }
     }
 
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(ResponseItem))]
+    private static DataContent GetContentFromImageGen(ResponseItem outputItem)
+    {
+        const BindingFlags InternalBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance;
+        var imageGenResultType = Type.GetType("OpenAI.Responses.InternalImageGenToolCallItemResource, OpenAI");
+        if (imageGenResultType == null)
+        {
+            throw new InvalidOperationException("Unable to determine the type of the image generation result.");
+        }
+
+        var imageGenStatus = imageGenResultType.GetProperty("Status", InternalBindingFlags)?.GetValue(outputItem)?.ToString();
+        var imageGenResult = imageGenResultType.GetProperty("Result", InternalBindingFlags)?.GetValue(outputItem) as string;
+
+        IDictionary<string, BinaryData>? additionalRawData = imageGenResultType
+            .GetProperty("SerializedAdditionalRawData", InternalBindingFlags)
+            ?.GetValue(outputItem) as IDictionary<string, BinaryData>;
+
+        // Properties
+        //   background
+        //   output_format
+        //   quality
+        //   revised_prompt
+        //   size
+
+        string outputFormat = getStringProperty("output_format") ?? "png";
+
+        var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty);
+
+        return new DataContent(resultBytes, $"image/{outputFormat}")
+        {
+            RawRepresentation = outputItem,
+            AdditionalProperties = new()
+            {
+                ["background"] = getStringProperty("background"),
+                ["output_format"] = outputFormat,
+                ["quality"] = getStringProperty("quality"),
+                ["revised_prompt"] = getStringProperty("revised_prompt"),
+                ["size"] = getStringProperty("size"),
+                ["status"] = imageGenStatus,
+            }
+        };
+
+        string? getStringProperty(string name)
+        {
+            if (additionalRawData?.TryGetValue(name, out var outputFormat) == true)
+            {
+                var stringJsonTypeInfo = (JsonTypeInfo<string>)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string));
+                return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo);
+            }
+
+            return null;
+        }
+    }
+
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(ResponseItem))]
+    private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseUpdate update)
+    {
+        const BindingFlags InternalBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance;
+        var partialImageEventType = Type.GetType("OpenAI.Responses.InternalResponseImageGenCallPartialImageEvent, OpenAI");
+        if (partialImageEventType == null)
+        {
+            throw new InvalidOperationException("Unable to determine the type of the image generation result.");
+        }
+
+        var imageGenResult = partialImageEventType.GetProperty("PartialImageB64", InternalBindingFlags)?.GetValue(update) as string;
+        var imageGenItemId = partialImageEventType.GetProperty("ItemId", InternalBindingFlags)?.GetValue(update) as string;
+        var imageGenOutputIndex = partialImageEventType.GetProperty("OutputIndex", InternalBindingFlags)?.GetValue(update) as int?;
+        var imageGenPartialImageIndex = partialImageEventType.GetProperty("PartialImageIndex", InternalBindingFlags)?.GetValue(update) as int?;
+
+        IDictionary<string, BinaryData>? additionalRawData = partialImageEventType
+            .GetProperty("SerializedAdditionalRawData", InternalBindingFlags)
+            ?.GetValue(update) as IDictionary<string, BinaryData>;
+
+        // Properties
+        //   background
+        //   output_format
+        //   quality
+        //   revised_prompt
+        //   size
+
+        string outputFormat = getStringProperty("output_format") ?? "png";
+
+        var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty);
+
+        return new DataContent(resultBytes, $"image/{outputFormat}")
+        {
+            RawRepresentation = update,
+            AdditionalProperties = new()
+            {
+                ["ItemId"] = imageGenItemId,
+                ["OutputIndex"] = imageGenOutputIndex,
+                ["PartialImageIndex"] = imageGenPartialImageIndex,
+                ["background"] = getStringProperty("background"),
+                ["output_format"] = outputFormat,
+                ["quality"] = getStringProperty("quality"),
+                ["revised_prompt"] = getStringProperty("revised_prompt"),
+                ["size"] = getStringProperty("size"),
+            }
+        };
+
+        string? getStringProperty(string name)
+        {
+            if (additionalRawData?.TryGetValue(name, out var outputFormat) == true)
+            {
+                var stringJsonTypeInfo = (JsonTypeInfo<string>)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string));
+                return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo);
+            }
+
+            return null;
+        }
+    }
+
     /// <inheritdoc />
     public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
         IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
@@ -324,7 +447,16 @@ ChatResponseUpdate CreateUpdate(AIContent? content = null) =>
                     break;
 
                 default:
-                    yield return CreateUpdate();
+
+                    if (streamingUpdate.GetType().Name == "InternalResponseImageGenCallPartialImageEvent")
+                    {
+                        yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingUpdate));
+                    }
+                    else
+                    {
+                        yield return CreateUpdate();
+                    }
+
                     break;
             }
         }
@@ -349,6 +481,59 @@ internal static ResponseTool ToResponseTool(AIFunction aiFunction, ChatOptions?
             strict ?? false);
     }
 
+    internal static ResponseTool ToImageResponseTool(ImageGenerationTool imageGenerationTool, ChatOptions? options = null)
+    {
+        ImageGenerationOptions? imageGenerationOptions = null;
+        if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj))
+        {
+            imageGenerationOptions = optionsObj as ImageGenerationOptions;
+        }
+        else if (options?.AdditionalProperties?.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj2) ?? false)
+        {
+            imageGenerationOptions = optionsObj2 as ImageGenerationOptions;
+        }
+
+        var toolOptions = imageGenerationOptions?.RawRepresentationFactory?.Invoke(null!) as Dictionary<string, object> ?? new();
+        toolOptions["type"] = "image_generation";
+
+        // Size: Image dimensions (e.g., 1024x1024, 1024x1536)
+        if (imageGenerationOptions?.ImageSize is not null && !toolOptions.ContainsKey("size"))
+        {
+            // Use a custom type to ensure the size is formatted correctly.
+            // This is a workaround for OpenAI's specific size format requirements.
+            toolOptions["size"] = new GeneratedImageSize(
+                imageGenerationOptions.ImageSize.Value.Width,
+                imageGenerationOptions.ImageSize.Value.Height).ToString();
+        }
+
+        // Format: File output format
+        if (imageGenerationOptions?.MediaType is not null && !toolOptions.ContainsKey("format"))
+        {
+            toolOptions["output_format"] = imageGenerationOptions.MediaType switch
+            {
+                "image/png" => GeneratedImageFileFormat.Png.ToString(),
+                "image/jpeg" => GeneratedImageFileFormat.Jpeg.ToString(),
+                "image/webp" => GeneratedImageFileFormat.Webp.ToString(),
+                _ => string.Empty,
+            };
+        }
+
+        // unexposed properties, string unless noted
+        // background: transparent, opaque, auto
+        // input_fidelity: effort model exerts to match input (high, low)
+        // input_image_mask: optional image mask for inpainting.  Object with property file_id string or image_url data string.
+        // model: Model ID to use for image generation
+        // moderation: Moderation level (auto, low)
+        // output_compression: (int) Compression level (0-100%) for JPEG and WebP formats
+        // partial_images: (int) Number of partial images to return (0-3)
+        // quality: Rendering quality (e.g. low, medium, high)
+
+        // Can't create the tool, but we can deserialize it from Json
+        BinaryData? toolOptionsData = BinaryData.FromBytes(
+            JsonSerializer.SerializeToUtf8Bytes(toolOptions, OpenAIJsonContext.Default.IDictionaryStringObject));
+        return ModelReaderWriter.Read<ResponseTool>(toolOptionsData, ModelReaderWriterOptions.Json)!;
+    }
+
     /// <summary>Creates a <see cref="ChatRole"/> from a <see cref="MessageRole"/>.</summary>
     private static ChatRole ToChatRole(MessageRole? role) =>
         role switch
@@ -403,6 +588,10 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt
                         result.Tools.Add(ToResponseTool(aiFunction, options));
                         break;
 
+                    case ImageGenerationTool imageGenerationTool:
+                        result.Tools.Add(ToImageResponseTool(imageGenerationTool, options));
+                        break;
+
                     case HostedWebSearchTool webSearchTool:
                         WebSearchUserLocation? location = null;
                         if (webSearchTool.AdditionalProperties.TryGetValue(nameof(WebSearchUserLocation), out object? objLocation))