feat(api): add message image content (#834)

Author: stainless-bot

.stats.yml

@@ -1,2 +1,2 @@
 configured_endpoints: 64
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-2e14236d4015bf3b956290ea8b656224a0c7b206a356c6af2a7ae43fdbceb04c.yml
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-084b8f68408c6b689a55200a78bcf233769bfcd8e999d9fadaeb399152b05bcd.yml

api.md

@@ -349,14 +349,20 @@ Types:
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileContentBlock</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileDeltaBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURL</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLContentBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLDelta</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLDeltaBlock</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">Message</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageContent</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageContentDelta</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">MessageContentPartParam</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDeleted</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDeltaEvent</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">Text</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextContentBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">TextContentBlockParam</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextDeltaBlock</a></code>
 

src/lib/AssistantStream.ts

@@ -7,6 +7,7 @@ import {
   ImageFile,
   TextDelta,
   Messages,
+  MessageContent,
 } from 'openai/resources/beta/threads/messages';
 import * as Core from 'openai/core';
 import { RequestOptions } from 'openai/core';
@@ -87,7 +88,7 @@ export class AssistantStream
   #messageSnapshot: Message | undefined;
   #finalRun: Run | undefined;
   #currentContentIndex: number | undefined;
-  #currentContent: TextContentBlock | ImageFileContentBlock | undefined;
+  #currentContent: MessageContent | undefined;
   #currentToolCallIndex: number | undefined;
   #currentToolCall: ToolCall | undefined;
 
@@ -624,10 +625,8 @@ export class AssistantStream
                 currentContent,
               );
             } else {
-              snapshot.content[contentElement.index] = contentElement as
-                | TextContentBlock
-                | ImageFileContentBlock;
-              //This is a new element
+              snapshot.content[contentElement.index] = contentElement as MessageContent;
+              // This is a new element
               newContent.push(contentElement);
             }
           }
@@ -650,7 +649,7 @@ export class AssistantStream
 
   #accumulateContent(
     contentElement: MessageContentDelta,
-    currentContent: TextContentBlock | ImageFileContentBlock | undefined,
+    currentContent: MessageContent | undefined,
   ): TextContentBlock | ImageFileContentBlock {
     return AssistantStream.accumulateDelta(currentContent as unknown as Record<any, any>, contentElement) as
       | TextContentBlock

src/resources/beta/threads/index.ts

@@ -11,14 +11,20 @@ export {
   ImageFileContentBlock,
   ImageFileDelta,
   ImageFileDeltaBlock,
+  ImageURL,
+  ImageURLContentBlock,
+  ImageURLDelta,
+  ImageURLDeltaBlock,
   Message,
   MessageContent,
   MessageContentDelta,
+  MessageContentPartParam,
   MessageDeleted,
   MessageDelta,
   MessageDeltaEvent,
   Text,
   TextContentBlock,
+  TextContentBlockParam,
   TextDelta,
   TextDeltaBlock,
   MessageCreateParams,

src/resources/beta/threads/messages.ts

@@ -249,9 +249,16 @@ export namespace FilePathDeltaAnnotation {
 export interface ImageFile {
   /**
    * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
-   * in the message content.
+   * in the message content. Set `purpose="vision"` when uploading the File if you
+   * need to later display the file content.
    */
   file_id: string;
+
+  /**
+   * Specifies the detail level of the image if specified by the user. `low` uses
+   * fewer tokens, you can opt in to high resolution using `high`.
+   */
+  detail?: 'auto' | 'low' | 'high';
 }
 
 /**
@@ -268,9 +275,16 @@ export interface ImageFileContentBlock {
 }
 
 export interface ImageFileDelta {
+  /**
+   * Specifies the detail level of the image if specified by the user. `low` uses
+   * fewer tokens, you can opt in to high resolution using `high`.
+   */
+  detail?: 'auto' | 'low' | 'high';
+
   /**
    * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
-   * in the message content.
+   * in the message content. Set `purpose="vision"` when uploading the File if you
+   * need to later display the file content.
    */
   file_id?: string;
 }
@@ -293,6 +307,63 @@ export interface ImageFileDeltaBlock {
   image_file?: ImageFileDelta;
 }
 
+export interface ImageURL {
+  /**
+   * The external URL of the image, must be a supported image types: jpeg, jpg, png,
+   * gif, webp.
+   */
+  url: string;
+
+  /**
+   * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+   * to high resolution using `high`. Default value is `auto`
+   */
+  detail?: 'auto' | 'low' | 'high';
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLContentBlock {
+  image_url: ImageURL;
+
+  /**
+   * The type of the content part.
+   */
+  type: 'image_url';
+}
+
+export interface ImageURLDelta {
+  /**
+   * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+   * to high resolution using `high`.
+   */
+  detail?: 'auto' | 'low' | 'high';
+
+  /**
+   * The URL of the image, must be a supported image types: jpeg, jpg, png, gif,
+   * webp.
+   */
+  url?: string;
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLDeltaBlock {
+  /**
+   * The index of the content part in the message.
+   */
+  index: number;
+
+  /**
+   * Always `image_url`.
+   */
+  type: 'image_url';
+
+  image_url?: ImageURLDelta;
+}
+
 /**
  * Represents a message within a
  * [thread](https://platform.openai.com/docs/api-reference/threads).
@@ -406,13 +477,19 @@ export namespace Message {
  * References an image [File](https://platform.openai.com/docs/api-reference/files)
  * in the content of a message.
  */
-export type MessageContent = ImageFileContentBlock | TextContentBlock;
+export type MessageContent = ImageFileContentBlock | ImageURLContentBlock | TextContentBlock;
+
+/**
+ * References an image [File](https://platform.openai.com/docs/api-reference/files)
+ * in the content of a message.
+ */
+export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock | ImageURLDeltaBlock;
 
 /**
  * References an image [File](https://platform.openai.com/docs/api-reference/files)
  * in the content of a message.
  */
-export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock;
+export type MessageContentPartParam = ImageFileContentBlock | ImageURLContentBlock | TextContentBlockParam;
 
 export interface MessageDeleted {
   id: string;
@@ -479,6 +556,21 @@ export interface TextContentBlock {
   type: 'text';
 }
 
+/**
+ * The text content that is part of a message.
+ */
+export interface TextContentBlockParam {
+  /**
+   * Text content to be sent to the model
+   */
+  text: string;
+
+  /**
+   * Always `text`.
+   */
+  type: 'text';
+}
+
 export interface TextDelta {
   annotations?: Array<AnnotationDelta>;
 
@@ -507,9 +599,9 @@ export interface TextDeltaBlock {
 
 export interface MessageCreateParams {
   /**
-   * The content of the message.
+   * The text contents of the message.
    */
-  content: string;
+  content: string | Array<MessageContentPartParam>;
 
   /**
    * The role of the entity that is creating the message. Allowed values include:
@@ -591,14 +683,20 @@ export namespace Messages {
   export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
   export import ImageFileDelta = MessagesAPI.ImageFileDelta;
   export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+  export import ImageURL = MessagesAPI.ImageURL;
+  export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+  export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+  export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
   export import Message = MessagesAPI.Message;
   export import MessageContent = MessagesAPI.MessageContent;
   export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+  export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
   export import MessageDeleted = MessagesAPI.MessageDeleted;
   export import MessageDelta = MessagesAPI.MessageDelta;
   export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
   export import Text = MessagesAPI.Text;
   export import TextContentBlock = MessagesAPI.TextContentBlock;
+  export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
   export import TextDelta = MessagesAPI.TextDelta;
   export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
   export import MessagesPage = MessagesAPI.MessagesPage;

src/resources/beta/threads/runs/runs.ts

@@ -9,6 +9,7 @@ import { sleep } from 'openai/core';
 import { RunSubmitToolOutputsParamsStream } from 'openai/lib/AssistantStream';
 import * as RunsAPI from 'openai/resources/beta/threads/runs/runs';
 import * as AssistantsAPI from 'openai/resources/beta/assistants';
+import * as MessagesAPI from 'openai/resources/beta/threads/messages';
 import * as ThreadsAPI from 'openai/resources/beta/threads/threads';
 import * as StepsAPI from 'openai/resources/beta/threads/runs/steps';
 import { CursorPage, type CursorPageParams } from 'openai/pagination';
@@ -747,9 +748,9 @@ export interface RunCreateParamsBase {
 export namespace RunCreateParams {
   export interface AdditionalMessage {
     /**
-     * The content of the message.
+     * The text contents of the message.
      */
-    content: string;
+    content: string | Array<MessagesAPI.MessageContentPartParam>;
 
     /**
      * The role of the entity that is creating the message. Allowed values include:
@@ -999,9 +1000,9 @@ export interface RunCreateAndPollParams {
 export namespace RunCreateAndPollParams {
   export interface AdditionalMessage {
     /**
-     * The content of the message.
+     * The text contents of the message.
      */
-    content: string;
+    content: string | Array<MessagesAPI.MessageContentPartParam>;
 
     /**
      * The role of the entity that is creating the message. Allowed values include:
@@ -1204,9 +1205,9 @@ export interface RunCreateAndStreamParams {
 export namespace RunCreateAndStreamParams {
   export interface AdditionalMessage {
     /**
-     * The content of the message.
+     * The text contents of the message.
      */
-    content: string;
+    content: string | Array<MessagesAPI.MessageContentPartParam>;
 
     /**
      * The role of the entity that is creating the message. Allowed values include:
@@ -1409,9 +1410,9 @@ export interface RunStreamParams {
 export namespace RunStreamParams {
   export interface AdditionalMessage {
     /**
-     * The content of the message.
+     * The text contents of the message.
      */
-    content: string;
+    content: string | Array<MessagesAPI.MessageContentPartParam>;
 
     /**
      * The role of the entity that is creating the message. Allowed values include:

src/resources/beta/threads/threads.ts

@@ -284,9 +284,9 @@ export interface ThreadCreateParams {
 export namespace ThreadCreateParams {
   export interface Message {
     /**
-     * The content of the message.
+     * The text contents of the message.
      */
-    content: string;
+    content: string | Array<MessagesAPI.MessageContentPartParam>;
 
     /**
      * The role of the entity that is creating the message. Allowed values include:
@@ -623,9 +623,9 @@ export namespace ThreadCreateAndRunParams {
   export namespace Thread {
     export interface Message {
       /**
-       * The content of the message.
+       * The text contents of the message.
        */
-      content: string;
+      content: string | Array<MessagesAPI.MessageContentPartParam>;
 
       /**
        * The role of the entity that is creating the message. Allowed values include:
@@ -973,9 +973,9 @@ export namespace ThreadCreateAndRunPollParams {
   export namespace Thread {
     export interface Message {
       /**
-       * The content of the message.
+       * The text contents of the message.
        */
-      content: string;
+      content: string | Array<MessagesAPI.MessageContentPartParam>;
 
       /**
        * The role of the entity that is creating the message. Allowed values include:
@@ -1302,9 +1302,9 @@ export namespace ThreadCreateAndRunStreamParams {
   export namespace Thread {
     export interface Message {
       /**
-       * The content of the message.
+       * The text contents of the message.
        */
-      content: string;
+      content: string | Array<MessagesAPI.MessageContentPartParam>;
 
       /**
        * The role of the entity that is creating the message. Allowed values include:
@@ -1503,14 +1503,20 @@ export namespace Threads {
   export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
   export import ImageFileDelta = MessagesAPI.ImageFileDelta;
   export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+  export import ImageURL = MessagesAPI.ImageURL;
+  export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+  export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+  export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
   export import Message = MessagesAPI.Message;
   export import MessageContent = MessagesAPI.MessageContent;
   export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+  export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
   export import MessageDeleted = MessagesAPI.MessageDeleted;
   export import MessageDelta = MessagesAPI.MessageDelta;
   export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
   export import Text = MessagesAPI.Text;
   export import TextContentBlock = MessagesAPI.TextContentBlock;
+  export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
   export import TextDelta = MessagesAPI.TextDelta;
   export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
   export import MessagesPage = MessagesAPI.MessagesPage;