feat: add screen sharing support to realtime API

common/changes/@coze/realtime-api/feat-realtime-api_2025-01-02-11-41.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/realtime-api",
+      "comment": "add screen sharing support to realtime API",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@coze/realtime-api",
+  "email": "[email protected]"
+}

common/changes/@coze/realtime-api/feat-realtime-api_2025-01-02-12-21.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/realtime-api",
+      "comment": "add screen sharing support to realtime API",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@coze/realtime-api",
+  "email": "[email protected]"
+}

common/changes/@coze/realtime-api/feat-realtime-api_2025-01-02-12-49.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/realtime-api",
+      "comment": "add screen sharing support to realtime API",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@coze/realtime-api",
+  "email": "[email protected]"
+}

common/changes/@coze/realtime-api/feat-realtime-api_2025-01-02-13-03.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/realtime-api",
+      "comment": "add screen sharing support to realtime API",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@coze/realtime-api",
+  "email": "[email protected]"
+}

examples/realtime-console/src/pages/main/header.tsx

@@ -15,6 +15,7 @@ import { ChatEventType } from '@coze/api';
 
 import { LocalManager, LocalStorageKey } from '../../utils/local-manager';
 import { DISCONNECT_TIME } from '../../utils/constants';
+import { isShowVideo } from '../../utils/utils';
 
 const { Text, Link } = Typography;
 
@@ -49,12 +50,19 @@ const Header: React.FC<HeaderProps> = ({
   const [connectLeftTime, setConnectLeftTime] = useState(DISCONNECT_TIME);
   const [audioCapture, setAudioCapture] = useState<string>('default');
   const [audioPlayback, setAudioPlayback] = useState<string>('default');
+  const [videoInputDeviceId, setVideoInputDeviceId] =
+    useState<string>('default');
+
   const [inputDeviceOptions, setInputDeviceOptions] = useState<
     { label: string; value: string }[]
   >([]);
   const [outputDeviceOptions, setOutputDeviceOptions] = useState<
     { label: string; value: string }[]
   >([]);
+  const [videoInputDeviceOptions, setVideoInputDeviceOptions] = useState<
+    { label: string; value: string }[]
+  >([]);
+
   const formRef = useRef<MessageFormRef>(null);
 
   const checkMicrophonePermission = () => {
@@ -93,7 +101,7 @@ const Header: React.FC<HeaderProps> = ({
   useEffect(() => {
     async function getDevices() {
       try {
-        const devices = await RealtimeUtils.getAudioDevices();
+        const devices = await RealtimeUtils.getAudioDevices({ video: true });
         const inputDevices = devices.audioInputs.map(device => ({
           label: device.label || `Microphone ${device.deviceId}`,
           value: device.deviceId,
@@ -102,9 +110,14 @@ const Header: React.FC<HeaderProps> = ({
           label: device.label || `Speaker ${device.deviceId}`,
           value: device.deviceId,
         }));
+        const videoInputDevices = devices.videoInputs.map(device => ({
+          label: device.label || `Video Input ${device.deviceId}`,
+          value: device.deviceId,
+        }));
 
         setInputDeviceOptions(inputDevices);
         setOutputDeviceOptions(outputDevices);
+        setVideoInputDeviceOptions(videoInputDevices);
       } catch (err) {
         console.error('Failed to get devices:', err);
         message.error('Failed to get devices');
@@ -270,6 +283,11 @@ const Header: React.FC<HeaderProps> = ({
     clientRef.current?.setAudioOutputDevice(value);
   };
 
+  const handleVideoInputDeviceChange = (value: string) => {
+    setVideoInputDeviceId(value);
+    clientRef.current?.setVideoInputDevice(value);
+  };
+
   if (microphoneStatus === 'error') {
     return (
       <>
@@ -351,6 +369,17 @@ const Header: React.FC<HeaderProps> = ({
               options={outputDeviceOptions}
             />
           </span>
+          {isShowVideo() && (
+            <span>
+              <Text style={{ marginRight: 8 }}>Video Device:</Text>
+              <Select
+                style={{ width: 200 }}
+                value={videoInputDeviceId}
+                onChange={handleVideoInputDeviceChange}
+                options={videoInputDeviceOptions}
+              />
+            </span>
+          )}
         </div>
       </>
     );

packages/realtime-api/README.md

@@ -58,6 +58,10 @@ const client = new RealtimeClient({
   audioMutedDefault: false,           // Optional: Initial audio state (default: false)
   suppressStationaryNoise: false,     // Optional: Enable stationary noise suppression(default: false)
   suppressNonStationaryNoise: false,  // Optional: Enable non-stationary noise suppression(default: false)
+  videoConfig: { // Optional: Video configuration
+    videoOnDefault: true, // Optional: Whether to turn on video by default, defaults to true
+    renderDom: 'local-player' // Optional: The DOM element to render the video stream to
+  },
 });
 
 // Essential Setup

packages/realtime-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coze/realtime-api",
-  "version": "1.0.3",
+  "version": "1.0.3-beta.6",
   "description": "A powerful real-time communication SDK for voice interactions with Coze AI bots | 扣子官方实时通信 SDK，用于与 Coze AI bots 进行语音交互",
   "keywords": [
     "coze",
@@ -66,4 +66,4 @@
     "typescript": "^5.5.3",
     "vitest": "~2.1.4"
   }
-}
+}

packages/realtime-api/src/client.ts

@@ -9,7 +9,7 @@
   type UserMessageEvent,
 } from '@volcengine/rtc';
 
-import { getAudioDevices } from './utils';
+import { getAudioDevices, isScreenShareDevice } from './utils';
 import { EventNames, RealtimeEventHandler } from './event-handler';
 import { RealtimeAPIError, RealtimeError } from './error';
 import { type VideoConfig } from '.';
@@ -19,13 +19,17 @@
   private joinUserId = '';
   private _AIAnsExtension: RTCAIAnsExtension | null = null;
   private _isSupportVideo = false;
+  private _videoConfig?: VideoConfig;
+  private _streamIndex?: StreamIndex;
+  private _roomUserId?: string;
 
   // eslint-disable-next-line max-params
   constructor(
     appId: string,
     debug = false,
     isTestEnv = false,
     isSupportVideo = false,
+    videoConfig?: VideoConfig,
   ) {
     super(debug);
 
@@ -48,6 +52,7 @@
       this.handleRemoteAudioPropertiesReport.bind(this);
 
     this._isSupportVideo = isSupportVideo;
+    this._videoConfig = videoConfig;
   }
 
   bindEngineEvents() {
@@ -205,15 +210,52 @@
     await this.engine.setAudioPlaybackDevice(deviceId);
   }
 
+  async setVideoInputDevice(deviceId: string, isAutoCapture = true) {
+    const devices = await getAudioDevices({ video: true });
+    if (devices.videoInputs.findIndex(i => i.deviceId === deviceId) === -1) {
+      throw new RealtimeAPIError(
+        RealtimeError.DEVICE_ACCESS_ERROR,
+        `Video input device not found: ${deviceId}`,
+      );
+    }
+
+    await this.changeVideoState(false);
+    if (isScreenShareDevice(deviceId)) {
+      if (this._streamIndex === StreamIndex.STREAM_INDEX_MAIN) {
+        this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN);
+      }
+      if (isAutoCapture) {
+        await this.engine.startScreenCapture(this._videoConfig?.screenConfig);
+      }
+      this._streamIndex = StreamIndex.STREAM_INDEX_SCREEN;
+    } else {
+      if (this._streamIndex === StreamIndex.STREAM_INDEX_SCREEN) {
+        this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_SCREEN);
+      }
+
+      if (isAutoCapture) {
+        await this.engine.startVideoCapture(deviceId);
+      }
+      this._streamIndex = StreamIndex.STREAM_INDEX_MAIN;
+    }
+
+    this.engine.setLocalVideoPlayer(this._streamIndex, {
+      renderDom: this._videoConfig?.renderDom || 'local-player',
+      userId: this._roomUserId,
+    });
+  }
+
   async createLocalStream(userId?: string, videoConfig?: VideoConfig) {
+    this._roomUserId = userId;
     const devices = await getAudioDevices({ video: this._isSupportVideo });
     if (!devices.audioInputs.length) {
       throw new RealtimeAPIError(
         RealtimeError.DEVICE_ACCESS_ERROR,
         'Failed to get audio devices',
       );
     }
-    if (this._isSupportVideo && !devices.videoInputs.length) {
+
+    if (!devices.videoInputs.length) {
       throw new RealtimeAPIError(
         RealtimeError.DEVICE_ACCESS_ERROR,
         'Failed to get video devices',
@@ -222,24 +264,20 @@
 
     await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
 
-    if (this._isSupportVideo && videoConfig?.videoOnDefault) {
-      await this.engine.startVideoCapture(devices.videoInputs[0].deviceId);
-    }
-
     if (this._isSupportVideo) {
-      this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, {
-        renderDom: videoConfig?.renderDom || 'local-player',
-        userId,
-      });
+      this.setVideoInputDevice(
+        videoConfig?.videoInputDeviceId || devices.videoInputs[0].deviceId,
+        videoConfig?.videoOnDefault,
+      );
     }
   }
 
   async disconnect() {
     try {
       if (this._isSupportVideo) {
-        await this.engine.stopVideoCapture();
+        await this.changeVideoState(false);
       }
-      await this.engine.stopAudioCapture();
+      await this.changeAudioState(false);
       await this.engine.unpublishStream(MediaType.AUDIO);
       await this.engine.leaveRoom();
       this.removeEventListener();
@@ -265,9 +303,17 @@
   async changeVideoState(isVideoOn: boolean) {
     try {
       if (isVideoOn) {
-        await this.engine.startVideoCapture();
+        if (this._streamIndex === StreamIndex.STREAM_INDEX_MAIN) {
+          await this.engine.startVideoCapture();
+        } else {
+          await this.engine.startScreenCapture(this._videoConfig?.screenConfig);
+        }
       } else {
-        await this.engine.stopVideoCapture();
+        if (this._streamIndex === StreamIndex.STREAM_INDEX_MAIN) {
+          await this.engine.stopVideoCapture();
+        } else {
+          await this.engine.stopScreenCapture();
+        }
       }
     } catch (e) {
       this.dispatch(EventNames.ERROR, e);

packages/realtime-api/src/event-handler.ts

@@ -81,6 +81,11 @@ export enum EventNames {
    * zh: 音频输出设备改变
    */
   AUDIO_OUTPUT_DEVICE_CHANGED = 'client.output.device.changed',
+  /**
+   * en: Video input device changed
+   * zh: 视频输入设备改变
+   */
+  VIDEO_INPUT_DEVICE_CHANGED = 'client.video.input.device.changed',
   /**
    * en: Bot joined
    * zh: Bot 加入

packages/realtime-api/src/index.ts

@@ -1,4 +1,4 @@
-import { type AudioPropertiesConfig } from '@volcengine/rtc';
+import { type ScreenConfig, type AudioPropertiesConfig } from '@volcengine/rtc';
 import { CozeAPI, type CreateRoomData, type GetToken } from '@coze/api';
 
 import * as RealtimeUtils from './utils';
@@ -8,6 +8,8 @@ import { EngineClient } from './client';
 export interface VideoConfig {
   videoOnDefault?: boolean /** optional, Whether to turn on video by default, defaults to true */;
   renderDom?: string /** optional, The DOM element to render the video stream to */;
+  videoInputDeviceId?: string /** optional, The device ID of the video input device to use */;
+  screenConfig?: ScreenConfig; // optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details
 }
 
 export interface RealtimeClientConfig {
@@ -67,6 +69,16 @@ class RealtimeClient extends RealtimeEventHandler {
    * @param config.suppressNonStationaryNoise - Optional, suppress non-stationary noise, defaults to false. |
    *                                         可选，默认是否抑制非静态噪声，默认值为 false。
    * @param config.isAutoSubscribeAudio - Optional, whether to automatically subscribe to bot reply audio streams, defaults to true. |
+   * @param config.videoConfig - Optional, Video configuration. |
+   *                            可选，视频配置。
+   * @param config.videoConfig.videoOnDefault - Optional, Whether to turn on video by default, defaults to true. |
+   *                                            可选，默认是否开启视频，默认值为 true。
+   * @param config.videoConfig.renderDom - Optional, The DOM element to render the video stream to. |
+   *                                       可选，渲染视频流的 DOM 元素。
+   * @param config.videoConfig.videoInputDeviceId - Optional, The device ID of the video input device to use. |
+   *                                               可选，视频输入设备的设备 ID。
+   * @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
+   *                                         可选，屏幕共享配置，如果 videoInputDeviceId 是 'screenShare'，请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
    */
   constructor(config: RealtimeClientConfig) {
     super(config.debug);
@@ -117,6 +129,7 @@ class RealtimeClient extends RealtimeEventHandler {
       this._config.debug,
       this._isTestEnv,
       this._isSupportVideo,
+      this._config.videoConfig,
     );
 
     // Step3 bind engine events
@@ -287,6 +300,11 @@ class RealtimeClient extends RealtimeEventHandler {
     await this._client?.setAudioOutputDevice(deviceId);
     this.dispatch(EventNames.AUDIO_OUTPUT_DEVICE_CHANGED, { deviceId });
   }
+
+  async setVideoInputDevice(deviceId: string) {
+    await this._client?.setVideoInputDevice(deviceId);
+    this.dispatch(EventNames.VIDEO_INPUT_DEVICE_CHANGED, { deviceId });
+  }
 }
 
 export {