Skip to content

Commit 0f37a1b

Browse files
chenyuliang-starchenyuliang-star
authored
feat: dev realtime-api TurnDetection params (#300)
* feat: rpc demo add turn detection * fix: fix turnDetection params * feat: upgrade api version
1 parent ec619e9 commit 0f37a1b

File tree

13 files changed

+315
-62
lines changed

13 files changed

+315
-62
lines changed

‎common/changes/@coze/api/feat-dev-rpc-turn-detection_2025-09-10-12-10.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@coze/api",
5+
"comment": "rpc demo add turn detection",
6+
"type": "minor"
7+
}
8+
],
9+
"packageName": "@coze/api",
10+
"email": "[email protected]"
11+
}

‎common/changes/@coze/api/feat-dev-rpc-turn-detection_2025-09-16-05-11.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@coze/api",
5+
"comment": "upgrade api version",
6+
"type": "minor"
7+
}
8+
],
9+
"packageName": "@coze/api",
10+
"email": "[email protected]"
11+
}

‎common/changes/@coze/realtime-api/feat-dev-rpc-turn-detection_2025-09-10-12-10.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@coze/realtime-api",
5+
"comment": "rpc demo add turn detection",
6+
"type": "minor"
7+
}
8+
],
9+
"packageName": "@coze/realtime-api",
10+
"email": "[email protected]"
11+
}

‎common/changes/@coze/realtime-api/feat-dev-rpc-turn-detection_2025-09-16-05-11.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@coze/realtime-api",
5+
"comment": "upgrade api version",
6+
"type": "minor"
7+
}
8+
],
9+
"packageName": "@coze/realtime-api",
10+
"email": "[email protected]"
11+
}

‎examples/realtime-console/src/pages/main/header.tsx‎

Lines changed: 12 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
import { AudioOutlined, AudioMutedOutlined } from '@ant-design/icons';
1111

1212
import '../../App.css';
13-
import { ChatEventType } from '@coze/api';
13+
import { ChatEventType, CreateRoomTurnDetectionType } from '@coze/api';
1414

1515
import { isShowVideo } from '../../utils/utils';
1616
import { LocalManager, LocalStorageKey } from '../../utils/local-manager';
@@ -19,6 +19,7 @@ import useIsMobile from '../../hooks/use-is-mobile';
1919
import useCozeAPI from '../../hooks/use-coze-api';
2020
import MessageForm, { type MessageFormRef } from './message-form';
2121
import LiveInfo from './live-info';
22+
import HoldToTalk from './hold-to-talk';
2223
import ComfortStrategyForm from './comfort-strategy-form';
2324

2425
const { Text, Link } = Typography;
@@ -30,6 +31,7 @@ interface HeaderProps {
3031
isMicrophoneOn: boolean;
3132
isConnected?: boolean;
3233
clientRef: React.MutableRefObject<RealtimeClient | null>;
34+
turnDetectionType?: CreateRoomTurnDetectionType;
3335
}
3436

3537
const Header: React.FC<HeaderProps> = ({
@@ -39,18 +41,18 @@ const Header: React.FC<HeaderProps> = ({
3941
isConnected,
4042
clientRef,
4143
isMicrophoneOn,
44+
turnDetectionType,
4245
}) => {
4346
const localManager = new LocalManager();
4447
const [isConnecting, setIsConnecting] = useState(false);
4548
const [microphoneStatus, setMicrophoneStatus] = useState<'normal' | 'error'>(
4649
'normal',
4750
);
48-
// const [isAudioPlaybackDeviceTest, setIsAudioPlaybackDeviceTest] =
49-
// useState(false);
5051
const [noiseSuppression, setNoiseSuppression] = useState<string[]>(() => {
5152
const savedValue = localManager.get(LocalStorageKey.NOISE_SUPPRESSION);
5253
return savedValue ? JSON.parse(savedValue) : [];
5354
});
55+
5456
const [connectLeftTime, setConnectLeftTime] = useState(DISCONNECT_TIME);
5557
const [audioCapture, setAudioCapture] = useState<string>('default');
5658
const [audioPlayback, setAudioPlayback] = useState<string>('default');
@@ -92,6 +94,7 @@ const Header: React.FC<HeaderProps> = ({
9294
useEffect(() => {
9395
checkMicrophonePermission();
9496
}, []);
97+
9598
useEffect(() => {
9699
if (isConnected && connectLeftTime > 0) {
97100
const timer = setInterval(() => {
@@ -262,46 +265,6 @@ const Header: React.FC<HeaderProps> = ({
262265
}
263266
};
264267

265-
// const handleEnableAudioPropertiesReport = () => {
266-
// if (!clientRef?.current) {
267-
// message.error('Please click Settings to set configuration first');
268-
// return;
269-
// }
270-
// try {
271-
// clientRef?.current?.enableAudioPropertiesReport({ interval: 1000 });
272-
// message.success('Audio properties reporting enabled');
273-
// } catch (error) {
274-
// message.error('Failed to enable audio properties reporting');
275-
// console.error(error);
276-
// }
277-
// };
278-
279-
// const handleAudioPlaybackDeviceTest = () => {
280-
// if (!clientRef?.current) {
281-
// message.error('Please click Settings to set configuration first');
282-
// return;
283-
// }
284-
// if (isAudioPlaybackDeviceTest) {
285-
// try {
286-
// clientRef.current.stopAudioPlaybackDeviceTest();
287-
// setIsAudioPlaybackDeviceTest(false);
288-
// message.success('Audio playback device test stopped');
289-
// } catch (error) {
290-
// message.error('Failed to stop audio playback device test');
291-
// console.error(error);
292-
// }
293-
// } else {
294-
// try {
295-
// clientRef.current.startAudioPlaybackDeviceTest();
296-
// setIsAudioPlaybackDeviceTest(true);
297-
// message.success('Audio playback device test started');
298-
// } catch (error) {
299-
// message.error('Failed to start audio playback device test');
300-
// console.error(error);
301-
// }
302-
// }
303-
// };
304-
305268
const handleAudioCaptureChange = (value: string) => {
306269
setAudioCapture(value);
307270
clientRef.current?.setAudioInputDevice(value);
@@ -384,6 +347,7 @@ const Header: React.FC<HeaderProps> = ({
384347
</>
385348
);
386349
}
350+
387351
if (isConnected) {
388352
return (
389353
<>
@@ -427,20 +391,6 @@ const Header: React.FC<HeaderProps> = ({
427391
<>
428392
<div style={{ marginTop: '10px' }}></div>
429393
<MessageForm onSubmit={handleSendMessage} ref={formRef} />
430-
{/* <Button
431-
type="primary"
432-
style={{ marginRight: '10px', marginLeft: '10px' }}
433-
onClick={handleEnableAudioPropertiesReport}
434-
>
435-
Enable Audio Report
436-
</Button>
437-
<Button
438-
type="primary"
439-
className="button-margin-right"
440-
onClick={handleAudioPlaybackDeviceTest}
441-
>
442-
{isAudioPlaybackDeviceTest ? 'Stop' : 'Start'} Audio Device Test
443-
</Button> */}
444394
<Button
445395
type="primary"
446396
style={{ marginRight: '10px', marginLeft: '10px' }}
@@ -449,6 +399,10 @@ const Header: React.FC<HeaderProps> = ({
449399
Comfort Strategy
450400
</Button>
451401
<LiveInfo liveId={liveId} />
402+
{turnDetectionType ===
403+
CreateRoomTurnDetectionType.ClientInterrupt && (
404+
<HoldToTalk clientRef={clientRef} isConnected={isConnected} />
405+
)}
452406
<Row gutter={[16, 16]} justify="center" align="middle">
453407
<Col xs={24} sm={12} md={8}>
454408
<div>
@@ -496,6 +450,7 @@ const Header: React.FC<HeaderProps> = ({
496450
</>
497451
);
498452
}
453+
499454
return (
500455
<>
501456
<Text style={{ marginRight: 8 }}>Audio Noise Suppression:</Text>

‎examples/realtime-console/src/pages/main/hold-to-talk.tsx‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
import React, { useRef, useState } from 'react';
2+
3+
import { Button, message } from 'antd';
4+
import { type RealtimeClient } from '@coze/realtime-api';
5+
import { AudioOutlined } from '@ant-design/icons';
6+
7+
import '../../App.css';
8+
9+
const maxRecordingTime = 60; // 最大录音时长（秒）
10+
11+
export default (props: {
12+
clientRef: React.MutableRefObject<RealtimeClient | null>;
13+
isConnected?: boolean;
14+
}) => {
15+
const { clientRef, isConnected } = props;
16+
const [isPressRecording, setIsPressRecording] = useState(false);
17+
const [recordingDuration, setRecordingDuration] = useState(0);
18+
const [isCancelRecording, setIsCancelRecording] = useState(false);
19+
const startTouchY = useRef<number>(0);
20+
const recordTimer = useRef<NodeJS.Timeout | null>(null);
21+
22+
const handleVoiceButtonMouseDown = (
23+
e: React.MouseEvent | React.TouchEvent,
24+
) => {
25+
if (isConnected && clientRef.current) {
26+
startPressRecord(e);
27+
}
28+
};
29+
30+
const handleVoiceButtonMouseUp = (e: React.MouseEvent | React.TouchEvent) => {
31+
if (isPressRecording && !isCancelRecording) {
32+
finishPressRecord();
33+
} else if (isPressRecording && isCancelRecording) {
34+
cancelPressRecord();
35+
}
36+
};
37+
38+
const handleVoiceButtonMouseLeave = () => {
39+
if (isPressRecording) {
40+
cancelPressRecord();
41+
}
42+
};
43+
44+
const handleVoiceButtonMouseMove = (
45+
e: React.MouseEvent | React.TouchEvent,
46+
) => {
47+
if (isPressRecording && startTouchY.current) {
48+
// 上滑超过50px则取消发送
49+
const clientY =
50+
'touches' in e ? e.touches[0].clientY : (e as React.MouseEvent).clientY;
51+
if (clientY < startTouchY.current - 50) {
52+
setIsCancelRecording(true);
53+
} else {
54+
setIsCancelRecording(false);
55+
}
56+
}
57+
};
58+
59+
const startPressRecord = async (e: React.MouseEvent | React.TouchEvent) => {
60+
try {
61+
setIsPressRecording(true);
62+
setRecordingDuration(0);
63+
setIsCancelRecording(false);
64+
// Store initial touch position for determining sliding direction
65+
if ('clientY' in e) {
66+
startTouchY.current = (e as React.MouseEvent).clientY;
67+
} else if ('touches' in e && e.touches.length > 0) {
68+
startTouchY.current = e.touches[0].clientY;
69+
} else {
70+
startTouchY.current = 0;
71+
}
72+
73+
// 开始录音 TODO
74+
await clientRef.current?.sendMessage({
75+
id: `event_${Date.now()}`,
76+
event_type: 'input_audio_buffer.start',
77+
data: {},
78+
});
79+
80+
recordTimer.current = setInterval(() => {
81+
setRecordingDuration(prev => {
82+
const newDuration = prev + 1;
83+
// 超过最大录音时长自动结束
84+
if (newDuration >= maxRecordingTime) {
85+
finishPressRecord();
86+
}
87+
return newDuration;
88+
});
89+
}, 1000);
90+
} catch (error: unknown) {
91+
console.trace(error);
92+
message.error(`开始录音错误: ${error || '未知错误'}`);
93+
if (recordTimer.current) {
94+
clearInterval(recordTimer.current);
95+
recordTimer.current = null;
96+
}
97+
// Reset recording state
98+
setIsPressRecording(false);
99+
setRecordingDuration(0);
100+
}
101+
};
102+
103+
const finishPressRecord = async () => {
104+
if (isPressRecording && clientRef.current) {
105+
try {
106+
// 停止计时
107+
if (recordTimer.current) {
108+
clearInterval(recordTimer.current);
109+
recordTimer.current = null;
110+
}
111+
112+
// 如果录音时间太短（小于1秒），视为无效
113+
if (recordingDuration < 1) {
114+
cancelPressRecord();
115+
return;
116+
}
117+
118+
// 停止录音并发送 TODO
119+
120+
await clientRef.current?.sendMessage({
121+
id: `event_${Date.now()}`,
122+
event_type: 'input_audio_buffer.complete',
123+
data: {},
124+
});
125+
126+
setIsPressRecording(false);
127+
128+
// 显示提示
129+
message.success(`发送了 ${recordingDuration} 秒的语音消息`);
130+
} catch (error: unknown) {
131+
message.error(`结束录音错误: ${error || '未知错误'}`);
132+
console.error('结束录音错误:', error);
133+
}
134+
}
135+
};
136+
137+
const cancelPressRecord = async () => {
138+
if (isPressRecording && clientRef.current) {
139+
try {
140+
if (recordTimer.current) {
141+
clearInterval(recordTimer.current);
142+
recordTimer.current = null;
143+
}
144+
145+
// 取消录音 TODO
146+
await clientRef.current?.sendMessage({
147+
id: `event_${Date.now()}`,
148+
event_type: 'input_audio_buffer.complete',
149+
data: {},
150+
});
151+
152+
setIsPressRecording(false);
153+
setIsCancelRecording(false);
154+
155+
// 显示提示
156+
message.info('取消了语音消息');
157+
} catch (error: unknown) {
158+
message.error(`取消录音错误: ${error || '未知错误'}`);
159+
console.error('取消录音错误:', error);
160+
}
161+
}
162+
};
163+
164+
return (
165+
<Button
166+
color="default"
167+
onMouseDown={handleVoiceButtonMouseDown}
168+
onMouseUp={handleVoiceButtonMouseUp}
169+
onMouseLeave={handleVoiceButtonMouseLeave}
170+
onMouseMove={handleVoiceButtonMouseMove}
171+
onTouchStart={handleVoiceButtonMouseDown}
172+
onTouchEnd={handleVoiceButtonMouseUp}
173+
onTouchCancel={handleVoiceButtonMouseLeave}
174+
onTouchMove={handleVoiceButtonMouseMove}
175+
size="middle"
176+
icon={<AudioOutlined />}
177+
>
178+
{isPressRecording ? 'Release and send' : 'Hold to talk'}
179+
</Button>
180+
);
181+
};

0 commit comments

Comments
 (0)