Skip to content

Add AgentHandoff Chat Item #808

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 13, 2025
Merged

Add AgentHandoff Chat Item #808

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7e1d71c
save
toubatbrian Nov 6, 2025
1e39b1d
Merge branch 'brian/wait-python-1.3' into brian/handoff-chat-item
toubatbrian Nov 6, 2025
093e115
Merge branch 'brian/wait-python-1.3' into brian/handoff-chat-item
toubatbrian Nov 6, 2025
d57c817
Update agent_session.ts
toubatbrian Nov 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 53 additions & 1 deletion agents/src/llm/chat_context.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,59 @@ export class FunctionCallOutput {
}
}

export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput;
export class AgentHandoffItem {
readonly id: string;

readonly type = 'agent_handoff' as const;

oldAgentId: string | undefined;

newAgentId: string;

createdAt: number;

constructor(params: {
oldAgentId?: string;
newAgentId: string;
id?: string;
createdAt?: number;
}) {
const { oldAgentId, newAgentId, id = shortuuid('item_'), createdAt = Date.now() } = params;
this.id = id;
this.oldAgentId = oldAgentId;
this.newAgentId = newAgentId;
this.createdAt = createdAt;
}

static create(params: {
oldAgentId?: string;
newAgentId: string;
id?: string;
createdAt?: number;
}) {
return new AgentHandoffItem(params);
}

toJSON(excludeTimestamp: boolean = false): JSONValue {
const result: JSONValue = {
id: this.id,
type: this.type,
newAgentId: this.newAgentId,
};

if (this.oldAgentId !== undefined) {
result.oldAgentId = this.oldAgentId;
}

if (!excludeTimestamp) {
result.createdAt = this.createdAt;
}

return result;
}
}

export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput | AgentHandoffItem;

export class ChatContext {
protected _items: ChatItem[];
Expand Down
1 change: 1 addition & 0 deletions agents/src/llm/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export {
} from './tool_context.js';

export {
AgentHandoffItem,
ChatContext,
ChatMessage,
createAudioContent,
Expand Down
73 changes: 72 additions & 1 deletion agents/src/llm/provider_format/google.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { initializeLogger } from '../../log.js';
import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
import {
AgentHandoffItem,
ChatContext,
FunctionCall,
FunctionCallOutput,
} from '../chat_context.js';
import { serializeImage } from '../utils.js';
import { toChatCtx } from './google.js';

Expand Down Expand Up @@ -769,4 +774,70 @@ describe('Google Provider Format - toChatCtx', () => {
]);
expect(formatData.systemMessages).toBeNull();
});

it('should filter out agent handoff items', async () => {
const ctx = ChatContext.empty();

ctx.addMessage({ role: 'user', content: 'Hello' });

// Insert an agent handoff item
const handoff = new AgentHandoffItem({
oldAgentId: 'agent_1',
newAgentId: 'agent_2',
});
ctx.insert(handoff);

ctx.addMessage({ role: 'assistant', content: 'Hi there!' });

const [result, formatData] = await toChatCtx(ctx, false);

// Agent handoff should be filtered out, only messages should remain
expect(result).toEqual([
{
role: 'user',
parts: [{ text: 'Hello' }],
},
{
role: 'model',
parts: [{ text: 'Hi there!' }],
},
]);
expect(formatData.systemMessages).toBeNull();
});

it('should handle multiple agent handoffs without errors', async () => {
const ctx = ChatContext.empty();

ctx.addMessage({ role: 'user', content: 'Start' });

// Multiple handoffs
ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });

ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });

ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });

const [result, formatData] = await toChatCtx(ctx, false);

// All handoffs should be filtered out
// Note: Google provider groups consecutive messages by the same role
expect(result).toEqual([
{
role: 'user',
parts: [{ text: 'Start' }],
},
{
role: 'model',
parts: [
{ text: 'Response from agent 1' },
{ text: 'Response from agent 2' },
{ text: 'Response from agent 3' },
],
},
]);
expect(formatData.systemMessages).toBeNull();
});
});
56 changes: 55 additions & 1 deletion agents/src/llm/provider_format/openai.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { initializeLogger } from '../../log.js';
import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
import {
AgentHandoffItem,
ChatContext,
FunctionCall,
FunctionCallOutput,
} from '../chat_context.js';
import { serializeImage } from '../utils.js';
import { toChatCtx } from './openai.js';

Expand Down Expand Up @@ -578,4 +583,53 @@ describe('toChatCtx', () => {
},
]);
});

it('should filter out agent handoff items', async () => {
const ctx = ChatContext.empty();

ctx.addMessage({ role: 'user', content: 'Hello' });

// Insert an agent handoff item
const handoff = new AgentHandoffItem({
oldAgentId: 'agent_1',
newAgentId: 'agent_2',
});
ctx.insert(handoff);

ctx.addMessage({ role: 'assistant', content: 'Hi there!' });

const result = await toChatCtx(ctx);

// Agent handoff should be filtered out, only messages should remain
expect(result).toEqual([
{ role: 'user', content: 'Hello' },
{ role: 'assistant', content: 'Hi there!' },
]);
});

it('should handle multiple agent handoffs without errors', async () => {
const ctx = ChatContext.empty();

ctx.addMessage({ role: 'user', content: 'Start' });

// Multiple handoffs
ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });

ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });

ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });

const result = await toChatCtx(ctx);

// All handoffs should be filtered out
expect(result).toEqual([
{ role: 'user', content: 'Start' },
{ role: 'assistant', content: 'Response from agent 1' },
{ role: 'assistant', content: 'Response from agent 2' },
{ role: 'assistant', content: 'Response from agent 3' },
]);
});
});
5 changes: 3 additions & 2 deletions agents/src/llm/provider_format/openai.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,10 @@ async function toChatItem(item: ChatItem) {
tool_call_id: item.callId,
content: item.output,
};
} else {
throw new Error(`Unsupported item type: ${item['type']}`);
}
// Skip other item types (e.g., agent_handoff)
// These should be filtered by groupToolCalls, but this is a safety net
throw new Error(`Unsupported item type: ${item['type']}`);
}

async function toImageContent(content: ImageContent) {
Expand Down
22 changes: 22 additions & 0 deletions agents/src/voice/agent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export interface ModelSettings {
}

export interface AgentOptions<UserData> {
id?: string;
instructions: string;
Copy link
Contributor

@Shubhrakanti Shubhrakanti Nov 6, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this optional?

Copy link
Contributor Author

@toubatbrian toubatbrian Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This also follows python agents:

class Agent:
    def __init__(
        self,
        *,
        instructions: str,
        id: str | None = None,
        chat_ctx: NotGivenOr[llm.ChatContext | None] = NOT_GIVEN,
        tools: list[llm.FunctionTool | llm.RawFunctionTool] | None = None,
        turn_detection: NotGivenOr[TurnDetectionMode | None] = NOT_GIVEN,
        stt: NotGivenOr[stt.STT | STTModels | str | None] = NOT_GIVEN,
        vad: NotGivenOr[vad.VAD | None] = NOT_GIVEN,
        llm: NotGivenOr[llm.LLM | llm.RealtimeModel | LLMModels | str | None] = NOT_GIVEN,
        tts: NotGivenOr[tts.TTS | TTSModels | str | None] = NOT_GIVEN,
        mcp_servers: NotGivenOr[list[mcp.MCPServer] | None] = NOT_GIVEN,
        allow_interruptions: NotGivenOr[bool] = NOT_GIVEN,
        min_consecutive_speech_delay: NotGivenOr[float] = NOT_GIVEN,
        use_tts_aligned_transcript: NotGivenOr[bool] = NOT_GIVEN,
        min_endpointing_delay: NotGivenOr[float] = NOT_GIVEN,
        max_endpointing_delay: NotGivenOr[float] = NOT_GIVEN,
    ) -> None:
        tools = tools or []
        if type(self) is Agent:
            self._id = "default_agent"
        else:
            self._id = id or misc.camel_to_snake_case(type(self).__name__)

chatCtx?: ChatContext;
tools?: ToolContext<UserData>;
Expand All @@ -72,6 +73,7 @@ export interface AgentOptions<UserData> {
}

export class Agent<UserData = any> {
private _id: string;
private turnDetection?: TurnDetectionMode;
private _stt?: STT;
private _vad?: VAD;
Expand All @@ -91,6 +93,7 @@ export class Agent<UserData = any> {
_tools?: ToolContext<UserData>;

constructor({
id,
instructions,
chatCtx,
tools,
Expand All @@ -100,6 +103,21 @@ export class Agent<UserData = any> {
llm,
tts,
}: AgentOptions<UserData>) {
if (id) {
this._id = id;
} else {
// Convert class name to snake_case
const className = this.constructor.name;
if (className === 'Agent') {
this._id = 'default_agent';
} else {
this._id = className
.replace(/([A-Z])/g, '_$1')
.toLowerCase()
.replace(/^_/, '');
}
}

this._instructions = instructions;
this._tools = { ...tools };
this._chatCtx = chatCtx
Expand Down Expand Up @@ -152,6 +170,10 @@ export class Agent<UserData = any> {
return new ReadonlyChatContext(this._chatCtx.items);
}

get id(): string {
return this._id;
}

get instructions(): string {
return this._instructions;
}
Expand Down
13 changes: 12 additions & 1 deletion agents/src/voice/agent_session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
type TTSModelString,
} from '../inference/index.js';
import { getJobContext } from '../job.js';
import { ChatContext, ChatMessage } from '../llm/chat_context.js';
import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
import type { LLMError } from '../llm/llm.js';
import { log } from '../log.js';
Expand Down Expand Up @@ -362,6 +362,8 @@ export class AgentSession<
// TODO(AJS-129): add lock to agent activity core lifecycle
this.nextActivity = new AgentActivity(agent, this);

const previousActivity = this.activity;

if (this.activity) {
await this.activity.drain();
await this.activity.close();
Expand All @@ -370,6 +372,15 @@ export class AgentSession<
this.activity = this.nextActivity;
this.nextActivity = undefined;

// Insert agent handoff into chat context
this._chatCtx.insert(
new AgentHandoffItem({
oldAgentId: previousActivity?.agent.id,
newAgentId: agent.id,
}),
);
this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context');

await this.activity.start();

if (this._input.audio) {
Expand Down