Skip to content

Commit 3c658a8

Browse files
authored
fix: configure multimodal model capabilities (#508)
1 parent a3ea444 commit 3c658a8

5 files changed

Lines changed: 120 additions & 1 deletion

File tree

packages/domain/src/entities/model.entity.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ export const LLMModelItemSchema = z.object({
3737

3838
censor: z.boolean().optional(),
3939
vision: z.boolean(),
40+
audio: z.boolean().optional(),
41+
video: z.boolean().optional(),
4042
reasoning: z.boolean(),
4143
reasoningEffort: z.boolean(),
4244
toolChoice: z.boolean(),

packages/infrastructure/src/static-data/models/provider/Doubao/index.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ const models: ProviderConfigType = {
1616
quoteMaxToken: 1024000,
1717
maxTemperature: 1,
1818
vision: true,
19+
video: true,
1920
reasoning: true,
2021
reasoningEffort: true,
2122
toolChoice: true
@@ -28,6 +29,7 @@ const models: ProviderConfigType = {
2829
quoteMaxToken: 256000,
2930
maxTemperature: 1,
3031
vision: true,
32+
video: true,
3133
reasoning: true,
3234
reasoningEffort: true,
3335
toolChoice: true
@@ -40,6 +42,7 @@ const models: ProviderConfigType = {
4042
quoteMaxToken: 256000,
4143
maxTemperature: 1,
4244
vision: true,
45+
video: true,
4346
reasoning: true,
4447
reasoningEffort: true,
4548
toolChoice: true
@@ -52,6 +55,7 @@ const models: ProviderConfigType = {
5255
quoteMaxToken: 256000,
5356
maxTemperature: 1,
5457
vision: true,
58+
video: true,
5559
reasoning: true,
5660
reasoningEffort: true,
5761
toolChoice: true
@@ -64,6 +68,7 @@ const models: ProviderConfigType = {
6468
quoteMaxToken: 256000,
6569
maxTemperature: 1,
6670
vision: true,
71+
video: true,
6772
reasoning: true,
6873
reasoningEffort: true,
6974
toolChoice: true
@@ -76,6 +81,7 @@ const models: ProviderConfigType = {
7681
quoteMaxToken: 224000,
7782
maxTemperature: 1,
7883
vision: true,
84+
video: true,
7985
reasoning: true,
8086
reasoningEffort: true,
8187
toolChoice: true
@@ -88,6 +94,8 @@ const models: ProviderConfigType = {
8894
quoteMaxToken: 224000,
8995
maxTemperature: 1,
9096
vision: true,
97+
audio: true,
98+
video: true,
9199
reasoning: true,
92100
reasoningEffort: true,
93101
toolChoice: true
@@ -100,6 +108,7 @@ const models: ProviderConfigType = {
100108
quoteMaxToken: 224000,
101109
maxTemperature: 1,
102110
vision: true,
111+
video: true,
103112
reasoning: true,
104113
reasoningEffort: true,
105114
toolChoice: true
@@ -112,6 +121,8 @@ const models: ProviderConfigType = {
112121
quoteMaxToken: 224000,
113122
maxTemperature: 1,
114123
vision: true,
124+
audio: true,
125+
video: true,
115126
reasoning: true,
116127
reasoningEffort: true,
117128
toolChoice: true
@@ -124,6 +135,7 @@ const models: ProviderConfigType = {
124135
quoteMaxToken: 224000,
125136
maxTemperature: 1,
126137
vision: true,
138+
video: true,
127139
reasoning: true,
128140
reasoningEffort: true,
129141
toolChoice: true
@@ -136,6 +148,7 @@ const models: ProviderConfigType = {
136148
quoteMaxToken: 224000,
137149
maxTemperature: 1,
138150
vision: true,
151+
video: true,
139152
reasoning: true,
140153
reasoningEffort: true,
141154
toolChoice: true

packages/infrastructure/src/static-data/models/provider/Gemini/index.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ const models: ProviderConfigType = {
1111
quoteMaxToken: 1000000,
1212
maxTemperature: 1,
1313
vision: true,
14+
audio: true,
15+
video: true,
1416
reasoning: true,
1517
reasoningEffort: true,
1618
toolChoice: true
@@ -23,6 +25,8 @@ const models: ProviderConfigType = {
2325
quoteMaxToken: 1000000,
2426
maxTemperature: 1,
2527
vision: true,
28+
audio: true,
29+
video: true,
2630
reasoning: true,
2731
reasoningEffort: true,
2832
toolChoice: true
@@ -35,6 +39,8 @@ const models: ProviderConfigType = {
3539
quoteMaxToken: 1000000,
3640
maxTemperature: 1,
3741
vision: true,
42+
audio: true,
43+
video: true,
3844
reasoning: true,
3945
reasoningEffort: true,
4046
toolChoice: true
@@ -47,6 +53,8 @@ const models: ProviderConfigType = {
4753
quoteMaxToken: 1000000,
4854
maxTemperature: 1,
4955
vision: true,
56+
audio: true,
57+
video: true,
5058
reasoning: true,
5159
reasoningEffort: true,
5260
toolChoice: true
@@ -59,6 +67,8 @@ const models: ProviderConfigType = {
5967
quoteMaxToken: 1000000,
6068
maxTemperature: 1,
6169
vision: true,
70+
audio: true,
71+
video: true,
6272
reasoning: true,
6373
reasoningEffort: true,
6474
toolChoice: true
@@ -71,6 +81,8 @@ const models: ProviderConfigType = {
7181
quoteMaxToken: 1000000,
7282
maxTemperature: 1,
7383
vision: true,
84+
audio: true,
85+
video: true,
7486
reasoning: true,
7587
reasoningEffort: true,
7688
toolChoice: true
@@ -83,6 +95,8 @@ const models: ProviderConfigType = {
8395
quoteMaxToken: 1000000,
8496
maxTemperature: 1,
8597
vision: true,
98+
audio: true,
99+
video: true,
86100
reasoning: true,
87101
reasoningEffort: true,
88102
toolChoice: true
@@ -95,6 +109,8 @@ const models: ProviderConfigType = {
95109
quoteMaxToken: 1000000,
96110
maxTemperature: 1,
97111
vision: true,
112+
audio: true,
113+
video: true,
98114
reasoning: false,
99115
reasoningEffort: false,
100116
toolChoice: true
@@ -107,6 +123,8 @@ const models: ProviderConfigType = {
107123
quoteMaxToken: 1000000,
108124
maxTemperature: 1,
109125
vision: true,
126+
audio: true,
127+
video: true,
110128
reasoning: false,
111129
reasoningEffort: false,
112130
toolChoice: true
@@ -119,6 +137,8 @@ const models: ProviderConfigType = {
119137
quoteMaxToken: 1000000,
120138
maxTemperature: 1,
121139
vision: true,
140+
audio: true,
141+
video: true,
122142
reasoning: false,
123143
reasoningEffort: false,
124144
toolChoice: true

packages/infrastructure/src/static-data/models/provider/Qwen/index.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ const models: ProviderConfigType = {
1616
quoteMaxToken: 1000000,
1717
maxTemperature: 1,
1818
responseFormatList: ['text', 'json_object', 'json_schema'],
19-
vision: false,
19+
vision: true,
20+
video: true,
2021
reasoning: true,
2122
reasoningEffort: true,
2223
toolChoice: true
@@ -30,6 +31,7 @@ const models: ProviderConfigType = {
3031
maxTemperature: 1,
3132
responseFormatList: ['text', 'json_object', 'json_schema'],
3233
vision: true,
34+
video: true,
3335
reasoning: true,
3436
reasoningEffort: true,
3537
toolChoice: true
@@ -56,6 +58,7 @@ const models: ProviderConfigType = {
5658
maxTemperature: 1,
5759
responseFormatList: ['text', 'json_object', 'json_schema'],
5860
vision: true,
61+
video: true,
5962
reasoning: true,
6063
reasoningEffort: true,
6164
toolChoice: true
@@ -69,6 +72,7 @@ const models: ProviderConfigType = {
6972
maxTemperature: 1,
7073
responseFormatList: ['text', 'json_object', 'json_schema'],
7174
vision: true,
75+
video: true,
7276
reasoning: true,
7377
reasoningEffort: true,
7478
toolChoice: true
@@ -82,6 +86,7 @@ const models: ProviderConfigType = {
8286
maxTemperature: 1,
8387
responseFormatList: ['text', 'json_object', 'json_schema'],
8488
vision: true,
89+
video: true,
8590
reasoning: true,
8691
reasoningEffort: true,
8792
toolChoice: true
@@ -95,6 +100,7 @@ const models: ProviderConfigType = {
95100
maxTemperature: 1,
96101
responseFormatList: ['text', 'json_object', 'json_schema'],
97102
vision: true,
103+
video: true,
98104
reasoning: true,
99105
reasoningEffort: true,
100106
toolChoice: true
@@ -121,6 +127,7 @@ const models: ProviderConfigType = {
121127
maxTemperature: 1,
122128
responseFormatList: ['text', 'json_object'],
123129
vision: true,
130+
video: true,
124131
reasoning: false,
125132
reasoningEffort: false,
126133
toolChoice: true
@@ -134,6 +141,7 @@ const models: ProviderConfigType = {
134141
maxTemperature: 1,
135142
responseFormatList: ['text', 'json_object'],
136143
vision: true,
144+
video: true,
137145
reasoning: false,
138146
reasoningEffort: false,
139147
toolChoice: true
@@ -160,6 +168,7 @@ const models: ProviderConfigType = {
160168
maxTemperature: 1,
161169
responseFormatList: ['text', 'json_object'],
162170
vision: true,
171+
video: true,
163172
reasoning: false,
164173
reasoningEffort: false,
165174
toolChoice: false
@@ -186,6 +195,7 @@ const models: ProviderConfigType = {
186195
maxTemperature: 1,
187196
responseFormatList: ['text', 'json_object'],
188197
vision: true,
198+
video: true,
189199
reasoning: false,
190200
reasoningEffort: false,
191201
toolChoice: false
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { describe, expect, it } from 'vitest';
2+
3+
import {
4+
type LLMModelItemType,
5+
ModelItemSchema,
6+
ModelTypeEnum
7+
} from '@domain/entities/model.entity';
8+
9+
import doubao from './Doubao';
10+
import gemini from './Gemini';
11+
import qwen from './Qwen';
12+
13+
const staticModelList = [gemini, qwen, doubao].flatMap((provider) =>
14+
provider.list.map((model) =>
15+
ModelItemSchema.parse({
16+
...model,
17+
provider: provider.provider,
18+
name: model.name ?? model.model
19+
})
20+
)
21+
);
22+
23+
const getModel = (provider: string, model: string) =>
24+
staticModelList.find((item) => item.provider === provider && item.model === model);
25+
26+
describe('static model multimodal capabilities', () => {
27+
it('marks Gemini LLMs as supporting image, audio, and video inputs', () => {
28+
const llms = staticModelList.filter(
29+
(item): item is LLMModelItemType =>
30+
item.provider === 'Gemini' && item.type === ModelTypeEnum.llm
31+
);
32+
33+
expect(llms).not.toHaveLength(0);
34+
expect(llms.every((model) => model.vision && model.audio && model.video)).toBe(true);
35+
});
36+
37+
it('marks Qwen visual models with their video input capability', () => {
38+
expect(getModel('Qwen', 'qwen3.7-max')).toMatchObject({
39+
vision: true,
40+
video: true
41+
});
42+
expect(getModel('Qwen', 'qwen3.7-plus')).toMatchObject({
43+
vision: true,
44+
video: true
45+
});
46+
expect(getModel('Qwen', 'qwen3-vl-plus')).toMatchObject({
47+
vision: true,
48+
video: true
49+
});
50+
expect(getModel('Qwen', 'qwen-vl-max')).toMatchObject({
51+
vision: true,
52+
video: true
53+
});
54+
expect(getModel('Qwen', 'qwen3-max')).toMatchObject({ vision: false });
55+
});
56+
57+
it('distinguishes Doubao audio-capable versions from video-only versions', () => {
58+
expect(getModel('Doubao', 'doubao-seed-evolving')).toMatchObject({
59+
vision: true,
60+
video: true
61+
});
62+
expect(getModel('Doubao', 'doubao-seed-2-0-lite-260428')).toMatchObject({
63+
vision: true,
64+
audio: true,
65+
video: true
66+
});
67+
expect(getModel('Doubao', 'doubao-seed-2-0-mini-260428')).toMatchObject({
68+
vision: true,
69+
audio: true,
70+
video: true
71+
});
72+
expect(getModel('Doubao', 'doubao-seed-2-0-pro-260215')).not.toHaveProperty('audio');
73+
});
74+
});

0 commit comments

Comments
 (0)