From 2e4df98de0e77979b35979f73b3d95e986004517 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 18:00:30 -0800 Subject: [PATCH 01/11] Make tool tests to produce more deterministic results --- test-integration/node/generate-content-tools.test.ts | 8 ++++---- test-integration/node/start-chat-tools.test.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test-integration/node/generate-content-tools.test.ts b/test-integration/node/generate-content-tools.test.ts index 5bcb89519..28f73e2ef 100644 --- a/test-integration/node/generate-content-tools.test.ts +++ b/test-integration/node/generate-content-tools.test.ts @@ -49,7 +49,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, description: { type: SchemaType.STRING, @@ -70,7 +70,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, movie: { type: SchemaType.STRING, @@ -90,7 +90,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, movie: { type: SchemaType.STRING, @@ -119,7 +119,7 @@ describe("generateContent - tools", function () { role: "user", parts: [ { - text: "Which theaters in Mountain View show Barbie movie?", + text: "Which theaters in Mountain View (bay area) show Barbie movie?", }, ], }; diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index b4adf8136..360d9fffb 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -73,7 +73,7 @@ describe("startChat - tools", function () { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel( { - model: "gemini-1.5-flash-latest", + model: "gemini-1.5-pro", safetySettings: [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, From 3a7cf21063ce0aa82ecfda59214d5f88f1ec837d Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 18:00:30 -0800 Subject: [PATCH 02/11] Make tool tests to produce more deterministic results --- .changeset/lucky-grapes-rest.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-grapes-rest.md diff --git a/.changeset/lucky-grapes-rest.md b/.changeset/lucky-grapes-rest.md new file mode 100644 index 000000000..6817c0c08 --- /dev/null +++ b/.changeset/lucky-grapes-rest.md @@ -0,0 +1,5 @@ +--- +"@google/generative-ai": patch +--- + +Fix flaky integration test with tools From 07a1c6124d025922a75e8a52d269928c50c8f4d7 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 18:00:30 -0800 Subject: [PATCH 03/11] Make tool tests to produce more deterministic results --- test-integration/node/generate-content-tools.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-integration/node/generate-content-tools.test.ts b/test-integration/node/generate-content-tools.test.ts index 28f73e2ef..5779277bd 100644 --- a/test-integration/node/generate-content-tools.test.ts +++ b/test-integration/node/generate-content-tools.test.ts @@ -49,7 +49,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA", + "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", }, description: { type: SchemaType.STRING, @@ -70,7 +70,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA", + "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", }, movie: { type: SchemaType.STRING, @@ -90,7 +90,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA", + "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", }, movie: { type: SchemaType.STRING, @@ -119,7 +119,7 @@ describe("generateContent - tools", function () { role: "user", parts: [ { - text: "Which theaters in Mountain View (bay area) show Barbie movie?", + text: "Which theaters in Mountain View show Barbie movie?", }, ], }; @@ -185,7 +185,7 @@ describe("generateContent - tools", function () { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel( { - model: "gemini-1.5-flash-latest", + model: "gemini-1.5-pro", tools: [ { functionDeclarations: [ From edac5ff0d60e20ac017f43b85a8a33b364ae07c0 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 20:27:10 -0800 Subject: [PATCH 04/11] change model to 1.5 pro --- .../node/generate-content-tools.test.ts | 19 +++++++--------- .../node/start-chat-tools.test.ts | 22 ++++++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/test-integration/node/generate-content-tools.test.ts b/test-integration/node/generate-content-tools.test.ts index 5779277bd..062559079 100644 --- a/test-integration/node/generate-content-tools.test.ts +++ b/test-integration/node/generate-content-tools.test.ts @@ -35,7 +35,7 @@ describe("generateContent - tools", function () { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel( { - model: "gemini-1.5-flash-latest", + model: "gemini-1.5-pro-latest", tools: [ { functionDeclarations: [ @@ -49,7 +49,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, description: { type: SchemaType.STRING, @@ -70,7 +70,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, movie: { type: SchemaType.STRING, @@ -90,7 +90,7 @@ describe("generateContent - tools", function () { location: { type: SchemaType.STRING, description: - "The city and state, e.g. San Francisco, CA or a zip code e.g. 95616", + "The city and state, e.g. San Francisco, CA", }, movie: { type: SchemaType.STRING, @@ -111,15 +111,14 @@ describe("generateContent - tools", function () { ], }, ], - }, - { apiVersion: "v1beta" }, + } ); const src1 = { role: "user", parts: [ { - text: "Which theaters in Mountain View show Barbie movie?", + text: "Which theaters in Mountain View, CA show Barbie movie?", }, ], }; @@ -185,7 +184,7 @@ describe("generateContent - tools", function () { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel( { - model: "gemini-1.5-pro", + model: "gemini-1.5-pro-latest", tools: [ { functionDeclarations: [ @@ -204,8 +203,7 @@ describe("generateContent - tools", function () { ], }, ], - }, - { apiVersion: "v1beta" }, + } ); const src1: Content = { @@ -259,7 +257,6 @@ describe("generateContent - tools", function () { contents: [src1, src2, fn1, src3, fn1], }); const response = await result.response; - console.log(response.text()); expect(response.text()).to.match(/(\bsame\b|\byes\b)/i); }); }); diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index 360d9fffb..6c4714c52 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -32,18 +32,21 @@ use(chaiAsPromised); * Integration tests against live backend. */ -describe("startChat - tools", function () { +describe.only("startChat - tools", function () { const tools: Tool[] = [ { functionDeclarations: [ { - name: "getTemperature", + name: "get_current_weather", description: - "Get current temperature in degrees Celsius in a given city", + "Get the current weather in a given location", parameters: { type: SchemaType.OBJECT, properties: { - city: { type: SchemaType.STRING }, + city: { + type: SchemaType.STRING, + description: 'A city name, for example, San Francisco' + }, }, required: ["city"], }, @@ -53,13 +56,13 @@ describe("startChat - tools", function () { ]; const part1: Part = { - text: "What is the temperature in New York?", + text: "What is the current weather in New York?", }; const part2: Part = { functionResponse: { - name: "getTemperature", + name: "get_current_weather", response: { - name: "getTemperature", + name: "get_current_weather", content: { temperature: "30", }, @@ -73,7 +76,7 @@ describe("startChat - tools", function () { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel( { - model: "gemini-1.5-pro", + model: "gemini-1.5-pro-latest", safetySettings: [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, @@ -86,8 +89,7 @@ describe("startChat - tools", function () { ); const chat = model.startChat(); const result1 = await chat.sendMessage([part1]); - expect(result1.response.text()).to.be.empty; - expect(result1.response.functionCall()).not.to.be.empty; + expect(result1.response.functionCalls()).not.to.be.empty; const result2 = await chat.sendMessage([part2]); expect(result2.response.text()).to.not.be.empty; const history = await chat.getHistory(); From 7407e8159d9db47c1dcc4146aadc5ca7851c0960 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 20:27:10 -0800 Subject: [PATCH 05/11] change model to 1.5 pro --- test-integration/node/start-chat-tools.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index 6c4714c52..5adddc136 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -85,7 +85,6 @@ describe.only("startChat - tools", function () { ], tools, }, - { apiVersion: "v1beta" }, ); const chat = model.startChat(); const result1 = await chat.sendMessage([part1]); From 678ba57899b62f83ed5fd25984b0bad68dfb7d25 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 20:27:10 -0800 Subject: [PATCH 06/11] change model to 1.5 pro --- test-integration/node/generate-content-tools.test.ts | 2 +- test-integration/node/start-chat-tools.test.ts | 12 ++++++------ test-integration/node/start-chat.test.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test-integration/node/generate-content-tools.test.ts b/test-integration/node/generate-content-tools.test.ts index 062559079..003d5ab6c 100644 --- a/test-integration/node/generate-content-tools.test.ts +++ b/test-integration/node/generate-content-tools.test.ts @@ -118,7 +118,7 @@ describe("generateContent - tools", function () { role: "user", parts: [ { - text: "Which theaters in Mountain View, CA show Barbie movie?", + text: "Which theaters in Mountain View show Barbie movie?", }, ], }; diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index 5adddc136..f38bbdb3b 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -32,14 +32,14 @@ use(chaiAsPromised); * Integration tests against live backend. */ -describe.only("startChat - tools", function () { +describe("startChat - tools", function () { const tools: Tool[] = [ { functionDeclarations: [ { - name: "get_current_weather", + name: "getTemperature", description: - "Get the current weather in a given location", + "Get current temperature in degrees Celsius in a given city", parameters: { type: SchemaType.OBJECT, properties: { @@ -56,13 +56,13 @@ describe.only("startChat - tools", function () { ]; const part1: Part = { - text: "What is the current weather in New York?", + text: "What is the current temperature in New York?", }; const part2: Part = { functionResponse: { - name: "get_current_weather", + name: "getTemperature", response: { - name: "get_current_weather", + name: "getTemperature", content: { temperature: "30", }, diff --git a/test-integration/node/start-chat.test.ts b/test-integration/node/start-chat.test.ts index 5586f9263..2f944c2eb 100644 --- a/test-integration/node/start-chat.test.ts +++ b/test-integration/node/start-chat.test.ts @@ -90,7 +90,7 @@ describe("startChat", function () { it("stream true, try to send message before previous stream is done", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel({ - model: "gemini-1.5-flash-latest", + model: "gemini-1.5-pro-latest", safetySettings: [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, From 00ea1de4c74f385f25868e8f5fb65de827b8619c Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 20:27:10 -0800 Subject: [PATCH 07/11] change model to 1.5 pro --- test-integration/node/start-chat-tools.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index f38bbdb3b..f73fc4c66 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -56,7 +56,7 @@ describe("startChat - tools", function () { ]; const part1: Part = { - text: "What is the current temperature in New York?", + text: "What is the temperature in New York?", }; const part2: Part = { functionResponse: { From ddc777a465b3559595267663695f12b87a8cc1df Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Wed, 19 Feb 2025 20:45:17 -0800 Subject: [PATCH 08/11] formatting --- .../node/generate-content-tools.test.ts | 177 +++++++++--------- .../node/start-chat-tools.test.ts | 26 ++- 2 files changed, 97 insertions(+), 106 deletions(-) diff --git a/test-integration/node/generate-content-tools.test.ts b/test-integration/node/generate-content-tools.test.ts index 003d5ab6c..4b390c60d 100644 --- a/test-integration/node/generate-content-tools.test.ts +++ b/test-integration/node/generate-content-tools.test.ts @@ -33,86 +33,81 @@ describe("generateContent - tools", function () { // eslint-disable-next-line no-restricted-properties it("non-streaming, tools usage", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel( - { - model: "gemini-1.5-pro-latest", - tools: [ - { - functionDeclarations: [ - { - name: "find_movies", - description: - "find movie titles currently playing in theaters based on any description, genre, title words, etc.", - parameters: { - type: SchemaType.OBJECT, - properties: { - location: { - type: SchemaType.STRING, - description: - "The city and state, e.g. San Francisco, CA", - }, - description: { - type: SchemaType.STRING, - description: - "Any kind of description including category or genre, title words, attributes, etc.", - }, + const model = genAI.getGenerativeModel({ + model: "gemini-1.5-pro-latest", + tools: [ + { + functionDeclarations: [ + { + name: "find_movies", + description: + "find movie titles currently playing in theaters based on any description, genre, title words, etc.", + parameters: { + type: SchemaType.OBJECT, + properties: { + location: { + type: SchemaType.STRING, + description: "The city and state, e.g. San Francisco, CA", + }, + description: { + type: SchemaType.STRING, + description: + "Any kind of description including category or genre, title words, attributes, etc.", }, - required: ["description"], }, + required: ["description"], }, - { - name: "find_theaters", - description: - "find theaters based on location and optionally movie title which are is currently playing in theaters", - parameters: { - type: SchemaType.OBJECT, - properties: { - location: { - type: SchemaType.STRING, - description: - "The city and state, e.g. San Francisco, CA", - }, - movie: { - type: SchemaType.STRING, - description: "Any movie title", - }, + }, + { + name: "find_theaters", + description: + "find theaters based on location and optionally movie title which are is currently playing in theaters", + parameters: { + type: SchemaType.OBJECT, + properties: { + location: { + type: SchemaType.STRING, + description: "The city and state, e.g. San Francisco, CA", + }, + movie: { + type: SchemaType.STRING, + description: "Any movie title", }, - required: ["location"], }, + required: ["location"], }, - { - name: "get_showtimes", - description: - "Find the start times for movies playing in a specific theater", - parameters: { - type: SchemaType.OBJECT, - properties: { - location: { - type: SchemaType.STRING, - description: - "The city and state, e.g. San Francisco, CA", - }, - movie: { - type: SchemaType.STRING, - description: "Any movie title", - }, - theater: { - type: SchemaType.STRING, - description: "Name of the theater", - }, - date: { - type: SchemaType.STRING, - description: "Date for requested showtime", - }, + }, + { + name: "get_showtimes", + description: + "Find the start times for movies playing in a specific theater", + parameters: { + type: SchemaType.OBJECT, + properties: { + location: { + type: SchemaType.STRING, + description: "The city and state, e.g. San Francisco, CA", + }, + movie: { + type: SchemaType.STRING, + description: "Any movie title", + }, + theater: { + type: SchemaType.STRING, + description: "Name of the theater", + }, + date: { + type: SchemaType.STRING, + description: "Date for requested showtime", }, - required: ["location", "movie", "theater", "date"], }, + required: ["location", "movie", "theater", "date"], }, - ], - }, - ], - } - ); + }, + ], + }, + ], + }); const src1 = { role: "user", @@ -182,29 +177,27 @@ describe("generateContent - tools", function () { }); it("streaming, tools usage", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel( - { - model: "gemini-1.5-pro-latest", - tools: [ - { - functionDeclarations: [ - { - name: "getTemperature", - description: - "Get current temperature in degrees Celsius in a given city", - parameters: { - type: SchemaType.OBJECT, - properties: { - city: { type: SchemaType.STRING }, - }, - required: ["city"], + const model = genAI.getGenerativeModel({ + model: "gemini-1.5-pro-latest", + tools: [ + { + functionDeclarations: [ + { + name: "getTemperature", + description: + "Get current temperature in degrees Celsius in a given city", + parameters: { + type: SchemaType.OBJECT, + properties: { + city: { type: SchemaType.STRING }, }, + required: ["city"], }, - ], - }, - ], - } - ); + }, + ], + }, + ], + }); const src1: Content = { role: "user", diff --git a/test-integration/node/start-chat-tools.test.ts b/test-integration/node/start-chat-tools.test.ts index f73fc4c66..f773008d2 100644 --- a/test-integration/node/start-chat-tools.test.ts +++ b/test-integration/node/start-chat-tools.test.ts @@ -43,9 +43,9 @@ describe("startChat - tools", function () { parameters: { type: SchemaType.OBJECT, properties: { - city: { + city: { type: SchemaType.STRING, - description: 'A city name, for example, San Francisco' + description: "A city name, for example, San Francisco", }, }, required: ["city"], @@ -74,18 +74,16 @@ describe("startChat - tools", function () { this.slow(10e3); it("stream false", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel( - { - model: "gemini-1.5-pro-latest", - safetySettings: [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, - }, - ], - tools, - }, - ); + const model = genAI.getGenerativeModel({ + model: "gemini-1.5-pro-latest", + safetySettings: [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, + }, + ], + tools, + }); const chat = model.startChat(); const result1 = await chat.sendMessage([part1]); expect(result1.response.functionCalls()).not.to.be.empty; From e377fe9483c61896738c2f5ab70c7c789b6a01d7 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Thu, 20 Feb 2025 08:46:19 -0800 Subject: [PATCH 09/11] increase test timeout --- test-integration/node/cache-content.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/node/cache-content.test.ts b/test-integration/node/cache-content.test.ts index b6982a016..97b98ad0f 100644 --- a/test-integration/node/cache-content.test.ts +++ b/test-integration/node/cache-content.test.ts @@ -27,7 +27,7 @@ use(chaiAsPromised); */ describe("cacheContent", function () { - this.timeout(60e3); + this.timeout(120e3); this.slow(10e3); const model = "models/gemini-1.5-pro-001"; let text: string = ""; From 85acbf6546f110c59a40ee344cab6aebf694f5fd Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Thu, 20 Feb 2025 08:50:53 -0800 Subject: [PATCH 10/11] revert timeout back to 1m --- test-integration/node/cache-content.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/node/cache-content.test.ts b/test-integration/node/cache-content.test.ts index 97b98ad0f..b6982a016 100644 --- a/test-integration/node/cache-content.test.ts +++ b/test-integration/node/cache-content.test.ts @@ -27,7 +27,7 @@ use(chaiAsPromised); */ describe("cacheContent", function () { - this.timeout(120e3); + this.timeout(60e3); this.slow(10e3); const model = "models/gemini-1.5-pro-001"; let text: string = ""; From 19983673b8d81f4867d6ededa3853b3cc76d5d99 Mon Sep 17 00:00:00 2001 From: Kaituo Huang Date: Thu, 20 Feb 2025 10:08:51 -0800 Subject: [PATCH 11/11] reduce cache token size --- test-integration/node/cache-content.test.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test-integration/node/cache-content.test.ts b/test-integration/node/cache-content.test.ts index b6982a016..a16875cd1 100644 --- a/test-integration/node/cache-content.test.ts +++ b/test-integration/node/cache-content.test.ts @@ -29,14 +29,9 @@ use(chaiAsPromised); describe("cacheContent", function () { this.timeout(60e3); this.slow(10e3); - const model = "models/gemini-1.5-pro-001"; - let text: string = ""; + const model = "gemini-1.5-flash-002"; + const text = "Purple cats drink chicken soup."; - // Minimum cache size is 32768 tokens. - for (let i = 0; i < 6554; i++) { - text += "Purple cats drink chicken soup."; - text += i % 8 === 7 ? "\n" : " "; - } it("createCache", async () => { // cacheManager create const ttlSeconds = 5;