Skip to content

Make function calling tests to produce more deterministic results #343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hkt74 merged 11 commits into from
Feb 20, 2025
Merged

Make function calling tests to produce more deterministic results #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2e4df98
Make tool tests to produce more deterministic results
hkt74 Feb 20, 2025
3a7cf21
Make tool tests to produce more deterministic results
hkt74 Feb 20, 2025
07a1c61
Make tool tests to produce more deterministic results
hkt74 Feb 20, 2025
edac5ff
change model to 1.5 pro
hkt74 Feb 20, 2025
7407e81
change model to 1.5 pro
hkt74 Feb 20, 2025
678ba57
change model to 1.5 pro
hkt74 Feb 20, 2025
00ea1de
change model to 1.5 pro
hkt74 Feb 20, 2025
ddc777a
formatting
hkt74 Feb 20, 2025
e377fe9
increase test timeout
hkt74 Feb 20, 2025
85acbf6
revert timeout back to 1m
hkt74 Feb 20, 2025
1998367
reduce cache token size
hkt74 Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/lucky-grapes-rest.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@google/generative-ai": patch
---

Fix flaky integration test with tools
9 changes: 2 additions & 7 deletions test-integration/node/cache-content.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,9 @@ use(chaiAsPromised);
describe("cacheContent", function () {
this.timeout(60e3);
this.slow(10e3);
const model = "models/gemini-1.5-pro-001";
let text: string = "";
const model = "gemini-1.5-flash-002";
const text = "Purple cats drink chicken soup.";

// Minimum cache size is 32768 tokens.
for (let i = 0; i < 6554; i++) {
text += "Purple cats drink chicken soup.";
text += i % 8 === 7 ? "\n" : " ";
}
it("createCache", async () => {
// cacheManager create
const ttlSeconds = 5;
Expand Down
180 changes: 85 additions & 95 deletions test-integration/node/generate-content-tools.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,87 +33,81 @@ describe("generateContent - tools", function () {
// eslint-disable-next-line no-restricted-properties
it("non-streaming, tools usage", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel(
{
model: "gemini-1.5-flash-latest",
tools: [
{
functionDeclarations: [
{
name: "find_movies",
description:
"find movie titles currently playing in theaters based on any description, genre, title words, etc.",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description:
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
},
description: {
type: SchemaType.STRING,
description:
"Any kind of description including category or genre, title words, attributes, etc.",
},
const model = genAI.getGenerativeModel({
model: "gemini-1.5-pro-latest",
tools: [
{
functionDeclarations: [
{
name: "find_movies",
description:
"find movie titles currently playing in theaters based on any description, genre, title words, etc.",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description: "The city and state, e.g. San Francisco, CA",
},
description: {
type: SchemaType.STRING,
description:
"Any kind of description including category or genre, title words, attributes, etc.",
},
required: ["description"],
},
required: ["description"],
},
{
name: "find_theaters",
description:
"find theaters based on location and optionally movie title which are is currently playing in theaters",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description:
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
},
movie: {
type: SchemaType.STRING,
description: "Any movie title",
},
},
{
name: "find_theaters",
description:
"find theaters based on location and optionally movie title which are is currently playing in theaters",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description: "The city and state, e.g. San Francisco, CA",
},
movie: {
type: SchemaType.STRING,
description: "Any movie title",
},
required: ["location"],
},
required: ["location"],
},
{
name: "get_showtimes",
description:
"Find the start times for movies playing in a specific theater",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description:
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
},
movie: {
type: SchemaType.STRING,
description: "Any movie title",
},
theater: {
type: SchemaType.STRING,
description: "Name of the theater",
},
date: {
type: SchemaType.STRING,
description: "Date for requested showtime",
},
},
{
name: "get_showtimes",
description:
"Find the start times for movies playing in a specific theater",
parameters: {
type: SchemaType.OBJECT,
properties: {
location: {
type: SchemaType.STRING,
description: "The city and state, e.g. San Francisco, CA",
},
movie: {
type: SchemaType.STRING,
description: "Any movie title",
},
theater: {
type: SchemaType.STRING,
description: "Name of the theater",
},
date: {
type: SchemaType.STRING,
description: "Date for requested showtime",
},
required: ["location", "movie", "theater", "date"],
},
required: ["location", "movie", "theater", "date"],
},
],
},
],
},
{ apiVersion: "v1beta" },
);
},
],
},
],
});

const src1 = {
role: "user",
Expand Down Expand Up @@ -183,30 +177,27 @@ describe("generateContent - tools", function () {
});
it("streaming, tools usage", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel(
{
model: "gemini-1.5-flash-latest",
tools: [
{
functionDeclarations: [
{
name: "getTemperature",
description:
"Get current temperature in degrees Celsius in a given city",
parameters: {
type: SchemaType.OBJECT,
properties: {
city: { type: SchemaType.STRING },
},
required: ["city"],
const model = genAI.getGenerativeModel({
model: "gemini-1.5-pro-latest",
tools: [
{
functionDeclarations: [
{
name: "getTemperature",
description:
"Get current temperature in degrees Celsius in a given city",
parameters: {
type: SchemaType.OBJECT,
properties: {
city: { type: SchemaType.STRING },
},
required: ["city"],
},
],
},
],
},
{ apiVersion: "v1beta" },
);
},
],
},
],
});

const src1: Content = {
role: "user",
Expand Down Expand Up @@ -259,7 +250,6 @@ describe("generateContent - tools", function () {
contents: [src1, src2, fn1, src3, fn1],
});
const response = await result.response;
console.log(response.text());
expect(response.text()).to.match(/(\bsame\b|\byes\b)/i);
});
});
31 changes: 15 additions & 16 deletions test-integration/node/start-chat-tools.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ describe("startChat - tools", function () {
parameters: {
type: SchemaType.OBJECT,
properties: {
city: { type: SchemaType.STRING },
city: {
type: SchemaType.STRING,
description: "A city name, for example, San Francisco",
},
},
required: ["city"],
},
Expand Down Expand Up @@ -71,23 +74,19 @@ describe("startChat - tools", function () {
this.slow(10e3);
it("stream false", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel(
{
model: "gemini-1.5-flash-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
},
],
tools,
},
{ apiVersion: "v1beta" },
);
const model = genAI.getGenerativeModel({
model: "gemini-1.5-pro-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
},
],
tools,
});
const chat = model.startChat();
const result1 = await chat.sendMessage([part1]);
expect(result1.response.text()).to.be.empty;
expect(result1.response.functionCall()).not.to.be.empty;
expect(result1.response.functionCalls()).not.to.be.empty;
const result2 = await chat.sendMessage([part2]);
expect(result2.response.text()).to.not.be.empty;
const history = await chat.getHistory();
Expand Down
2 changes: 1 addition & 1 deletion test-integration/node/start-chat.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ describe("startChat", function () {
it("stream true, try to send message before previous stream is done", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
model: "gemini-1.5-flash-latest",
model: "gemini-1.5-pro-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
Expand Down