Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on:
push:
branches:
- 'main'
- 'abhipatel12/fix-e2e-windows-docker'
# This will run for PRs from the base repository, providing secrets.
pull_request:
branches:
Expand All @@ -16,7 +17,7 @@ on:

jobs:
e2e-test:
name: 'E2E Test (${{ matrix.os }}) - ${{ matrix.sandbox }}'
name: 'E2E Test (${{ matrix.os }}) - sandbox:${{ matrix.sandbox }}'
# This condition ensures the job runs for pushes to main, merge groups,
# PRs from the base repo, OR PRs from forks with the correct label.
if: |
Expand All @@ -33,16 +34,16 @@ jobs:
- 'macos-latest'
- 'gemini-cli-windows-16-core'
sandbox:
- 'sandbox:none'
- 'sandbox:docker'
- 'none'
- 'docker'
node-version:
- '20.x'
exclude:
# Docker tests are not supported on macOS or Windows
- os: 'macos-latest'
sandbox: 'sandbox:docker'
sandbox: 'docker'
- os: 'gemini-cli-windows-16-core'
sandbox: 'sandbox:docker'
sandbox: 'docker'

steps:
- name: 'Checkout (fork)'
Expand Down Expand Up @@ -72,14 +73,13 @@ jobs:

- name: 'Set up Docker'
if: |-
matrix.os == 'ubuntu-latest' && matrix.sandbox == 'sandbox:docker'
matrix.os == 'ubuntu-latest' && matrix.sandbox == 'docker'
uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3

- name: 'Run E2E tests'
env:
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
KEEP_OUTPUT: 'true'
SANDBOX: '${{ matrix.sandbox }}'
VERBOSE: 'true'
run: |-
npm run "test:integration:${SANDBOX}"
npm run "test:integration:sandbox:${{ matrix.sandbox }}"
8 changes: 7 additions & 1 deletion integration-tests/read_many_files.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ describe('read_many_files', () => {

const prompt = `Use the read_many_files tool to read the contents of file1.txt and file2.txt and then print the contents of each file.`;

const result = await rig.run(prompt);
let result;
try {
result = await rig.run(prompt);
} catch (e) {
console.error('Error running test rig:', e);
throw e;
}

// Check for either read_many_files or multiple read_file calls
const allTools = rig.readToolLogs();
Expand Down
55 changes: 29 additions & 26 deletions integration-tests/test-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
*/

import { execSync, spawn } from 'node:child_process';
import { parse } from 'shell-quote';
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { EOL } from 'node:os';
import { join, dirname, normalize } from 'node:path';
import { fileURLToPath } from 'node:url';
import { env } from 'node:process';
import fs from 'node:fs';
Expand Down Expand Up @@ -102,7 +102,7 @@ export function validateModelOutput(
'The tool was called successfully, which is the main requirement.',
);
return false;
} else if (process.env.VERBOSE === 'true') {
} else if (process.env['VERBOSE'] === 'true') {
console.log(`${testName}: Model output validated successfully.`);
}
return true;
Expand All @@ -118,14 +118,14 @@ export class TestRig {
_lastRunStdout?: string;

constructor() {
this.bundlePath = join(__dirname, '..', 'bundle/gemini.js');
this.bundlePath = normalize(join(__dirname, '..', 'bundle/gemini.js'));
this.testDir = null;
}

// Get timeout based on environment
getDefaultTimeout() {
if (env.CI) return 60000; // 1 minute in CI
if (env.GEMINI_SANDBOX) return 30000; // 30s in containers
if (env['CI']) return 60000; // 1 minute in CI
if (env['GEMINI_SANDBOX']) return 30000; // 30s in containers
return 15000; // 15s locally
}

Expand All @@ -135,7 +135,7 @@ export class TestRig {
) {
this.testName = testName;
const sanitizedName = sanitizeTestName(testName);
this.testDir = join(env.INTEGRATION_TEST_FILE_DIR!, sanitizedName);
this.testDir = join(env['INTEGRATION_TEST_FILE_DIR']!, sanitizedName);
mkdirSync(this.testDir, { recursive: true });

// Create a settings file to point the CLI to the local collector
Expand All @@ -152,7 +152,8 @@ export class TestRig {
otlpEndpoint: '',
outfile: telemetryPath,
},
sandbox: env.GEMINI_SANDBOX !== 'false' ? env.GEMINI_SANDBOX : false,
sandbox:
env['GEMINI_SANDBOX'] !== 'false' ? env['GEMINI_SANDBOX'] : false,
...options.settings, // Allow tests to override/add settings
};
writeFileSync(
Expand Down Expand Up @@ -182,7 +183,7 @@ export class TestRig {
| { prompt?: string; stdin?: string; stdinDoesNotEnd?: boolean },
...args: string[]
): Promise<string> {
let command = `node ${this.bundlePath} --yolo`;
const commandArgs = [this.bundlePath, '--yolo'];
const execOptions: {
cwd: string;
encoding: 'utf-8';
Expand All @@ -193,25 +194,22 @@ export class TestRig {
};

if (typeof promptOrOptions === 'string') {
command += ` --prompt ${JSON.stringify(promptOrOptions)}`;
commandArgs.push('--prompt', promptOrOptions);
} else if (
typeof promptOrOptions === 'object' &&
promptOrOptions !== null
) {
if (promptOrOptions.prompt) {
command += ` --prompt ${JSON.stringify(promptOrOptions.prompt)}`;
commandArgs.push('--prompt', promptOrOptions.prompt);
}
if (promptOrOptions.stdin) {
execOptions.input = promptOrOptions.stdin;
}
}

command += ` ${args.join(' ')}`;
commandArgs.push(...args);

const commandArgs = parse(command);
const node = commandArgs.shift() as string;

const child = spawn(node, commandArgs as string[], {
const child = spawn('node', commandArgs, {
cwd: this.testDir!,
stdio: 'pipe',
});
Expand All @@ -233,28 +231,29 @@ export class TestRig {

child.stdout!.on('data', (data: Buffer) => {
stdout += data;
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
if (env['KEEP_OUTPUT'] === 'true' || env['VERBOSE'] === 'true') {
process.stdout.write(data);
}
});

child.stderr!.on('data', (data: Buffer) => {
stderr += data;
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
if (env['KEEP_OUTPUT'] === 'true' || env['VERBOSE'] === 'true') {
process.stderr.write(data);
}
});

const promise = new Promise<string>((resolve, reject) => {
child.on('close', (code: number) => {
console.log(`Child process exited with code ${code}`);
if (code === 0) {
// Store the raw stdout for Podman telemetry parsing
this._lastRunStdout = stdout;

// Filter out telemetry output when running with Podman
// Podman seems to output telemetry to stdout even when writing to file
let result = stdout;
if (env.GEMINI_SANDBOX === 'podman') {
if (env['GEMINI_SANDBOX'] === 'podman') {
// Remove telemetry JSON objects from output
// They are multi-line JSON objects that start with { and contain telemetry fields
const lines = result.split(EOL);
Expand Down Expand Up @@ -300,13 +299,17 @@ export class TestRig {
});
});

child.on('error', (err) => {
console.error('Failed to start child process.', err);
});

return promise;
}

readFile(fileName: string) {
const filePath = join(this.testDir!, fileName);
const content = readFileSync(filePath, 'utf-8');
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
if (env['KEEP_OUTPUT'] === 'true' || env['VERBOSE'] === 'true') {
console.log(`--- FILE: ${filePath} ---`);
console.log(content);
console.log(`--- END FILE: ${filePath} ---`);
Expand All @@ -316,12 +319,12 @@ export class TestRig {

async cleanup() {
// Clean up test directory
if (this.testDir && !env.KEEP_OUTPUT) {
if (this.testDir && !env['KEEP_OUTPUT']) {
try {
execSync(`rm -rf ${this.testDir}`);
} catch (error) {
// Ignore cleanup errors
if (env.VERBOSE === 'true') {
if (env['VERBOSE'] === 'true') {
console.warn('Cleanup warning:', (error as Error).message);
}
}
Expand Down Expand Up @@ -447,7 +450,7 @@ export class TestRig {
while (Date.now() - startTime < timeout) {
attempts++;
const result = predicate();
if (env.VERBOSE === 'true' && attempts % 5 === 0) {
if (env['VERBOSE'] === 'true' && attempts % 5 === 0) {
console.log(
`Poll attempt ${attempts}: ${result ? 'success' : 'waiting...'}`,
);
Expand All @@ -457,7 +460,7 @@ export class TestRig {
}
await new Promise((resolve) => setTimeout(resolve, interval));
}
if (env.VERBOSE === 'true') {
if (env['VERBOSE'] === 'true') {
console.log(`Poll timed out after ${attempts} attempts`);
}
return false;
Expand Down Expand Up @@ -590,7 +593,7 @@ export class TestRig {
readToolLogs() {
// For Podman, first check if telemetry file exists and has content
// If not, fall back to parsing from stdout
if (env.GEMINI_SANDBOX === 'podman') {
if (env['GEMINI_SANDBOX'] === 'podman') {
// Try reading from file first
const logFilePath = join(this.testDir!, 'telemetry.log');

Expand Down Expand Up @@ -672,7 +675,7 @@ export class TestRig {
}
} catch (e) {
// Skip objects that aren't valid JSON
if (env.VERBOSE === 'true') {
if (env['VERBOSE'] === 'true') {
console.error('Failed to parse telemetry object:', e);
}
}
Expand Down
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
"test:scripts": "vitest run --config ./scripts/tests/vitest.config.ts",
"test:e2e": "cross-env VERBOSE=true KEEP_OUTPUT=true npm run test:integration:sandbox:none",
"test:integration:all": "npm run test:integration:sandbox:none && npm run test:integration:sandbox:docker && npm run test:integration:sandbox:podman",
"test:integration:sandbox:none": "GEMINI_SANDBOX=false vitest run --root ./integration-tests",
"test:integration:sandbox:docker": "GEMINI_SANDBOX=docker npm run build:sandbox && GEMINI_SANDBOX=docker vitest run --root ./integration-tests",
"test:integration:sandbox:podman": "GEMINI_SANDBOX=podman vitest run --root ./integration-tests",
"test:integration:sandbox:none": "cross-env GEMINI_SANDBOX=false vitest run --root ./integration-tests",
"test:integration:sandbox:docker": "cross-env GEMINI_SANDBOX=docker npm run build:sandbox && cross-env GEMINI_SANDBOX=docker vitest run --root ./integration-tests",
"test:integration:sandbox:podman": "cross-env GEMINI_SANDBOX=podman vitest run --root ./integration-tests",
"lint": "eslint . --ext .ts,.tsx && eslint integration-tests",
"lint:fix": "eslint . --fix && eslint integration-tests --fix",
"lint:ci": "eslint . --ext .ts,.tsx --max-warnings 0 && eslint integration-tests --max-warnings 0",
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/ide/detect-ide.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ function verifyVSCode(
if (ide !== DetectedIde.VSCode) {
return ide;
}
if (ideProcessInfo.command.toLowerCase().includes('code')) {
if (ideProcessInfo.command?.toLowerCase().includes('code')) {
return DetectedIde.VSCode;
}
return DetectedIde.VSCodeFork;
Expand Down