Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 66 additions & 4 deletions packages/core/src/utils/googleQuotaErrors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,21 +134,21 @@ describe('classifyGoogleError', () => {
expect((result as TerminalQuotaError).cause).toBe(apiError);
});

it('should return RetryableQuotaError for long retry delays', () => {
it('should return TerminalQuotaError for retry delays over 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message: 'Too many requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '301s', // Any delay is now retryable
retryDelay: '301s', // Over 5 min threshold => terminal
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(301000);
expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as TerminalQuotaError).retryDelayMs).toBe(301000);
});

it('should return RetryableQuotaError for short retry delays', () => {
Expand Down Expand Up @@ -285,6 +285,34 @@ describe('classifyGoogleError', () => {
);
});

it('should return TerminalQuotaError for Cloud Code RATE_LIMIT_EXCEEDED with retry delay over 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message:
'You have exhausted your capacity on this model. Your quota will reset after 10m.',
details: [
{
'@type': 'type.googleapis.com/google.rpc.ErrorInfo',
reason: 'RATE_LIMIT_EXCEEDED',
domain: 'cloudcode-pa.googleapis.com',
metadata: {
uiMessage: 'true',
model: 'gemini-2.5-pro',
},
},
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '600s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as TerminalQuotaError).retryDelayMs).toBe(600000);
expect((result as TerminalQuotaError).reason).toBe('RATE_LIMIT_EXCEEDED');
});

it('should return TerminalQuotaError for Cloud Code QUOTA_EXHAUSTED', () => {
const apiError: GoogleApiError = {
code: 429,
Expand Down Expand Up @@ -427,6 +455,40 @@ describe('classifyGoogleError', () => {
}
});

it('should return TerminalQuotaError when fallback "Please retry in" delay exceeds 5 minutes', () => {
const errorWithEmptyDetails = {
error: {
code: 429,
message: 'Resource exhausted. Please retry in 400s',
details: [],
},
};

const result = classifyGoogleError(errorWithEmptyDetails);

expect(result).toBeInstanceOf(TerminalQuotaError);
if (result instanceof TerminalQuotaError) {
expect(result.retryDelayMs).toBe(400000);
}
});

it('should return RetryableQuotaError when retry delay is exactly 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message: 'Too many requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '300s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(300000);
});

it('should return RetryableQuotaError without delay time for generic 429 without specific message', () => {
const generic429 = {
status: 429,
Expand Down
43 changes: 33 additions & 10 deletions packages/core/src/utils/googleQuotaErrors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@ function parseDurationInSeconds(duration: string): number | null {
return null;
}

/**
* Maximum retry delay (in seconds) before a retryable error is treated as terminal.
* If the server suggests waiting longer than this, the user is effectively locked out,
* so we trigger the fallback/credits flow instead of silently waiting.
*/
const MAX_RETRYABLE_DELAY_SECONDS = 300; // 5 minutes

/**
* Valid Cloud Code API domains for VALIDATION_REQUIRED errors.
*/
Expand Down Expand Up @@ -248,15 +255,15 @@ export function classifyGoogleError(error: unknown): unknown {
if (match?.[1]) {
const retryDelaySeconds = parseDurationInSeconds(match[1]);
if (retryDelaySeconds !== null) {
return new RetryableQuotaError(
errorMessage,
googleApiError ?? {
code: status ?? 429,
message: errorMessage,
details: [],
},
retryDelaySeconds,
);
const cause = googleApiError ?? {
code: status ?? 429,
message: errorMessage,
details: [],
};
if (retryDelaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(errorMessage, cause, retryDelaySeconds);
}
return new RetryableQuotaError(errorMessage, cause, retryDelaySeconds);
}
} else if (status === 429 || status === 499) {
// Fallback: If it is a 429 or 499 but doesn't have a specific "retry in" message,
Expand Down Expand Up @@ -325,10 +332,19 @@ export function classifyGoogleError(error: unknown): unknown {
if (errorInfo.domain) {
if (isCloudCodeDomain(errorInfo.domain)) {
if (errorInfo.reason === 'RATE_LIMIT_EXCEEDED') {
const effectiveDelay = delaySeconds ?? 10;
if (effectiveDelay > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}`,
googleApiError,
effectiveDelay,
errorInfo.reason,
);
}
return new RetryableQuotaError(
`${googleApiError.message}`,
googleApiError,
delaySeconds ?? 10,
effectiveDelay,
);
}
if (errorInfo.reason === 'QUOTA_EXHAUSTED') {
Expand All @@ -345,6 +361,13 @@ export function classifyGoogleError(error: unknown): unknown {

// 2. Check for delays in RetryInfo
if (retryInfo?.retryDelay && delaySeconds) {
if (delaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError,
delaySeconds,
);
}
return new RetryableQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError,
Expand Down
Loading