Skip to content

Commit 54d14ea

Browse files
msukkarimsukkari
authored
add retries for 429 issues (#196)
* add connection compile retry and hard repo limit * add more retry checks * cleanup unused change * address feedback
1 parent 331a418 commit 54d14ea

14 files changed

Lines changed: 256 additions & 105 deletions

File tree

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,9 @@ clean:
2828
packages/crypto/dist \
2929
.sourcebot
3030

31+
soft-reset:
32+
rm -rf .sourcebot
33+
redis-cli FLUSHALL
34+
35+
3136
.PHONY: bin

‎demo-site-config.json‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
},
1919
"repos": [
2020
"torvalds/linux",
21-
"pytorch/pytorch",
21+
"pytorch/pytorch",
2222
"commaai/openpilot",
2323
"ggerganov/whisper.cpp",
2424
"ggerganov/llama.cpp",
@@ -42,7 +42,6 @@
4242
"TheAlgorithms/Python",
4343
"trekhleb/javascript-algorithms",
4444
"tensorflow/tensorflow",
45-
"torvalds/linux",
4645
"getify/You-Dont-Know-JS",
4746
"CyC2018/CS-Notes",
4847
"ohmyzsh/ohmyzsh",
@@ -106,10 +105,8 @@
106105
"Hack-with-Github/Awesome-Hacking",
107106
"nvbn/thefuck",
108107
"mtdvio/every-programmer-should-know",
109-
"pytorch/pytorch",
110108
"storybookjs/storybook",
111109
"neovim/neovim",
112-
"tailwindlabs/tailwindcss",
113110
"microsoft/Web-Dev-For-Beginners",
114111
"django/django",
115112
"florinpop17/app-ideas",
@@ -153,7 +150,6 @@
153150
"fighting41love/funNLP",
154151
"vitejs/vite",
155152
"thedaviddias/Front-End-Checklist",
156-
"ggerganov/llama.cpp",
157153
"coder/code-server",
158154
"moby/moby",
159155
"CompVis/stable-diffusion",

‎packages/backend/src/constants.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { Settings } from "./types.js";
66
export const DEFAULT_SETTINGS: Settings = {
77
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
88
autoDeleteStaleRepos: true,
9-
reindexIntervalMs: 1000 * 60,
9+
reindexIntervalMs: 1000 * 60 * 60, // 1 hour
1010
resyncConnectionPollingIntervalMs: 1000,
1111
reindexRepoPollingIntervalMs: 1000,
1212
indexConcurrencyMultiple: 3,

‎packages/backend/src/environment.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ export const getEnvBoolean = (env: string | undefined, defaultValue: boolean) =>
1818
dotenv.config({
1919
path: './.env',
2020
});
21+
dotenv.config({
22+
path: './.env.local',
23+
override: true
24+
});
2125

2226

2327
export const SOURCEBOT_LOG_LEVEL = getEnv(process.env.SOURCEBOT_LOG_LEVEL, 'info')!;
@@ -26,3 +30,7 @@ export const SOURCEBOT_INSTALL_ID = getEnv(process.env.SOURCEBOT_INSTALL_ID, 'un
2630
export const SOURCEBOT_VERSION = getEnv(process.env.SOURCEBOT_VERSION, 'unknown')!;
2731
export const POSTHOG_PAPIK = getEnv(process.env.POSTHOG_PAPIK);
2832
export const POSTHOG_HOST = getEnv(process.env.POSTHOG_HOST);
33+
34+
export const FALLBACK_GITHUB_TOKEN = getEnv(process.env.FALLBACK_GITHUB_TOKEN);
35+
export const FALLBACK_GITLAB_TOKEN = getEnv(process.env.FALLBACK_GITLAB_TOKEN);
36+
export const FALLBACK_GITEA_TOKEN = getEnv(process.env.FALLBACK_GITEA_TOKEN);

‎packages/backend/src/gerrit.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import fetch from 'cross-fetch';
22
import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
33
import { createLogger } from './logger.js';
44
import micromatch from "micromatch";
5-
import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
5+
import { measure, marshalBool, excludeReposByName, includeReposByName, fetchWithRetry } from './utils.js';
66

77
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
88
interface GerritProjects {
@@ -30,21 +30,21 @@ interface GerritWebLink {
3030
const logger = createLogger('Gerrit');
3131

3232
export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
33-
3433
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
3534
const hostname = new URL(config.url).hostname;
3635

3736
let { durationMs, data: projects } = await measure(async () => {
3837
try {
39-
return fetchAllProjects(url)
38+
const fetchFn = () => fetchAllProjects(url);
39+
return fetchWithRetry(fetchFn, `projects from ${url}`, logger);
4040
} catch (err) {
4141
logger.error(`Failed to fetch projects from ${url}`, err);
4242
return null;
4343
}
4444
});
4545

4646
if (!projects) {
47-
return [];
47+
throw new Error(`Failed to fetch projects from ${url}`);
4848
}
4949

5050
// exclude "All-Projects" and "All-Users" projects

‎packages/backend/src/git.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
9999
.filter(Boolean)
100100
.map(branch => branch.replace('refs/heads/', ''));
101101

102-
repo.branches = branches.filter(branch =>
102+
repo.branches = branches.filter(branch =>
103103
branchGlobs.some(glob => new RegExp(glob).test(branch))
104104
);
105105
}
@@ -114,7 +114,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
114114
.filter(Boolean)
115115
.map(tag => tag.replace('refs/tags/', ''));
116116

117-
repo.tags = tags.filter(tag =>
117+
repo.tags = tags.filter(tag =>
118118
tagGlobs.some(glob => new RegExp(glob).test(tag))
119119
);
120120
}

‎packages/backend/src/gitea.ts‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,47 @@
11
import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
22
import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
3-
import { getTokenFromConfig, measure } from './utils.js';
3+
import { getTokenFromConfig, measure, fetchWithRetry } from './utils.js';
44
import fetch from 'cross-fetch';
55
import { createLogger } from './logger.js';
66
import micromatch from 'micromatch';
77
import { PrismaClient } from '@sourcebot/db';
8-
8+
import { FALLBACK_GITEA_TOKEN } from './environment.js';
99
const logger = createLogger('Gitea');
1010

1111
export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
12-
// TODO: pass in DB here to fetch secret properly
1312
const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
1413

1514
const api = giteaApi(config.url ?? 'https://gitea.com', {
16-
token,
15+
token: token ?? FALLBACK_GITEA_TOKEN,
1716
customFetch: fetch,
1817
});
1918

2019
let allRepos: GiteaRepository[] = [];
2120

2221
if (config.orgs) {
23-
const _repos = await getReposForOrgs(config.orgs, api);
22+
const _repos = await fetchWithRetry(
23+
() => getReposForOrgs(config.orgs!, api),
24+
`orgs ${config.orgs.join(', ')}`,
25+
logger
26+
);
2427
allRepos = allRepos.concat(_repos);
2528
}
2629

2730
if (config.repos) {
28-
const _repos = await getRepos(config.repos, api);
31+
const _repos = await fetchWithRetry(
32+
() => getRepos(config.repos!, api),
33+
`repos ${config.repos.join(', ')}`,
34+
logger
35+
);
2936
allRepos = allRepos.concat(_repos);
3037
}
3138

3239
if (config.users) {
33-
const _repos = await getReposOwnedByUsers(config.users, api);
40+
const _repos = await fetchWithRetry(
41+
() => getReposOwnedByUsers(config.users!, api),
42+
`users ${config.users.join(', ')}`,
43+
logger
44+
);
3445
allRepos = allRepos.concat(_repos);
3546
}
3647

@@ -50,7 +61,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
5061
allRepos = await Promise.all(
5162
allRepos.map(async (repo) => {
5263
const [owner, name] = repo.full_name!.split('/');
53-
let branches = (await getBranchesForRepo(owner, name, api)).map(branch => branch.name!);
64+
let branches = (await fetchWithRetry(
65+
() => getBranchesForRepo(owner, name, api),
66+
`branches for ${owner}/${name}`,
67+
logger
68+
)).map(branch => branch.name!);
5469
branches = micromatch.match(branches, branchGlobs);
5570

5671
return {
@@ -66,7 +81,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
6681
allRepos = await Promise.all(
6782
allRepos.map(async (allRepos) => {
6883
const [owner, name] = allRepos.name!.split('/');
69-
let tags = (await getTagsForRepo(owner, name, api)).map(tag => tag.name!);
84+
let tags = (await fetchWithRetry(
85+
() => getTagsForRepo(owner, name, api),
86+
`tags for ${owner}/${name}`,
87+
logger
88+
)).map(tag => tag.name!);
7089
tags = micromatch.match(tags, tagGlobs);
7190

7291
return {
@@ -146,7 +165,7 @@ const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
146165
return tags;
147166
} catch (e) {
148167
logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
149-
return [];
168+
throw e;
150169
}
151170
}
152171

@@ -162,7 +181,7 @@ const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) =
162181
return branches;
163182
} catch (e) {
164183
logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
165-
return [];
184+
throw e;
166185
}
167186
}
168187

@@ -181,7 +200,7 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
181200
return data;
182201
} catch (e) {
183202
logger.error(`Failed to fetch repos for user ${user}.`, e);
184-
return [];
203+
throw e;
185204
}
186205
}))).flat();
187206

@@ -204,7 +223,7 @@ const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
204223
return data;
205224
} catch (e) {
206225
logger.error(`Failed to fetch repos for org ${org}.`, e);
207-
return [];
226+
throw e;
208227
}
209228
}))).flat();
210229
}
@@ -224,7 +243,7 @@ const getRepos = async <T>(repos: string[], api: Api<T>) => {
224243
return [response.data];
225244
} catch (e) {
226245
logger.error(`Failed to fetch repository info for ${repo}.`, e);
227-
return [];
246+
throw e;
228247
}
229248
}))).flat();
230249
}

0 commit comments

Comments
 (0)