diff --git a/src/gitlab.ts b/src/gitlab.ts index 691ca76..707f15a 100644 --- a/src/gitlab.ts +++ b/src/gitlab.ts @@ -11,7 +11,7 @@ import { createCiConfig } from "src/gitlab/createCiConfig"; import { CommandRunner } from "src/shell"; import { Task, taskExecutionTerminationEvent, TaskGitlabPipeline } from "src/task"; import { Context } from "src/types"; -import { millisecondsDelay, validatedFetch } from "src/utils"; +import { millisecondsDelay, retriable, validatedFetch } from "src/utils"; type GitlabTaskContext = TaskGitlabPipeline & { terminate: () => Promise; @@ -122,9 +122,6 @@ export const runCommandInGitlabPipeline = async (ctx: Context, task: Task): Prom const branchPresenceUrl = `${gitlabProjectApi}/repository/branches/${branchNameUrlEncoded}`; - // add small preventive delay, as checking right-away most probably would cause a retry - await millisecondsDelay(waitForBranchRetryDelayMs); - for (let waitForBranchTryCount = 0; waitForBranchTryCount < waitForBranchMaxTries; waitForBranchTryCount++) { logger.debug({ branchPresenceUrl }, `Sending request to see if the branch for task ${task.id} is ready`); const response = await fetch(branchPresenceUrl, { headers: { "PRIVATE-TOKEN": gitlab.accessToken } }); @@ -152,16 +149,23 @@ export const runCommandInGitlabPipeline = async (ctx: Context, task: Task): Prom ); } + // add small preventive delay, as checking right-away most probably would cause a retry + await millisecondsDelay(waitForBranchRetryDelayMs); + const pipelineCreationUrl = `${gitlabProjectApi}/pipeline?ref=${branchNameUrlEncoded}`; logger.debug({ pipelineCreationUrl, task }, `Sending request to create a pipeline for task ${task.id}`); - const pipeline = await validatedFetch<{ - id: number; - project_id: number; - }>( - fetch(pipelineCreationUrl, { method: "POST", headers: { "PRIVATE-TOKEN": gitlab.accessToken } }), - Joi.object() - .keys({ id: Joi.number().required(), project_id: Joi.number().required() }) - .options({ allowUnknown: true }), + + const pipeline = await retriable( + async () => + await validatedFetch<{ + id: number; + project_id: number; + }>( + fetch(pipelineCreationUrl, { method: "POST", headers: { "PRIVATE-TOKEN": gitlab.accessToken } }), + Joi.object() + .keys({ id: Joi.number().required(), project_id: Joi.number().required() }) + .options({ allowUnknown: true }), + ), ); logger.info({ pipeline, task }, `Created pipeline for task ${task.id}`); diff --git a/src/utils.spec.ts b/src/utils.spec.ts new file mode 100644 index 0000000..7e78493 --- /dev/null +++ b/src/utils.spec.ts @@ -0,0 +1,36 @@ +import { retriable } from "src/utils"; + +describe("retriable", () => { + const wrapRetriable = async (fakeRetryCount: number) => + await retriable( + async () => + await new Promise((resolve, reject) => { + if (fakeRetryCount > 0) { + fakeRetryCount--; + reject(0); + } else { + resolve(1); + } + }), + { timeoutMs: 100, attempts: 3 }, + ); + + test("resolve after 1/3 times", async () => { + const res = await wrapRetriable(1); + expect(res).toBe(1); + }); + + test("resolve after 2/3 times", async () => { + const res = await wrapRetriable(2); + + expect(res).toBe(1); + }); + + test("reject after 3/3 times", async () => { + try { + await wrapRetriable(3); + } catch (e) { + expect(e).toEqual(0); + } + }); +}); diff --git a/src/utils.ts b/src/utils.ts index 564a6ed..e3c5538 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -110,3 +110,32 @@ export const arrayify = (value: undefined | null | T | T[]): T[] => { } return [value]; }; + +export type RetriableConfig = { + attempts: number; + timeoutMs: number; +}; + +/** @throws Error */ +export const retriable = async ( + callback: () => Promise, + options: RetriableConfig = { attempts: 3, timeoutMs: 2000 }, +): Promise => { + let { attempts } = options; + const { timeoutMs } = options; + + for (attempts; attempts > 0; attempts--) { + try { + return await callback(); + } catch (e) { + await millisecondsDelay(timeoutMs); + + // last failed attempt + if (attempts === 1) { + throw e; + } + } + } + + throw Error(`Couldn't resolve a promise after ${options.attempts} attempts with ${options.timeoutMs}ms timeout`); +};