wrap pipeline query into retriable

src/gitlab.ts

@@ -11,7 +11,7 @@ import { createCiConfig } from "src/gitlab/createCiConfig";
 import { CommandRunner } from "src/shell";
 import { Task, taskExecutionTerminationEvent, TaskGitlabPipeline } from "src/task";
 import { Context } from "src/types";
-import { millisecondsDelay, validatedFetch } from "src/utils";
+import { millisecondsDelay, retriable, validatedFetch } from "src/utils";
 
 type GitlabTaskContext = TaskGitlabPipeline & {
   terminate: () => Promise<Error | undefined>;
@@ -122,9 +122,6 @@ export const runCommandInGitlabPipeline = async (ctx: Context, task: Task): Prom
 
   const branchPresenceUrl = `${gitlabProjectApi}/repository/branches/${branchNameUrlEncoded}`;
 
-  // add small preventive delay, as checking right-away most probably would cause a retry
-  await millisecondsDelay(waitForBranchRetryDelayMs);
-
   for (let waitForBranchTryCount = 0; waitForBranchTryCount < waitForBranchMaxTries; waitForBranchTryCount++) {
     logger.debug({ branchPresenceUrl }, `Sending request to see if the branch for task ${task.id} is ready`);
     const response = await fetch(branchPresenceUrl, { headers: { "PRIVATE-TOKEN": gitlab.accessToken } });
@@ -152,16 +149,23 @@ export const runCommandInGitlabPipeline = async (ctx: Context, task: Task): Prom
     );
   }
 
+  // add small preventive delay, as checking right-away most probably would cause a retry
+  await millisecondsDelay(waitForBranchRetryDelayMs);
+
   const pipelineCreationUrl = `${gitlabProjectApi}/pipeline?ref=${branchNameUrlEncoded}`;
   logger.debug({ pipelineCreationUrl, task }, `Sending request to create a pipeline for task ${task.id}`);
-  const pipeline = await validatedFetch<{
-    id: number;
-    project_id: number;
-  }>(
-    fetch(pipelineCreationUrl, { method: "POST", headers: { "PRIVATE-TOKEN": gitlab.accessToken } }),
-    Joi.object()
-      .keys({ id: Joi.number().required(), project_id: Joi.number().required() })
-      .options({ allowUnknown: true }),
+
+  const pipeline = await retriable(
+    async () =>
+      await validatedFetch<{
+        id: number;
+        project_id: number;
+      }>(
+        fetch(pipelineCreationUrl, { method: "POST", headers: { "PRIVATE-TOKEN": gitlab.accessToken } }),
+        Joi.object()
+          .keys({ id: Joi.number().required(), project_id: Joi.number().required() })
+          .options({ allowUnknown: true }),
+      ),
   );
 
   logger.info({ pipeline, task }, `Created pipeline for task ${task.id}`);

src/utils.spec.ts

@@ -0,0 +1,36 @@
+import { retriable } from "src/utils";
+
+describe("retriable", () => {
+  const wrapRetriable = async (fakeRetryCount: number) =>
+    await retriable(
+      async () =>
+        await new Promise((resolve, reject) => {
+          if (fakeRetryCount > 0) {
+            fakeRetryCount--;
+            reject(0);
+          } else {
+            resolve(1);
+          }
+        }),
+      { timeoutMs: 100, attempts: 3 },
+    );
+
+  test("resolve after 1/3 times", async () => {
+    const res = await wrapRetriable(1);
+    expect(res).toBe(1);
+  });
+
+  test("resolve after 2/3 times", async () => {
+    const res = await wrapRetriable(2);
+
+    expect(res).toBe(1);
+  });
+
+  test("reject after 3/3 times", async () => {
+    try {
+      await wrapRetriable(3);
+    } catch (e) {
+      expect(e).toEqual(0);
+    }
+  });
+});

src/utils.ts

@@ -110,3 +110,32 @@ export const arrayify = <T>(value: undefined | null | T | T[]): T[] => {
   }
   return [value];
 };
+
+export type RetriableConfig = {
+  attempts: number;
+  timeoutMs: number;
+};
+
+/** @throws Error */
+export const retriable = async <T>(
+  callback: () => Promise<T>,
+  options: RetriableConfig = { attempts: 3, timeoutMs: 2000 },
+): Promise<T> => {
+  let { attempts } = options;
+  const { timeoutMs } = options;
+
+  for (attempts; attempts > 0; attempts--) {
+    try {
+      return await callback();
+    } catch (e) {
+      await millisecondsDelay(timeoutMs);
+
+      // last failed attempt
+      if (attempts === 1) {
+        throw e;
+      }
+    }
+  }
+
+  throw Error(`Couldn't resolve a promise after ${options.attempts} attempts with ${options.timeoutMs}ms timeout`);
+};