fix: add circuit breakers to prevent re-queue infinite retry loop

functions/src/logic/buildQueue/scheduler.ts

@@ -85,6 +85,16 @@ export class Scheduler {
 
     // Schedule it
     if (['created', 'failed'].includes(job.status)) {
+      // Check retry limit before re-dispatching
+      if (CiJobs.hasExceededRetryLimit(job, settings.maxFailuresPerBuild)) {
+        const message =
+          `[Scheduler] Base image job ${jobId} has exceeded the maximum retry limit ` +
+          `(${settings.maxFailuresPerBuild}). Manual action is required.`;
+        logger.warn(message);
+        await Discord.sendAlert(message);
+        return false;
+      }
+
       const { repoVersionFull, repoVersionMinor, repoVersionMajor } = this;
       const response = await this.gitHub.repos.createDispatchEvent({
         owner: 'unity-ci',
@@ -126,6 +136,16 @@ export class Scheduler {
 
     // Schedule it
     if (['created', 'failed'].includes(job.status)) {
+      // Check retry limit before re-dispatching
+      if (CiJobs.hasExceededRetryLimit(job, settings.maxFailuresPerBuild)) {
+        const message =
+          `[Scheduler] Hub image job ${jobId} has exceeded the maximum retry limit ` +
+          `(${settings.maxFailuresPerBuild}). Manual action is required.`;
+        logger.warn(message);
+        await Discord.sendAlert(message);
+        return false;
+      }
+
       const { repoVersionFull, repoVersionMinor, repoVersionMajor } = this;
       const response = await this.gitHub.repos.createDispatchEvent({
         owner: 'unity-ci',

functions/src/model/ciBuilds.ts

@@ -135,14 +135,33 @@ export class CiBuilds {
 
     let result;
     if (snapshot.exists) {
-      // Builds can be retried after a failure.
-      if (snapshot.data()?.status === 'failed') {
-        // In case or reporting a new build during retry step, only overwrite these fields
+      const existingStatus = snapshot.data()?.status;
+      if (existingStatus === 'failed') {
+        // Builds can be retried after a failure.
+        // In case of reporting a new build during retry step, only overwrite these fields
         result = await ref.set(data, {
           mergeFields: ['status', 'meta.lastBuildStart', 'modifiedDate'],
         });
+      } else if (existingStatus === 'started') {
+        const existingJobId = snapshot.data()?.relatedJobId;
+        if (existingJobId !== relatedJobId) {
+          // Different job trying to register same build - this is unexpected and should fail
+          throw new Error(
+            `Build "${buildId}" already exists with jobId "${existingJobId}", ` +
+              `but received conflicting request from jobId "${relatedJobId}"`,
+          );
+        }
+        // Idempotent - same job retrying after network timeout, safe to skip
+        logger.info(`Build "${buildId}" already exists with status "started", skipping`);
+        return;
+      } else if (existingStatus === 'published') {
+        // Build is already done, nothing to do
+        logger.info(`Build "${buildId}" already published, skipping`);
+        return;
       } else {
-        throw new Error(`A build with "${buildId}" as identifier already exists`);
+        throw new Error(
+          `A build with "${buildId}" already exists with status "${existingStatus}"`,
+        );
       }
     } else {
       result = await ref.create(data);

functions/src/model/ciJobs.ts

@@ -205,7 +205,7 @@ export class CiJobs {
 
     // Do not override failure or completed
     let { status } = currentBuild;
-    if (['scheduled'].includes(status)) {
+    if (['created', 'scheduled'].includes(status)) {
       status = 'inProgress';
     }
 
@@ -227,6 +227,10 @@ export class CiJobs {
     });
   };
 
+  static hasExceededRetryLimit = (job: CiJob, maxRetries: number): boolean => {
+    return job.meta.failureCount >= maxRetries;
+  };
+
   static markJobAsCompleted = async (jobId: string) => {
     const job = await db.collection(CiJobs.collection).doc(jobId);