add better visualization for connection/repo errors and warnings (#201)

* replace upsert with seperate create many and raw update many calls

* add bulk repo status update and queue addition with priority

* add support for managed redis

* add note for changing raw sql on schema change

* add error package and use BackendException in connection manager

* handle connection failure display on web app

* add warning banner for not found orgs/repos/users

* add failure handling for gerrit

* add gitea notfound warning support

* add warning icon in connections list

* style nits

* add failed repo vis in connections list

* added retry failed repo index buttons

* move nav indicators to client with polling

* fix indicator flash issue and truncate large list results

* display error nav better

* truncate failed repo list in connection list item

* fix merge error

* fix merge bug

* add connection util file [wip]

* refactor notfound fetch logic and add missing error package to dockerfile

* move repeated logic to function and add zod schema for syncStatusMetadata

Author: msukkari

Dockerfile

@@ -18,9 +18,11 @@ COPY package.json yarn.lock* ./
 COPY ./packages/db ./packages/db
 COPY ./packages/schemas ./packages/schemas
 COPY ./packages/crypto ./packages/crypto
+COPY ./packages/error ./packages/error
 RUN yarn workspace @sourcebot/db install --frozen-lockfile
 RUN yarn workspace @sourcebot/schemas install --frozen-lockfile
 RUN yarn workspace @sourcebot/crypto install --frozen-lockfile
+RUN yarn workspace @sourcebot/error install --frozen-lockfile
 
 # ------ Build Web ------
 FROM node-alpine AS web-builder
@@ -33,6 +35,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules
 COPY --from=shared-libs-builder /app/packages/db ./packages/db
 COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
+COPY --from=shared-libs-builder /app/packages/error ./packages/error
 
 # Fixes arm64 timeouts
 RUN yarn config set registry https://registry.npmjs.org/
@@ -66,6 +69,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules
 COPY --from=shared-libs-builder /app/packages/db ./packages/db
 COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
+COPY --from=shared-libs-builder /app/packages/error ./packages/error
 RUN yarn workspace @sourcebot/backend install --frozen-lockfile
 RUN yarn workspace @sourcebot/backend build
 
@@ -138,6 +142,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules
 COPY --from=shared-libs-builder /app/packages/db ./packages/db
 COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
+COPY --from=shared-libs-builder /app/packages/error ./packages/error
 
 # Configure the database
 RUN mkdir -p /run/postgresql && \

Makefile

@@ -26,6 +26,8 @@ clean:
 		packages/schemas/dist \
 		packages/crypto/node_modules \
 		packages/crypto/dist \
+		packages/error/node_modules \
+		packages/error/dist \
 		.sourcebot
 
 soft-reset:

packages/backend/package.json

@@ -34,6 +34,7 @@
         "@sourcebot/crypto": "^0.1.0",
         "@sourcebot/db": "^0.1.0",
         "@sourcebot/schemas": "^0.1.0",
+        "@sourcebot/error": "^0.1.0",
         "simple-git": "^3.27.0",
         "strip-json-comments": "^5.0.1",
         "winston": "^3.15.0",

packages/backend/src/connectionManager.ts

@@ -6,6 +6,7 @@ import { createLogger } from "./logger.js";
 import os from 'os';
 import { Redis } from 'ioredis';
 import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js";
+import { BackendError, BackendException } from "@sourcebot/error";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -81,26 +82,93 @@ export class ConnectionManager implements IConnectionManager {
         // @note: We aren't actually doing anything with this atm.
         const abortController = new AbortController();
 
-        const repoData: RepoData[] = await (async () => {
-            switch (config.type) {
-                case 'github': {
-                    return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
-                }
-                case 'gitlab': {
-                    return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
-                }
-                case 'gitea': {
-                    return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
-                }
-                case 'gerrit': {
-                    return await compileGerritConfig(config, job.data.connectionId, orgId);
-                }
-                default: {
-                    return [];
+        const connection = await this.db.connection.findUnique({
+            where: {
+                id: job.data.connectionId,
+            },
+        });
+
+        if (!connection) {
+            throw new BackendException(BackendError.CONNECTION_SYNC_CONNECTION_NOT_FOUND, {
+                message: `Connection ${job.data.connectionId} not found`,
+            });
+        }
+        
+        // Reset the syncStatusMetadata to an empty object at the start of the sync job
+        await this.db.connection.update({
+            where: {
+                id: job.data.connectionId,
+            },
+            data: {
+                syncStatusMetadata: {}
+            }
+        })
+        
+
+        let result: {
+            repoData: RepoData[],
+            notFound: {
+                users: string[],
+                orgs: string[],
+                repos: string[],
+            }
+        } = {
+            repoData: [],
+            notFound: {
+                users: [],
+                orgs: [],
+                repos: [],
+            }
+        };
+
+        try {
+            result = await (async () => {
+                switch (config.type) {
+                    case 'github': {
+                        return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
+                    }
+                    case 'gitlab': {
+                        return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
+                    }
+                    case 'gitea': {
+                        return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
+                    }
+                    case 'gerrit': {
+                        return await compileGerritConfig(config, job.data.connectionId, orgId);
+                    }
+                    default: {
+                        return {repoData: [], notFound: {
+                            users: [],
+                            orgs: [],
+                            repos: [],
+                        }};
+                    }
                 }
+            })();
+        } catch (err) {
+            this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`);
+            if (err instanceof BackendException) {
+                throw err;
+            } else {
+                throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
+                    message: `Failed to compile repo data for connection ${job.data.connectionId}`,
+                });
             }
-        })();
+        }
+
+        const { repoData, notFound } = result;
 
+        // Push the information regarding not found users, orgs, and repos to the connection's syncStatusMetadata. Note that 
+        // this won't be overwritten even if the connection job fails
+        await this.db.connection.update({
+            where: {
+                id: job.data.connectionId,
+            },
+            data: {
+                syncStatusMetadata: { notFound }
+            }
+        });
+            
         // Filter out any duplicates by external_id and external_codeHostUrl.
         repoData.filter((repo, index, self) => {
             return index === self.findIndex(r =>
@@ -265,16 +333,37 @@ export class ConnectionManager implements IConnectionManager {
     private async onSyncJobFailed(job: Job | undefined, err: unknown) {
         this.logger.info(`Connection sync job failed with error: ${err}`);
         if (job) {
+
+            // We may have pushed some metadata during the execution of the job, so we make sure to not overwrite the metadata here
             const { connectionId } = job.data;
+            let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
+                where: { id: connectionId },
+                select: { syncStatusMetadata: true }
+            }))?.syncStatusMetadata as Record<string, unknown> ?? {};
+
+            if (err instanceof BackendException) {
+                syncStatusMetadata = {
+                    ...syncStatusMetadata,
+                    error: err.code,
+                    ...err.metadata,
+                }
+            } else {
+                syncStatusMetadata = {
+                    ...syncStatusMetadata, 
+                    error: 'UNKNOWN',
+                }
+            }
+
             await this.db.connection.update({
                 where: {
                     id: connectionId,
                 },
                 data: {
                     syncStatus: ConnectionSyncStatus.FAILED,
-                    syncedAt: new Date()
+                    syncedAt: new Date(),
+                    syncStatusMetadata: syncStatusMetadata as Prisma.InputJsonValue,
                 }
-            })
+            });
         }
     }
 

packages/backend/src/connectionUtils.ts

@@ -0,0 +1,44 @@
+type ValidResult<T> = {
+    type: 'valid';
+    data: T[];
+};
+
+type NotFoundResult = {
+    type: 'notFound';
+    value: string;
+};
+
+type CustomResult<T> = ValidResult<T> | NotFoundResult;
+
+export function processPromiseResults<T>(
+    results: PromiseSettledResult<CustomResult<T>>[],
+): {
+    validItems: T[];
+    notFoundItems: string[];
+} {
+    const validItems: T[] = [];
+    const notFoundItems: string[] = [];
+
+    results.forEach(result => {
+        if (result.status === 'fulfilled') {
+            const value = result.value;
+            if (value.type === 'valid') {
+                validItems.push(...value.data);
+            } else {
+                notFoundItems.push(value.value);
+            }
+        }
+    });
+
+    return {
+        validItems,
+        notFoundItems,
+    };
+}
+
+export function throwIfAnyFailed<T>(results: PromiseSettledResult<T>[]) {
+    const failedResult = results.find(result => result.status === 'rejected');
+    if (failedResult) {
+        throw failedResult.reason;
+    }
+}

packages/backend/src/gerrit.ts

@@ -3,6 +3,8 @@ import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
 import { createLogger } from './logger.js';
 import micromatch from "micromatch";
 import { measure, marshalBool, excludeReposByName, includeReposByName, fetchWithRetry } from './utils.js';
+import { BackendError } from '@sourcebot/error';
+import { BackendException } from '@sourcebot/error';
 
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html
 interface GerritProjects {
@@ -38,6 +40,10 @@ export const getGerritReposFromConfig = async (config: GerritConfig): Promise<Ge
          const fetchFn = () => fetchAllProjects(url);
          return fetchWithRetry(fetchFn, `projects from ${url}`, logger);
       } catch (err) {
+         if (err instanceof BackendException) {
+            throw err;
+         }
+
          logger.error(`Failed to fetch projects from ${url}`, err);
          return null;
       }
@@ -78,9 +84,25 @@ const fetchAllProjects = async (url: string): Promise<GerritProject[]> => {
       const endpointWithParams = `${projectsEndpoint}?S=${start}`;
       logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`);
 
-      const response = await fetch(endpointWithParams);
-      if (!response.ok) {
-         throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
+      let response: Response;
+      try {
+         response = await fetch(endpointWithParams);
+         if (!response.ok) {
+            console.log(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`);
+            throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, {
+               status: response.status,
+            });
+         }
+      } catch (err) {
+         if (err instanceof BackendException) {
+            throw err;
+         }
+
+         const status = (err as any).code;
+         console.log(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${status}`);
+         throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, {
+            status: status,
+         });
       }
 
       const text = await response.text();