add back gitlab, gitea, and gerrit support

packages/backend/src/connectionManager.ts

@@ -5,8 +5,7 @@ import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
 import { createLogger } from "./logger.js";
 import os from 'os';
 import { Redis } from 'ioredis';
-import { marshalBool } from "./utils.js";
-import { getGitHubReposFromConfig } from "./github.js";
+import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -79,64 +78,44 @@ export class ConnectionManager implements IConnectionManager {
         // @note: We aren't actually doing anything with this atm.
         const abortController = new AbortController();
 
-        type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
-        const repoData: RepoData[] = (
-            await (async () => {
-                switch (config.type) {
-                    case 'github': {
-                        const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
-                        const hostUrl = config.url ?? 'https://github.com';
-                        const hostname = config.url ? new URL(config.url).hostname : 'github.com';
-
-                        return gitHubRepos.map((repo) => {
-                            const repoName = `${hostname}/${repo.full_name}`;
-                            const cloneUrl = new URL(repo.clone_url!);
-
-                            const record: RepoData = {
-                                external_id: repo.id.toString(),
-                                external_codeHostType: 'github',
-                                external_codeHostUrl: hostUrl,
-                                cloneUrl: cloneUrl.toString(),
-                                imageUrl: repo.owner.avatar_url,
-                                name: repoName,
-                                isFork: repo.fork,
-                                isArchived: !!repo.archived,
-                                org: {
-                                    connect: {
-                                        id: orgId,
-                                    },
-                                },
-                                connections: {
-                                    create: {
-                                        connectionId: job.data.connectionId,
-                                    }
-                                },
-                                metadata: {
-                                    'zoekt.web-url-type': 'github',
-                                    'zoekt.web-url': repo.html_url,
-                                    'zoekt.name': repoName,
-                                    'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
-                                    'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
-                                    'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
-                                    'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
-                                    'zoekt.archived': marshalBool(repo.archived),
-                                    'zoekt.fork': marshalBool(repo.fork),
-                                    'zoekt.public': marshalBool(repo.private === false)
-                                },
-                            };
-
-                            return record;
-                        })
-                    }
-                    case 'gitlab': {
-                        // @todo
-                        return [];
-                    }
+        const repoData: RepoData[] = await (async () => {
+            switch (config.type) {
+                case 'github': {
+                    return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
+                }
+                case 'gitlab': {
+                    return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'gitea': {
+                    return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'gerrit': {
+                    return await compileGerritConfig(config, job.data.connectionId, orgId);
+                }
+                default: {
+                    return [];
                 }
-            })()
-        )
+            }
+        )();
+
+        // Filter out any duplicates by external_id and external_codeHostUrl.
+        repoData.filter((repo, index, self) => {
+            return index === self.findIndex(r =>
+                r.external_id === repo.external_id &&
+                r.external_codeHostUrl === repo.external_codeHostUrl
+            );
+        })
+
+        // Filter out any duplicates by external_id and external_codeHostUrl.
+        repoData.filter((repo, index, self) => {
+            return index === self.findIndex(r =>
+                r.external_id === repo.external_id &&
+                r.external_codeHostUrl === repo.external_codeHostUrl
+            );
+        })
+
         // Filter out any duplicates by external_id and external_codeHostUrl.
-        .filter((repo, index, self) => {
+        repoData.filter((repo, index, self) => {
             return index === self.findIndex(r =>
                 r.external_id === repo.external_id &&
                 r.external_codeHostUrl === repo.external_codeHostUrl

packages/backend/src/gerrit.ts

@@ -1,8 +1,7 @@
 import fetch from 'cross-fetch';
 import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
-import { AppContext, GitRepository } from './types.js';
 import { createLogger } from './logger.js';
-import path from 'path';
+import micromatch from "micromatch";
 import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
 
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html
@@ -16,19 +15,26 @@ interface GerritProjectInfo {
    web_links?: GerritWebLink[];
 }
 
+interface GerritProject {
+   name: string;
+   id: string;
+   state?: string;
+   web_links?: GerritWebLink[];
+}
+
 interface GerritWebLink {
    name: string;
    url: string;
 }
 
 const logger = createLogger('Gerrit');
 
-export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
+export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
 
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
 
-   const { durationMs, data: projects } = await measure(async () => {
+   let { durationMs, data: projects } = await measure(async () => {
       try {
          return fetchAllProjects(url)
       } catch (err) {
@@ -42,67 +48,29 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
    }
 
    // exclude "All-Projects" and "All-Users" projects
-   delete projects['All-Projects'];
-   delete projects['All-Users'];
-   delete projects['All-Avatars']
-   delete projects['All-Archived-Projects']
-
-   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
-
-   let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
-      const project = projects[projectName];
-      let webUrl = "https://www.gerritcodereview.com/";
-      // Gerrit projects can have multiple web links; use the first one
-      if (project.web_links) {
-         const webLink = project.web_links[0];
-         if (webLink) {
-            webUrl = webLink.url;
-         }
-      }
-      const repoId = `${hostname}/${projectName}`;
-      const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
-
-      const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
-
-      return {
-         vcs: 'git',
-         codeHost: 'gerrit',
-         name: projectName,
-         id: repoId,
-         cloneUrl: cloneUrl,
-         path: repoPath,
-         isStale: false, // Gerrit projects are typically not stale
-         isFork: false, // Gerrit doesn't have forks in the same way as GitHub
-         isArchived: false,
-         gitConfigMetadata: {
-            // Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
-            'zoekt.web-url-type': 'gitiles',
-            'zoekt.web-url': webUrl,
-            'zoekt.name': repoId,
-            'zoekt.archived': marshalBool(false),
-            'zoekt.fork': marshalBool(false),
-            'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
-         },
-         branches: [],
-         tags: []
-      } satisfies GitRepository;
-   });
-
+   const excludedProjects = ['All-Projects', 'All-Users', 'All-Avatars', 'All-Archived-Projects'];
+   projects = projects.filter(project => !excludedProjects.includes(project.name));
+
    // include repos by glob if specified in config
    if (config.projects) {
-      repos = includeReposByName(repos, config.projects);
+      projects = projects.filter((project) => {
+         return micromatch.isMatch(project.name, config.projects!);
+      });
    }
-
+   
    if (config.exclude && config.exclude.projects) {
-      repos = excludeReposByName(repos, config.exclude.projects);
+      projects = projects.filter((project) => {
+         return !micromatch.isMatch(project.name, config.exclude!.projects!);
+      });
    }
 
-   return repos;
+   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
+   return projects;
 };
 
-const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
+const fetchAllProjects = async (url: string): Promise<GerritProject[]> => {
    const projectsEndpoint = `${url}projects/`;
-   let allProjects: GerritProjects = {};
+   let allProjects: GerritProject[] = [];
    let start = 0; // Start offset for pagination
    let hasMoreProjects = true;
 
@@ -119,8 +87,15 @@ const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
       const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
       const data: GerritProjects = JSON.parse(jsonText);
 
-      // Merge the current batch of projects with allProjects
-      Object.assign(allProjects, data);
+      // Add fetched projects to allProjects
+      for (const [projectName, projectInfo] of Object.entries(data)) {
+         allProjects.push({
+            name: projectName,
+            id: projectInfo.id,
+            state: projectInfo.state,
+            web_links: projectInfo.web_links
+         })
+      }
 
       // Check if there are more projects to fetch
       hasMoreProjects = Object.values(data).some(