fix: Detection of repository type for self hosted instances (#381)

* fix repository type selection and usage

* Update src/app/[owner]/[repo]/page.tsx

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update api/data_pipeline.py

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: DaniilSmirnov

api/data_pipeline.py

@@ -7,7 +7,6 @@
 import tiktoken
 import logging
 import base64
-import re
 import glob
 from adalflow.utils import get_adalflow_default_root_path
 from adalflow.core.db import LocalDB
@@ -67,11 +66,12 @@ def count_tokens(text: str, embedder_type: str = None, is_ollama_embedder: bool
         # Rough approximation: 4 characters per token
         return len(text) // 4
 
-def download_repo(repo_url: str, local_path: str, type: str = "github", access_token: str = None) -> str:
+def download_repo(repo_url: str, local_path: str, repo_type: str = None, access_token: str = None) -> str:
     """
     Downloads a Git repository (GitHub, GitLab, or Bitbucket) to a specified local path.
 
     Args:
+        repo_type(str): Type of repository
         repo_url (str): The URL of the Git repository to clone.
         local_path (str): The local directory where the repository will be cloned.
         access_token (str, optional): Access token for private repositories.
@@ -103,14 +103,14 @@ def download_repo(repo_url: str, local_path: str, type: str = "github", access_t
         if access_token:
             parsed = urlparse(repo_url)
             # Determine the repository type and format the URL accordingly
-            if type == "github":
+            if repo_type == "github":
                 # Format: https://{token}@{domain}/owner/repo.git
                 # Works for both github.com and enterprise GitHub domains
                 clone_url = urlunparse((parsed.scheme, f"{access_token}@{parsed.netloc}", parsed.path, '', '', ''))
-            elif type == "gitlab":
+            elif repo_type == "gitlab":
                 # Format: https://oauth2:{token}@gitlab.com/owner/repo.git
                 clone_url = urlunparse((parsed.scheme, f"oauth2:{access_token}@{parsed.netloc}", parsed.path, '', '', ''))
-            elif type == "bitbucket":
+            elif repo_type == "bitbucket":
                 # Format: https://x-token-auth:{token}@bitbucket.org/owner/repo.git
                 clone_url = urlunparse((parsed.scheme, f"x-token-auth:{access_token}@{parsed.netloc}", parsed.path, '', '', ''))
 
@@ -675,11 +675,12 @@ def get_bitbucket_file_content(repo_url: str, file_path: str, access_token: str
         raise ValueError(f"Failed to get file content: {str(e)}")
 
 
-def get_file_content(repo_url: str, file_path: str, type: str = "github", access_token: str = None) -> str:
+def get_file_content(repo_url: str, file_path: str, repo_type: str = None, access_token: str = None) -> str:
     """
     Retrieves the content of a file from a Git repository (GitHub or GitLab).
 
     Args:
+        repo_type (str): Type of repository
         repo_url (str): The URL of the repository
         file_path (str): The path to the file within the repository
         access_token (str, optional): Access token for private repositories
@@ -690,14 +691,14 @@ def get_file_content(repo_url: str, file_path: str, type: str = "github", access
     Raises:
         ValueError: If the file cannot be fetched or if the URL is not valid
     """
-    if type == "github":
+    if repo_type == "github":
         return get_github_file_content(repo_url, file_path, access_token)
-    elif type == "gitlab":
+    elif repo_type == "gitlab":
         return get_gitlab_file_content(repo_url, file_path, access_token)
-    elif type == "bitbucket":
+    elif repo_type == "bitbucket":
         return get_bitbucket_file_content(repo_url, file_path, access_token)
     else:
-        raise ValueError("Unsupported repository URL. Only GitHub and GitLab are supported.")
+        raise ValueError("Unsupported repository type. Only GitHub, GitLab, and Bitbucket are supported.")
 
 class DatabaseManager:
     """
@@ -709,14 +710,15 @@ def __init__(self):
         self.repo_url_or_path = None
         self.repo_paths = None
 
-    def prepare_database(self, repo_url_or_path: str, type: str = "github", access_token: str = None, 
-                       embedder_type: str = None, is_ollama_embedder: bool = None,
-                       excluded_dirs: List[str] = None, excluded_files: List[str] = None,
-                       included_dirs: List[str] = None, included_files: List[str] = None) -> List[Document]:
+    def prepare_database(self, repo_url_or_path: str, repo_type: str = None, access_token: str = None,
+                         embedder_type: str = None, is_ollama_embedder: bool = None,
+                         excluded_dirs: List[str] = None, excluded_files: List[str] = None,
+                         included_dirs: List[str] = None, included_files: List[str] = None) -> List[Document]:
         """
         Create a new database from the repository.
 
         Args:
+            repo_type(str): Type of repository
             repo_url_or_path (str): The URL or local path of the repository
             access_token (str, optional): Access token for private repositories
             embedder_type (str, optional): Embedder type to use ('openai', 'google', 'ollama').
@@ -736,7 +738,7 @@ def prepare_database(self, repo_url_or_path: str, type: str = "github", access_t
             embedder_type = 'ollama' if is_ollama_embedder else None
 
         self.reset_database()
-        self._create_repo(repo_url_or_path, type, access_token)
+        self._create_repo(repo_url_or_path, repo_type, access_token)
         return self.prepare_db_index(embedder_type=embedder_type, excluded_dirs=excluded_dirs, excluded_files=excluded_files,
                                    included_dirs=included_dirs, included_files=included_files)
 
@@ -763,14 +765,15 @@ def _extract_repo_name_from_url(self, repo_url_or_path: str, repo_type: str) ->
             repo_name = url_parts[-1].replace(".git", "")
         return repo_name
 
-    def _create_repo(self, repo_url_or_path: str, repo_type: str = "github", access_token: str = None) -> None:
+    def _create_repo(self, repo_url_or_path: str, repo_type: str = None, access_token: str = None) -> None:
         """
         Download and prepare all paths.
         Paths:
         ~/.adalflow/repos/{owner}_{repo_name} (for url, local path will be the same)
         ~/.adalflow/databases/{owner}_{repo_name}.pkl
 
         Args:
+            repo_type(str): Type of repository
             repo_url_or_path (str): The URL or local path of the repository
             access_token (str, optional): Access token for private repositories
         """
@@ -866,16 +869,17 @@ def prepare_db_index(self, embedder_type: str = None, is_ollama_embedder: bool =
         logger.info(f"Total transformed documents: {len(transformed_docs)}")
         return transformed_docs
 
-    def prepare_retriever(self, repo_url_or_path: str, type: str = "github", access_token: str = None):
+    def prepare_retriever(self, repo_url_or_path: str, repo_type: str = None, access_token: str = None):
         """
         Prepare the retriever for a repository.
         This is a compatibility method for the isolated API.
 
         Args:
+            repo_type(str): Type of repository
             repo_url_or_path (str): The URL or local path of the repository
             access_token (str, optional): Access token for private repositories
 
         Returns:
             List[Document]: List of Document objects
         """
-        return self.prepare_database(repo_url_or_path, type, access_token)
+        return self.prepare_database(repo_url_or_path, repo_type, access_token)

src/app/[owner]/[repo]/page.tsx

@@ -192,11 +192,20 @@ export default function RepoWikiPage() {
   const isCustomModelParam = searchParams.get('is_custom_model') === 'true';
   const customModelParam = searchParams.get('custom_model') || '';
   const language = searchParams.get('language') || 'en';
-  const repoType = repoUrl?.includes('bitbucket.org')
+  const repoHost = (() => {
+    if (!repoUrl) return '';
+    try {
+      return new URL(repoUrl).hostname.toLowerCase();
+    } catch (e) {
+      console.warn(`Invalid repoUrl provided: ${repoUrl}`);
+      return '';
+    }
+  })();
+  const repoType = repoHost?.includes('bitbucket')
     ? 'bitbucket'
-    : repoUrl?.includes('gitlab.com')
+    : repoHost?.includes('gitlab')
       ? 'gitlab'
-      : repoUrl?.includes('github.com')
+      : repoHost?.includes('github')
         ? 'github'
         : searchParams.get('type') || 'github';