Merge pull request #146 from jhogstrom/feat/parallel-page-export-upstream

feat: Add parallel page export with ThreadPoolExecutor

Author: Spenhouet

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Test CLI commands
         run: |
           uv run --with dist/*.whl --no-project confluence-markdown-exporter --help
-          uv run --with dist/*.whl --no-project cf-export --help
+          uv run --with dist/*.whl --no-project cme --help
 
       - name: Upload build artifacts for inspection
         uses: actions/upload-artifact@v7

CONTRIBUTING.md

@@ -65,7 +65,7 @@ echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc
 
    ```bash
    uv run confluence-markdown-exporter --help
-   uv run cf-export --help
+   uv run cme --help
    ```
 
 ## Development Workflow
@@ -75,7 +75,7 @@ echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc
 ```bash
 # Run with uv (recommended)
 uv run confluence-markdown-exporter [commands]
-uv run cf-export [commands]
+uv run cme [commands]
 
 # Or activate the virtual environment
 source .venv/bin/activate

README.md

@@ -56,51 +56,48 @@ pip install confluence-markdown-exporter
 
 Run the exporter with the desired Confluence page ID or space key. Execute the console application by typing `confluence-markdown-exporter` and one of the commands `pages`, `pages-with-descendants`, `spaces`, `all-spaces` or `config`. If a command is unclear, you can always add `--help` to get additional information.
 
-> [!TIP]
-> Instead of `confluence-markdown-exporter` you can also use the shorthand `cf-export`.
-
 #### 2.1. Export Page
 
 Export a single Confluence page by ID:
 
 ```sh
-confluence-markdown-exporter pages <page-id e.g. 645208921> --output-path <output path e.g. ./output_path/>
+cme pages <page-id e.g. 645208921> --output-path <output path e.g. ./output_path/>
 ```
 
 or by URL:
 
 ```sh
-confluence-markdown-exporter pages <page-url e.g. https://company.atlassian.net/MySpace/My+Page+Title> --output-path <output path e.g. ./output_path/>
+cme pages <page-url e.g. https://company.atlassian.net/MySpace/My+Page+Title> --output-path <output path e.g. ./output_path/>
 ```
 
 #### 2.2. Export Page with Descendants
 
 Export a Confluence page and all its descendant pages by page ID:
 
 ```sh
-confluence-markdown-exporter pages-with-descendants <page-id e.g. 645208921> --output-path <output path e.g. ./output_path/>
+cme pages-with-descendants <page-id e.g. 645208921> --output-path <output path e.g. ./output_path/>
 ```
 
 or by URL:
 
 ```sh
-confluence-markdown-exporter pages-with-descendants <page-url e.g. https://company.atlassian.net/MySpace/My+Page+Title> --output-path <output path e.g. ./output_path/>
+cme pages-with-descendants <page-url e.g. https://company.atlassian.net/MySpace/My+Page+Title> --output-path <output path e.g. ./output_path/>
 ```
 
 #### 2.3. Export Space
 
 Export all Confluence pages of a single Space:
 
 ```sh
-confluence-markdown-exporter spaces <space-key e.g. MYSPACE> --output-path <output path e.g. ./output_path/>
+cme spaces <space-key e.g. MYSPACE> --output-path <output path e.g. ./output_path/>
 ```
 
 #### 2.4. Export all Spaces
 
 Export all Confluence pages across all spaces:
 
 ```sh
-confluence-markdown-exporter all-spaces --output-path <output path e.g. ./output_path/>
+cme all-spaces --output-path <output path e.g. ./output_path/>
 ```
 
 ### 3. Output
@@ -127,7 +124,7 @@ All configuration and authentication is stored in a single JSON file managed by
 To interactively view and change configuration, run:
 
 ```sh
-confluence-markdown-exporter config
+cme config
 ```
 
 This will open a menu where you can:

confluence_markdown_exporter/api_clients.py

@@ -1,7 +1,6 @@
 import logging
 import os
 from functools import lru_cache
-from typing import Any
 
 import questionary
 import requests
@@ -10,6 +9,7 @@
 from questionary import Style
 
 from confluence_markdown_exporter.utils.app_data_store import ApiDetails
+from confluence_markdown_exporter.utils.app_data_store import AtlassianSdkConnectionConfig
 from confluence_markdown_exporter.utils.app_data_store import get_settings
 from confluence_markdown_exporter.utils.app_data_store import set_setting
 from confluence_markdown_exporter.utils.config_interactive import main_config_menu_loop
@@ -49,8 +49,12 @@ def response_hook(
 class ApiClientFactory:
     """Factory for creating authenticated Confluence and Jira API clients with retry config."""
 
-    def __init__(self, connection_config: dict[str, Any]) -> None:
-        self.connection_config = connection_config
+    def __init__(self, connection_config: AtlassianSdkConnectionConfig) -> None:
+        # Reconstruct as the base SDK type so model_dump() only yields SDK-compatible fields,
+        # even when a ConnectionConfig subclass is passed.
+        self.connection_config = AtlassianSdkConnectionConfig.model_validate(
+            connection_config.model_dump()
+        )
 
     def create_confluence(self, auth: ApiDetails) -> ConfluenceApiSdk:
         try:
@@ -59,7 +63,7 @@ def create_confluence(self, auth: ApiDetails) -> ConfluenceApiSdk:
                 username=auth.username.get_secret_value() if auth.api_token else None,
                 password=auth.api_token.get_secret_value() if auth.api_token else None,
                 token=auth.pat.get_secret_value() if auth.pat else None,
-                **self.connection_config,
+                **self.connection_config.model_dump(),
             )
             instance.get_all_spaces(limit=1)
         except Exception as e:
@@ -74,7 +78,7 @@ def create_jira(self, auth: ApiDetails) -> JiraApiSdk:
                 username=auth.username.get_secret_value() if auth.api_token else None,
                 password=auth.api_token.get_secret_value() if auth.api_token else None,
                 token=auth.pat.get_secret_value() if auth.pat else None,
-                **self.connection_config,
+                **self.connection_config.model_dump(),
             )
             instance.get_all_projects()
         except Exception as e:
@@ -87,11 +91,12 @@ def get_confluence_instance() -> ConfluenceApiSdk:
     """Get authenticated Confluence API client using current settings."""
     settings = get_settings()
     auth = settings.auth
-    connection_config = settings.connection_config.model_dump(exclude={"use_v2_api"})
 
     while True:
         try:
-            confluence = ApiClientFactory(connection_config).create_confluence(auth.confluence)
+            confluence = ApiClientFactory(settings.connection_config).create_confluence(
+                auth.confluence
+            )
             break
         except ConnectionError as e:
             questionary.print(
@@ -113,11 +118,10 @@ def get_jira_instance() -> JiraApiSdk:
     """Get authenticated Jira API client using current settings with required authentication."""
     settings = get_settings()
     auth = settings.auth
-    connection_config = settings.connection_config.model_dump(exclude={"use_v2_api"})
 
     while True:
         try:
-            jira = ApiClientFactory(connection_config).create_jira(auth.jira)
+            jira = ApiClientFactory(settings.connection_config).create_jira(auth.jira)
             break
         except ConnectionError:
             # Ask if user wants to use Confluence credentials for Jira

confluence_markdown_exporter/confluence.py

@@ -11,16 +11,20 @@
 import re
 import urllib.parse
 from collections.abc import Set
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
 from os import PathLike
 from pathlib import Path
 from string import Template
+from threading import local
 from typing import Literal
 from typing import TypeAlias
 from typing import cast
 from urllib.parse import unquote
 from urllib.parse import urlparse
 
 import yaml
+from atlassian import Confluence as ConfluenceApiSdk
 from atlassian.errors import ApiError
 from atlassian.errors import ApiNotFoundError
 from bs4 import BeautifulSoup
@@ -56,6 +60,23 @@
 settings = get_settings()
 confluence = get_confluence_instance()
 
+# Thread-local storage for API client instances (one per worker thread)
+_thread_local = local()
+
+
+def get_thread_confluence() -> ConfluenceApiSdk:
+    """Get or create Confluence instance for current thread.
+
+    The atlassian-python-api Confluence client uses requests.Session,
+    which is NOT thread-safe. Each worker thread needs its own instance.
+
+    Returns:
+        Confluence: A thread-local Confluence API client instance.
+    """
+    if not hasattr(_thread_local, "confluence"):
+        _thread_local.confluence = get_confluence_instance()
+    return _thread_local.confluence
+
 
 class JiraIssue(BaseModel):
     key: str
@@ -190,6 +211,7 @@ def pages(self) -> list["Page | Descendant"]:
         return [homepage, *homepage.descendants]
 
     def export(self) -> None:
+        """Export all pages in this space to Markdown."""
         export_pages(self.pages)
 
     @classmethod
@@ -1437,9 +1459,41 @@ def sync_removed_pages() -> None:
     LockfileManager.remove_pages(deleted)
 
 
+def _export_page_worker(page: "Page | Descendant", use_thread_local: bool = False) -> None:  # noqa: FBT001, FBT002
+    """Export a single Confluence page to Markdown (worker function).
+
+    Args:
+        page: The page or descendant to export.
+        use_thread_local: If True, use thread-local Confluence instance
+                         (required for parallel export).
+    """
+    if use_thread_local:
+        # Use thread-local confluence instance for thread safety
+        global confluence  # noqa: PLW0603
+        old_confluence = confluence
+        confluence = get_thread_confluence()
+        try:
+            _page = Page.from_id(page.id)
+            _page.export()
+            # Record to lockfile if enabled
+            LockfileManager.record_page(_page)
+        finally:
+            confluence = old_confluence
+    else:
+        # Serial mode - use global confluence instance
+        _page = Page.from_id(page.id)
+        _page.export()
+        # Record to lockfile if enabled
+        LockfileManager.record_page(_page)
+
+
 def export_pages(pages: list["Page | Descendant"]) -> None:
     """Export a list of Confluence pages to Markdown.
 
+    Pages are exported in parallel using ThreadPoolExecutor for significant
+    performance improvement. Worker count is read from
+    settings.connection_config.max_workers (default: 20).
+
     Args:
         pages: List of pages to export.
     """
@@ -1451,9 +1505,34 @@ def export_pages(pages: list["Page | Descendant"]) -> None:
         logger.info("No pages to export based on lockfile state.")
         return
 
-    for page in (pbar := tqdm(pages_to_export, smoothing=0.05)):
-        pbar.set_postfix_str(f"Exporting page {page.id}")
-        _page = Page.from_id(page.id)
-        _page.export()
-        # Record to lockfile if enabled
-        LockfileManager.record_page(_page)
+    # Get worker count from config
+    max_workers = settings.connection_config.max_workers
+
+    # Serial mode for debugging or single worker
+    if DEBUG or max_workers <= 1:
+        logger.info("Using serial export mode (max_workers=1)")
+        for page in (pbar := tqdm(pages_to_export, smoothing=0.05)):
+            pbar.set_postfix_str(f"Exporting page {page.id}")
+            _export_page_worker(page, use_thread_local=False)
+        return
+
+    # Parallel mode
+    logger.info(f"Using parallel export mode ({max_workers} workers)")
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit all export tasks
+        futures = {
+            executor.submit(_export_page_worker, page, use_thread_local=True): page
+            for page in pages_to_export
+        }
+
+        # Track progress with tqdm
+        with tqdm(total=len(pages_to_export), smoothing=0.05) as pbar:
+            for future in as_completed(futures):
+                page = futures[future]
+                try:
+                    future.result()  # Raise exception if export failed
+                    pbar.set_postfix_str(f"Completed page {page.id}")
+                except Exception:
+                    logger.exception(f"Failed to export page {page.id}")
+                finally:
+                    pbar.update(1)

confluence_markdown_exporter/utils/app_data_store.py

@@ -30,8 +30,13 @@ def get_app_config_path() -> Path:
 APP_CONFIG_PATH = get_app_config_path()
 
 
-class ConnectionConfig(BaseModel):
-    """Configuration for the connection like retry options."""
+class AtlassianSdkConnectionConfig(BaseModel):
+    """Connection parameters forwarded directly to the Atlassian SDK client constructors.
+
+    Only fields that are valid constructor keyword arguments for
+    atlassian.Confluence (ConfluenceApiSdk) and atlassian.Jira (JiraApiSdk)
+    may be added here.
+    """
 
     backoff_and_retry: bool = Field(
         default=True,
@@ -76,6 +81,11 @@ class ConnectionConfig(BaseModel):
             "Timeout in seconds for API requests. Prevents hanging on slow/unresponsive servers."
         ),
     )
+
+
+class ConnectionConfig(AtlassianSdkConnectionConfig):
+    """Full connection configuration, extending the Atlassian SDK config with app-level settings."""
+
     use_v2_api: bool = Field(
         default=False,
         title="Use Confluence v2 REST API",
@@ -85,6 +95,15 @@ class ConnectionConfig(BaseModel):
             "Must be disabled for older self-hosted Confluence Server instances."
         ),
     )
+    max_workers: int = Field(
+        default=20,
+        title="Max Workers",
+        description=(
+            "Maximum number of parallel workers for page export. "
+            "Set to 1 for serial mode (useful for debugging). "
+            "Higher values improve performance but may hit API rate limits."
+        ),
+    )
 
 
 class ApiDetails(BaseModel):

pyproject.toml

@@ -42,7 +42,7 @@ Tracker = "https://github.com/Spenhouet/confluence-markdown-exporter/issues"
 
 [project.scripts]
 confluence-markdown-exporter = "confluence_markdown_exporter.main:app"
-cf-export = "confluence_markdown_exporter.main:app"
+cme = "confluence_markdown_exporter.main:app"
 
 [tool.hatch.build.targets.wheel]
 packages = ["confluence_markdown_exporter"]