feat: incremental download using lockfile

confluence_markdown_exporter/confluence.py

@@ -39,6 +39,7 @@
 from confluence_markdown_exporter.utils.export import sanitize_filename
 from confluence_markdown_exporter.utils.export import sanitize_key
 from confluence_markdown_exporter.utils.export import save_file
+from confluence_markdown_exporter.utils.lockfile import LockfileManager
 from confluence_markdown_exporter.utils.table_converter import TableConverter
 from confluence_markdown_exporter.utils.type_converter import str_to_bool
 
@@ -133,7 +134,7 @@ class Organization(BaseModel):
     spaces: list["Space"]
 
     @property
-    def pages(self) -> list[int]:
+    def pages(self) -> list["Page"]:
         return [page for space in self.spaces for page in space.pages]
 
     def export(self) -> None:
@@ -165,15 +166,15 @@ class Space(BaseModel):
     homepage: int | None
 
     @property
-    def pages(self) -> list[int]:
+    def pages(self) -> list["Page"]:
         if self.homepage is None:
             logger.warning(
                 f"Space '{self.name}' (key: {self.key}) has no homepage. No pages will be exported."
             )
             return []
 
         homepage = Page.from_id(self.homepage)
-        return [self.homepage, *homepage.descendants]
+        return [homepage, *homepage.descendants]
 
     def export(self) -> None:
         export_pages(self.pages)
@@ -340,12 +341,14 @@ class Page(Document):
     editor2: str
     labels: list["Label"]
     attachments: list["Attachment"]
+    version: Version
 
     @property
-    def descendants(self) -> list[int]:
+    def descendants(self) -> list["Page"]:
         url = "rest/api/content/search"
         params = {
             "cql": f"type=page AND ancestor={self.id}",
+            "expand": "metadata.properties,ancestors,version",
             "limit": 100,
         }
         results = []
@@ -372,8 +375,7 @@ def descendants(self) -> list[int]:
                 f"Unexpected error when fetching descendants for content ID {self.id}."
             )
             return []
-
-        return [result["id"] for result in results]
+        return [self.from_json(result) for result in results]
 
     @property
     def _template_vars(self) -> dict[str, str]:
@@ -409,8 +411,11 @@ def export(self) -> None:
         self.export_attachments()
         self.export_markdown()
 
+        # Record to lockfile if enabled
+        LockfileManager.record_page(self)
+
     def export_with_descendants(self) -> None:
-        export_pages([self.id, *self.descendants])
+        export_pages([self, *self.descendants])
 
     def export_body(self) -> None:
         soup = BeautifulSoup(self.html, "html.parser")
@@ -499,6 +504,7 @@ def from_json(cls, data: JsonResponse) -> "Page":
             ],
             attachments=Attachment.from_page_id(data.get("id", 0)),
             ancestors=[ancestor.get("id") for ancestor in data.get("ancestors", [])][1:],
+            version=Version.from_json(data.get("version", {})),
         )
 
     @classmethod
@@ -511,7 +517,7 @@ def from_id(cls, page_id: int) -> "Page":
                     confluence.get_page_by_id(
                         page_id,
                         expand="body.view,body.export_view,body.editor2,metadata.labels,"
-                        "metadata.properties,ancestors",
+                        "metadata.properties,ancestors,version",
                     ),
                 )
             )
@@ -528,6 +534,7 @@ def from_id(cls, page_id: int) -> "Page":
                 labels=[],
                 attachments=[],
                 ancestors=[],
+                version=Version.from_json({}),
             )
 
     @classmethod
@@ -1001,7 +1008,7 @@ def convert_drawio(self, el: BeautifulSoup, text: str, parent_tags: list[str]) -
 
             return ""
 
-        def convert_plantuml(self, el: BeautifulSoup, text: str, parent_tags: list[str]) -> str: # noqa: PLR0911
+        def convert_plantuml(self, el: BeautifulSoup, text: str, parent_tags: list[str]) -> str:  # noqa: PLR0911
             """Convert PlantUML diagrams from editor2 XML to Markdown code blocks.
 
             PlantUML diagrams are stored in the editor2 XML as structured macros with
@@ -1103,13 +1110,17 @@ def export_page(page_id: int) -> None:
     page.export()
 
 
-def export_pages(page_ids: list[int]) -> None:
+def export_pages(pages: list["Page"]) -> None:
     """Export a list of Confluence pages to Markdown.
 
     Args:
-        page_ids: List of pages to export.
+        pages: List of pages to export.
         output_path: The output path.
     """
-    for page_id in (pbar := tqdm(page_ids, smoothing=0.05)):
-        pbar.set_postfix_str(f"Exporting page {page_id}")
-        export_page(page_id)
+    for page in (pbar := tqdm(pages, smoothing=0.05)):
+        # filter pages new and updated only
+        if LockfileManager.should_export(page):
+            pbar.set_postfix_str(f"Exporting page {page.id}")
+            export_page(page.id)
+        else:
+            pbar.set_postfix_str(f"Skipping page {page.id} (no changes)")

confluence_markdown_exporter/main.py

@@ -8,6 +8,7 @@
 from confluence_markdown_exporter.utils.app_data_store import get_settings
 from confluence_markdown_exporter.utils.app_data_store import set_setting
 from confluence_markdown_exporter.utils.config_interactive import main_config_menu_loop
+from confluence_markdown_exporter.utils.lockfile import LockfileManager
 from confluence_markdown_exporter.utils.measure_time import measure
 from confluence_markdown_exporter.utils.platform_compat import handle_powershell_tilde_expansion
 from confluence_markdown_exporter.utils.type_converter import str_to_bool
@@ -32,12 +33,22 @@ def pages(
             help="Directory to write exported Markdown files to. Overrides config if set."
         ),
     ] = None,
+    *,
+    lockfile: Annotated[
+        bool,
+        typer.Option(
+            "--lockfile",
+            help="Enable lock file tracking for exported pages.",
+        ),
+    ] = False,
 ) -> None:
     from confluence_markdown_exporter.confluence import Page
 
     with measure(f"Export pages {', '.join(pages)}"):
+        override_output_path_config(output_path)
+        if lockfile:
+            LockfileManager.init()
         for page in pages:
-            override_output_path_config(output_path)
             _page = Page.from_id(int(page)) if page.isdigit() else Page.from_url(page)
             _page.export()
 
@@ -51,12 +62,29 @@ def pages_with_descendants(
             help="Directory to write exported Markdown files to. Overrides config if set."
         ),
     ] = None,
+    *,
+    lockfile: Annotated[
+        bool,
+        typer.Option(
+            "--lockfile",
+            help="Enable lock file tracking for exported pages.",
+        ),
+    ] = False,
+    incremental: Annotated[
+        bool,
+        typer.Option(
+            "--incremental",
+            help="Only export pages that have changed since last export.",
+        ),
+    ] = False,
 ) -> None:
     from confluence_markdown_exporter.confluence import Page
 
     with measure(f"Export pages {', '.join(pages)} with descendants"):
+        override_output_path_config(output_path)
+        if lockfile or incremental:
+            LockfileManager.init()
         for page in pages:
-            override_output_path_config(output_path)
             _page = Page.from_id(int(page)) if page.isdigit() else Page.from_url(page)
             _page.export_with_descendants()
 
@@ -70,6 +98,21 @@ def spaces(
             help="Directory to write exported Markdown files to. Overrides config if set."
         ),
     ] = None,
+    *,
+    lockfile: Annotated[
+        bool,
+        typer.Option(
+            "--lockfile",
+            help="Enable lock file tracking for exported pages.",
+        ),
+    ] = False,
+    incremental: Annotated[
+        bool,
+        typer.Option(
+            "--incremental",
+            help="Only export pages that have changed since last export.",
+        ),
+    ] = False,
 ) -> None:
     from confluence_markdown_exporter.confluence import Space
 
@@ -78,8 +121,10 @@ def spaces(
     normalized_space_keys = [handle_powershell_tilde_expansion(key) for key in space_keys]
 
     with measure(f"Export spaces {', '.join(normalized_space_keys)}"):
+        override_output_path_config(output_path)
+        if lockfile or incremental:
+            LockfileManager.init()
         for space_key in normalized_space_keys:
-            override_output_path_config(output_path)
             space = Space.from_key(space_key)
             space.export()
 
@@ -92,11 +137,28 @@ def all_spaces(
             help="Directory to write exported Markdown files to. Overrides config if set."
         ),
     ] = None,
+    *,
+    lockfile: Annotated[
+        bool,
+        typer.Option(
+            "--lockfile",
+            help="Enable lock file tracking for exported pages.",
+        ),
+    ] = False,
+    incremental: Annotated[
+        bool,
+        typer.Option(
+            "--incremental",
+            help="Only export pages that have changed since last export.",
+        ),
+    ] = False,
 ) -> None:
     from confluence_markdown_exporter.confluence import Organization
 
     with measure("Export all spaces"):
         override_output_path_config(output_path)
+        if lockfile or incremental:
+            LockfileManager.init()
         org = Organization.from_api()
         org.export()
 
@@ -125,6 +187,33 @@ def config(
         main_config_menu_loop(jump_to)
 
 
+@app.command(help="Delete exported files that are not tracked in the lockfile.")
+def prune(
+    output_path: Annotated[
+        Path | None,
+        typer.Option(help="Directory containing exported Markdown files. Overrides config if set."),
+    ] = None,
+    *,
+    dry_run: Annotated[
+        bool,
+        typer.Option(
+            "--dry-run",
+            help="Show files that would be deleted without actually deleting them.",
+        ),
+    ] = False,
+) -> None:
+    """Delete exported files not tracked in the lockfile."""
+    override_output_path_config(output_path)
+    LockfileManager.init()
+    deleted = LockfileManager.cleanup_untracked(dry_run=dry_run)
+    if dry_run:
+        typer.echo(f"Would delete {len(deleted)} file(s):")
+        for path in deleted:
+            typer.echo(f"  {path}")
+    else:
+        typer.echo(f"Deleted {len(deleted)} file(s).")
+
+
 @app.command(help="Show the current version of confluence-markdown-exporter.")
 def version() -> None:
     """Display the current version."""