feat: switch from kohlrahbi to fundamend as data source; re-vibe entire business logic

.github/workflows/coverage.yml

@@ -13,8 +13,10 @@ jobs:
       SQLITE_AHB_DB_7Z_ARCHIVE_PASSWORD: ${{ secrets.SQLITE_AHB_DB_7Z_ARCHIVE_PASSWORD }}
     steps:
       - uses: actions/checkout@v6
-      - name: Set up Python
-        uses: actions/setup-python@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-python@v6
+        name: Setup Python
         with:
           python-version: "3.14"
       - name: Install dependencies

.github/workflows/unittests.yml

@@ -3,9 +3,9 @@ name: "Unittests"
 on:
   push:
     branches: [main]
+  workflow_dispatch: {}
   pull_request:
     branches: [main]
-  workflow_dispatch: {}
 jobs:
   pytest:
     runs-on: ${{ matrix.os }}
@@ -17,6 +17,8 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v6
         with:

README.md

@@ -11,12 +11,8 @@
 ![Pylint status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Linting/badge.svg)
 ![Formatting status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Formatting/badge.svg)
 
-Tool for automatic AHB comparison of consecutive `Formatversionen`. Highlighted changes between `PruefIDs` of various 
-`Nachrichtenformate` are provided by already formatted `.xlsx` files or raw `.csv` files located inside the 
-[machine-readable-anwendungshandbücher](https://github.com/Hochfrequenz/machine-readable_anwendungshandbuecher/tree/main/diff) 
-repository.
-
-<img src="https://raw.githubusercontent.com/Hochfrequenz/ahlbatross/main/output.png" alt="output.png">
+Tool for comparing AHB (Anwendungshandbuch) documents across different format versions.
+Compares Prüfidentifikators and stores the results in a SQLite database for further analysis.
 
 ## Installation
 Install this package from [PyPI](https://pypi.org/project/ahlbatross/) using
@@ -25,6 +21,35 @@ Install this package from [PyPI](https://pypi.org/project/ahlbatross/) using
 pip install ahlbatross
 ```
 
+## Usage
+
+### Populate Comparison Database
+
+To create comparison tables for all format version pairs in an existing AHB database:
+
+```shell
+ahlbatross populate-db path/to/ahb.db
+```
+
+For encrypted `.7z` archives, provide the password via `--password` or environment variable:
+
+```shell
+ahlbatross populate-db path/to/ahb.db.7z --password YOUR_PASSWORD
+
+# Or via environment variable
+export SQLITE_AHB_DB_7Z_ARCHIVE_PASSWORD=YOUR_PASSWORD
+ahlbatross populate-db path/to/ahb.db.7z
+```
+
+This command:
+- Finds all format versions present in the database
+- Compares all Prüfidentifikators across all pairs of format versions
+- Creates and populates `ahb_line_comparison` and `ahb_comparison_summary` tables
+- Vacuums the database for optimal performance
+- For encrypted archives: decrypts, processes, and re-encrypts with the same password
+
+The comparison tables can then be queried to find changes between any two format versions.
+
 ## Development Setup
 
 To set up the python development environment, install the required dependencies via

pyproject.toml

@@ -20,11 +20,11 @@ classifiers = [
     "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
-    "efoli>=1.4.0",
-    "kohlrahbi>=1.11.2",
+    "efoli>=2.1.0",
     "pydantic>=2.10.6",
-    "xlsxwriter>=3.2.0",
-    "typer>=0.15.4"
+    "typer>=0.15.4",
+    "fundamend[sqlmodels]>=0.27.0",
+    "py7zr"
 ]
 dynamic = ["version"]
 
@@ -37,7 +37,8 @@ Homepage = "https://github.com/Hochfrequenz/ahlbatross"
 
 [project.optional-dependencies]
 tests = [
-    "pytest==9.0.1"
+    "pytest==9.0.1",
+    "syrupy==5.0.0"
 ]
 linting = [
     "pylint==4.0.2"
@@ -73,6 +74,13 @@ profile = "black"
 
 [tool.pylint."MESSAGES CONTROL"]
 max-line-length = 120
+extension-pkg-whitelist = "pydantic"
+
+
+[tool.pytest.ini_options]
+markers = [
+    "snapshot: marks tests as snapshot tests (deselect with '-m \"not snapshot\"')",
+]
 
 [mypy]
 truethy-bool = true

requirements.txt

@@ -1,112 +1,88 @@
 #
-# This file is autogenerated by pip-compile with Python 3.13
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile pyproject.toml
 #
-aiohappyeyeballs==2.6.1
-    # via aiohttp
-aiohttp==3.13.2
-    # via edi-energy-scraper
-aiosignal==1.4.0
-    # via aiohttp
 annotated-types==0.7.0
     # via pydantic
-attrs==25.4.0
-    # via aiohttp
+brotli==1.2.0
+    # via py7zr
 click==8.3.1
-    # via
-    #   kohlrahbi
-    #   typer
-colorlog==6.10.1
-    # via kohlrahbi
-edi-energy-scraper==2.2.3
-    # via kohlrahbi
+    # via typer
+colorama==0.4.6
+    # via click
 efoli==2.1.1
     # via
     #   ahlbatross (pyproject.toml)
-    #   edi-energy-scraper
-    #   kohlrahbi
-et-xmlfile==2.0.0
-    # via openpyxl
-frozenlist==1.8.0
-    # via
-    #   aiohttp
-    #   aiosignal
-idna==3.11
-    # via yarl
-kohlrahbi==1.13.0
+    #   fundamend
+fundamend[sqlmodels]==0.27.0
     # via ahlbatross (pyproject.toml)
-lxml==6.0.2
-    # via python-docx
+greenlet==3.2.4
+    # via sqlalchemy
+inflate64==1.0.4
+    # via py7zr
+librt==0.6.3
+    # via mypy
 markdown-it-py==4.0.0
     # via rich
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.8.0
-    # via
-    #   edi-energy-scraper
-    #   kohlrahbi
-multidict==6.7.0
-    # via
-    #   aiohttp
-    #   yarl
-numpy==2.3.5
-    # via pandas
-openpyxl==3.1.5
-    # via kohlrahbi
-pandas==2.3.3
-    # via kohlrahbi
-propcache==0.4.1
-    # via
-    #   aiohttp
-    #   yarl
+multivolumefile==0.2.3
+    # via py7zr
+mypy==1.19.0
+    # via sqlalchemy
+mypy-extensions==1.1.0
+    # via mypy
+pathspec==0.12.1
+    # via mypy
+psutil==7.1.3
+    # via py7zr
+py7zr==1.0.0
+    # via ahlbatross (pyproject.toml)
+pybcj==1.0.7
+    # via py7zr
+pycryptodomex==3.23.0
+    # via py7zr
 pydantic==2.12.5
     # via
     #   ahlbatross (pyproject.toml)
-    #   edi-energy-scraper
-    #   kohlrahbi
+    #   fundamend
+    #   sqlmodel
 pydantic-core==2.41.5
     # via pydantic
 pygments==2.19.2
     # via rich
-pypdf==6.4.0
-    # via edi-energy-scraper
-python-dateutil==2.9.0.post0
-    # via pandas
-python-docx==1.2.0
-    # via kohlrahbi
+pyppmd==1.2.0
+    # via py7zr
 pytz==2025.2
-    # via
-    #   edi-energy-scraper
-    #   efoli
-    #   kohlrahbi
-    #   pandas
+    # via efoli
+pyzstd==0.18.0
+    # via py7zr
 rich==14.2.0
     # via typer
 shellingham==1.5.4
     # via typer
-six==1.17.0
-    # via python-dateutil
-tomlkit==0.13.3
-    # via kohlrahbi
+sqlalchemy[mypy]==2.0.44
+    # via
+    #   fundamend
+    #   sqlmodel
+sqlmodel==0.0.27
+    # via fundamend
+texttable==1.7.0
+    # via py7zr
 typer==0.20.0
     # via ahlbatross (pyproject.toml)
 typing-extensions==4.15.0
     # via
-    #   aiosignal
+    #   mypy
     #   pydantic
     #   pydantic-core
-    #   python-docx
+    #   pyzstd
+    #   sqlalchemy
     #   typer
     #   typing-inspection
 typing-inspection==0.4.2
     # via pydantic
-tzdata==2025.2
-    # via pandas
 xlsxwriter==3.2.9
-    # via
-    #   ahlbatross (pyproject.toml)
-    #   kohlrahbi
-yarl==1.22.0
-    # via aiohttp
+    # via ahlbatross (pyproject.toml)