ENH: Enable the running of notebooks in the mapie/tests/notebooks in CI. (#786)

* ENH: add the theoretical risk control validation notebook in the CI.

* commande "make notebook-tests"

* ajoute de commande

* Add command in CI pipeline

* make pytest to skip notebooks

* install nbclient and jupyter in CI

* handle exit(0)

* ignore notebooks when checking coverage

Author: hmaissoro

.github/workflows/test.yml

@@ -73,5 +73,7 @@ jobs:
         run: make type-check
       - name: Test and coverage with pytest
         run: make coverage
+      - name: Test notebooks
+        run: make notebook-tests
       - name: Code coverage
         run: codecov

CONTRIBUTING.rst

@@ -81,6 +81,13 @@ The coverage should absolutely be 100%.
 
     $ make coverage
 
+If your modification affects notebook-based experiments in ``mapie/tests/notebooks/``,
+you should also ensure that all notebooks run successfully.
+
+.. code-block:: sh
+
+    $ make notebook-tests
+
 Documenting your change
 -----------------------
 

Makefile

@@ -22,7 +22,6 @@ coverage:
 		--no-cov-on-fail \
 		--doctest-modules
 
-
 ### Checks that are run in ReadTheDocs CI ###
 doc:
 	$(MAKE) html -C doc
@@ -39,7 +38,7 @@ all-checks:
 	$(MAKE) coverage
 
 tests:
-	pytest -vs --doctest-modules mapie
+	pytest -vs --doctest-modules mapie --ignore=mapie/tests/notebooks
 
 clean-doc:
 	$(MAKE) clean -C doc
@@ -55,3 +54,8 @@ clean:
 	rm -rf **__pycache__
 	$(MAKE) clean-build
 	$(MAKE) clean-doc
+
+# Run all notebooks located in mapie/tests/notebooks/
+notebook-tests:
+	@echo "Executing all notebooks in mapie/tests/notebooks/..."
+	python mapie/tests/notebooks/_run_notebooks.py

environment.ci.yml

@@ -8,3 +8,5 @@ dependencies:
     - mypy<1.15
     - pandas
     - pytest-cov
+    - jupyter
+    - nbclient

mapie/tests/notebooks/__init__.py

@@ -0,0 +1,24 @@
+# pragma: no cover
+import sys
+from pathlib import Path
+
+from nbclient import NotebookClient
+from nbformat import read
+
+NOTEBOOKS_DIR = Path(__file__).parent
+notebooks = NOTEBOOKS_DIR.rglob("*.ipynb")
+
+if __name__ == "__main__":  # pragma: no cover
+    for nb_path in notebooks:
+        print(f"Running {nb_path} ...")
+        try:
+            with nb_path.open() as f:
+                nb = read(f, as_version=4)
+            client = NotebookClient(nb, timeout=3600)
+            client.execute()
+        except Exception as e:
+            print(f"Notebook {nb_path} failed:\n{e}")
+            sys.exit(1)
+
+    print("\nAll notebooks executed successfully. 100% OK.\n")
+    sys.exit(0)