[GSK-2159] Using a subprocess for pytest

.github/workflows/build-python.yml

@@ -155,22 +155,6 @@ jobs:
           pdm run pip freeze | grep '^pandas'
           pdm run pip freeze | grep -q '^pandas==${{ matrix.pandas_v1 && '1' || '2' }}\.'
 
-      - name: Test code (RAM expensive in tests/models, tests/testing, tests/test_metamorphic_invariance.py)
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram tests/models tests/testing tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, /tests/models and /tests/testing)
-        if: ${{ !startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram --memray tests/models tests/testing tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, others)
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram tests/ --ignore=tests/models  --ignore=tests/testing --ignore=tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, others)
-        if: ${{ !startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram --memray tests/ --ignore=tests/models --ignore=tests/testing --ignore=tests/test_metamorphic_invariance.py
-
       - name: Test code (concurrency)
         run: pdm test-worker
 
@@ -199,13 +183,7 @@ jobs:
         if: ${{ inputs.run-integration-tests }}
         env:
           PYTEST_XDIST_AUTO_NUM_WORKERS: 2
-        run: pdm test-slow tests/integrations
-
-      - name: Run integration tests for python
-        if: ${{ inputs.run-integration-tests }}
-        env:
-          PYTEST_XDIST_AUTO_NUM_WORKERS: 2
-        run: pdm test-slow tests/ --ignore=tests/integrations
+        run: pdm test-slow tests/
 
       - name: "Memory csv"
         if: ${{ always() && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 }}

conftest.py

@@ -1,11 +1,19 @@
+from typing import List
+
 import gc
+import json
 import os
+import subprocess
+import sys
 import time
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 import psutil
 import pytest
+from _pytest.config.argparsing import Parser
 from _pytest.python import Function
+from _pytest.reports import TestReport
 
 
 @pytest.hookimpl(hookwrapper=True)
@@ -60,3 +68,43 @@ def pytest_runtest_protocol(item: Function, nextitem: Function):
 
         # Add overall test results
         writer.write(f"{item.nodeid},{test_memory_usage:.3f},{full_memory_usage:.3f}\n")
+
+
+# we know this bit is bad, but we cant help it with the current pytest setup
+def pytest_addoption(parser: Parser):
+    parser.addoption("--use-subprocess", action="store_true", default=False, help="Whether to use subprocess")
+
+
+def separate_process(item: Function) -> List[TestReport]:
+    with NamedTemporaryFile(delete=False) as fp:
+        proc = subprocess.run(
+            shell=True,
+            check=False,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+            args=f"{sys.executable} -m pytest {item.nodeid} -vvv --tb=long --report-log={fp.name} --no-header --no-summary",
+            cwd=Path(__file__).parent,
+        )
+
+    reports = []
+    try:
+        for line in Path(fp.name).read_text().splitlines():
+            report_dict = json.loads(line)
+            if report_dict["$report_type"] == "TestReport":
+                reports.append(TestReport._from_json(report_dict))
+        return reports
+    finally:
+        # Force deletion of the temp file
+        Path(fp.name).unlink(missing_ok=True)
+
+
+# https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_runtest_protocol
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_call(item: Function):
+    mark = item.get_closest_marker("skip")
+    skip = mark is not None
+    mark = item.get_closest_marker("skipif")
+    skip |= mark is not None and ((len(mark.args) == 1 and mark.args[0]) or mark.kwargs.get("condition", False))
+    if not skip and item.get_closest_marker("memory_expensive") and item.config.getoption("--use-subprocess"):
+        reports = separate_process(item)
+        return reports