refactor(tests): Use pytest collection to load JSON fixtures

src/ethereum_spec_tools/evm_tools/statetest/__init__.py

@@ -9,7 +9,7 @@
 from copy import deepcopy
 from dataclasses import dataclass
 from io import StringIO
-from typing import Any, Dict, Iterable, List, Optional, TextIO
+from typing import Any, Dict, Generator, Iterable, List, Optional, TextIO
 
 from ethereum.utils.hexadecimal import hex_to_bytes
 
@@ -35,6 +35,41 @@ class TestCase:
     transaction: Dict
 
 
+def read_test_case(
+    test_file_path: str, key: str, test: Dict[str, Any]
+) -> Generator[TestCase, None, None]:
+    """
+    Given a key and a value, return a `TestCase` object.
+    """
+    env = test["env"]
+    if not isinstance(env, dict):
+        raise TypeError("env not dict")
+
+    pre = test["pre"]
+    if not isinstance(pre, dict):
+        raise TypeError("pre not dict")
+
+    transaction = test["transaction"]
+    if not isinstance(transaction, dict):
+        raise TypeError("transaction not dict")
+
+    for fork_name, content in test["post"].items():
+        for idx, post in enumerate(content):
+            if not isinstance(post, dict):
+                raise TypeError(f'post["{fork_name}"] not dict')
+
+            yield TestCase(
+                path=test_file_path,
+                key=key,
+                index=idx,
+                fork_name=fork_name,
+                post=post,
+                env=env,
+                pre=pre,
+                transaction=transaction,
+            )
+
+
 def read_test_cases(test_file_path: str) -> Iterable[TestCase]:
     """
     Given a path to a filled state test in JSON format, return all the
@@ -44,33 +79,7 @@ def read_test_cases(test_file_path: str) -> Iterable[TestCase]:
         tests = json.load(test_file)
 
     for key, test in tests.items():
-        env = test["env"]
-        if not isinstance(env, dict):
-            raise TypeError("env not dict")
-
-        pre = test["pre"]
-        if not isinstance(pre, dict):
-            raise TypeError("pre not dict")
-
-        transaction = test["transaction"]
-        if not isinstance(transaction, dict):
-            raise TypeError("transaction not dict")
-
-        for fork_name, content in test["post"].items():
-            for idx, post in enumerate(content):
-                if not isinstance(post, dict):
-                    raise TypeError(f'post["{fork_name}"] not dict')
-
-                yield TestCase(
-                    path=test_file_path,
-                    key=key,
-                    index=idx,
-                    fork_name=fork_name,
-                    post=post,
-                    env=env,
-                    pre=pre,
-                    transaction=transaction,
-                )
+        yield from read_test_case(test_file_path, key, test)
 
 
 def run_test_case(

tests/json_infra/conftest.py

@@ -3,8 +3,15 @@
 import os
 import shutil
 import tarfile
+from glob import glob
 from pathlib import Path
-from typing import Callable, Final, Optional, Set
+from typing import (
+    Callable,
+    Final,
+    Optional,
+    Self,
+    Set,
+)
 
 import git
 import requests_cache
@@ -13,12 +20,12 @@
 from _pytest.nodes import Item
 from filelock import FileLock
 from git.exc import GitCommandError, InvalidGitRepositoryError
-from pytest import Session, StashKey, fixture
+from pytest import Collector, Session, StashKey, fixture
 from requests_cache import CachedSession
 from requests_cache.backends.sqlite import SQLiteCache
-from typing_extensions import Self
 
 from . import TEST_FIXTURES
+from .helpers import FixturesFile, FixtureTestItem
 
 try:
     from xdist import get_xdist_worker_id
@@ -260,6 +267,17 @@ def pytest_sessionstart(session: Session) -> None:
                     fixture_path,
                 )
 
+            # Remove any python files in the downloaded files to avoid
+            # importing them.
+            for python_file in glob(
+                os.path.join(fixture_path, "**/*.py"), recursive=True
+            ):
+                try:
+                    os.unlink(python_file)
+                except FileNotFoundError:
+                    # Not breaking error, another process deleted it first
+                    pass
+
 
 def pytest_sessionfinish(session: Session, exitstatus: int) -> None:
     """Clean up file locks at session finish."""
@@ -272,3 +290,28 @@ def pytest_sessionfinish(session: Session, exitstatus: int) -> None:
 
     assert lock_file is not None
     lock_file.release()
+
+
+def pytest_collect_file(
+    file_path: Path, parent: Collector
+) -> Collector | None:
+    """
+    Pytest hook that collects test cases from fixture JSON files.
+    """
+    if file_path.suffix == ".json":
+        return FixturesFile.from_parent(parent, path=file_path)
+    return None
+
+
+def pytest_runtest_teardown(item: Item, nextitem: Item) -> None:
+    """
+    Drop cache from a `FixtureTestItem` if the next one is not of the
+    same type or does not belong to the same fixtures file.
+    """
+    if isinstance(item, FixtureTestItem):
+        if (
+            nextitem is None
+            or not isinstance(nextitem, FixtureTestItem)
+            or item.fixtures_file != nextitem.fixtures_file
+        ):
+            item.fixtures_file.clear_data_cache()

tests/json_infra/helpers/__init__.py

@@ -1 +1,10 @@
 """Helpers to load tests from JSON files."""
+
+from .fixtures import ALL_FIXTURE_TYPES, Fixture, FixturesFile, FixtureTestItem
+from .load_blockchain_tests import BlockchainTestFixture
+from .load_state_tests import StateTestFixture
+
+ALL_FIXTURE_TYPES.append(BlockchainTestFixture)
+ALL_FIXTURE_TYPES.append(StateTestFixture)
+
+__all__ = ["ALL_FIXTURE_TYPES", "Fixture", "FixturesFile", "FixtureTestItem"]

tests/json_infra/helpers/fixtures.py

@@ -0,0 +1,105 @@
+"""Base class for all fixture loaders."""
+
+import json
+from abc import ABC, abstractmethod
+from functools import cached_property
+from typing import Any, Dict, Generator, List, Self, Type
+
+from _pytest.nodes import Node
+from pytest import Collector, File, Item
+
+
+class FixtureTestItem(Item):
+    """
+    Test item that comes from a fixture file.
+    """
+
+    @property
+    def fixtures_file(self) -> "FixturesFile":
+        """Return the fixtures file from which the test was extracted."""
+        raise NotImplementedError()
+
+
+class Fixture(ABC):
+    """
+    Single fixture from a JSON file.
+
+    It can be subclassed in combination with Item or Collector to create a
+    fixture that can be collected by pytest.
+    """
+
+    test_file: str
+    test_key: str
+
+    def __init__(
+        self,
+        *args: Any,
+        test_file: str,
+        test_key: str,
+        **kwargs: Any,
+    ):
+        super().__init__(*args, **kwargs)
+        self.test_file = test_file
+        self.test_key = test_key
+
+    @classmethod
+    def from_parent(
+        cls,
+        parent: Node,
+        **kwargs: Any,
+    ) -> Self:
+        """Pytest hook that returns a fixture from a JSON file."""
+        return super().from_parent(  # type: ignore[misc]
+            parent=parent, **kwargs
+        )
+
+    @classmethod
+    @abstractmethod
+    def is_format(cls, test_dict: Dict[str, Any]) -> bool:
+        """Return true if the object can be parsed as the fixture type."""
+        pass
+
+
+ALL_FIXTURE_TYPES: List[Type[Fixture]] = []
+
+
+class FixturesFile(File):
+    """Single JSON file containing fixtures."""
+
+    @cached_property
+    def data(self) -> Dict[str, Any]:
+        """Return the JSON data of the full file."""
+        # loaded once per worker per file (thanks to cached_property)
+        with self.fspath.open("r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def clear_data_cache(self) -> None:
+        """Drop the data cache."""
+        self.__dict__.pop("data", None)
+
+    def collect(
+        self: Self,
+    ) -> Generator[Item | Collector, None, None]:
+        """Collect test cases from a single JSON fixtures file."""
+        try:
+            loaded_file = self.data
+        except Exception:
+            return  # Skip *.json files that are unreadable.
+        if isinstance(loaded_file, dict):
+            for key, test_dict in loaded_file.items():
+                if not isinstance(test_dict, dict):
+                    continue
+                for fixture_type in ALL_FIXTURE_TYPES:
+                    if not fixture_type.is_format(test_dict):
+                        continue
+                    name = key
+                    if "::" in name:
+                        name = name.split("::")[1]
+                    yield fixture_type.from_parent(  # type: ignore
+                        parent=self,
+                        name=name,
+                        test_file=str(self.path),
+                        test_key=key,
+                    )
+        # Make sure we don't keep anything from collection in memory.
+        self.clear_data_cache()