Skip to content

Commit afaa270

Browse files
marioevzgurukamath
marioevz
and
gurukamath
authored
refactor(tests): Use pytest collection to load JSON fixtures (#1666)
* refactor(tests): Refactor json_infra using `pytest_collect_file` * fix(tests): json collecting * fix(tests): blockchain test execution * fix(tests): blockchain test execution * refactor(tests): Refactor types in json_infra * fix(tests): json_infra, imports, parse `exceptions` in some tests * refactor(tests): move some definitions * fix(tox.ini): Remove `--ignore-glob` * fix(tests): workaround for FileNotFoundError * fix(tests): revamp cache fix(tests): Don't cache fixtures Try to implement cache Fix caching feat(tests): Manage cache during execution * fix(tox): Use `--dist=loadfile` * fix(tests): json files cache * Run selective tests based on changed files (#1) * fix(tests): remove evm_tools marker from blockchain tests * remove coverage from json_infra * enhance(tools): add json_test_name to Hardfork * fix(tests): handle failing transactions in state tests * enhance(tests): add from and until fork option to json_infra * enhance(tests): run json_infra selectively * enhance(tests): subclass Hardfork * bug(tests): run all tests for t8n changes * enhance(tests): minor fix * fix(tests): ignore expectSection tests and add coverage * enhance(tests): refactor exception markers This commit refactors exception markers and marks the EEST static tests as slow * fix(tests): provide unique name to tests * fix(tests): post review changes * fix(tests): set BASE_SHA to merge base --------- Co-authored-by: Guruprasad Kamath <[email protected]> Co-authored-by: Guruprasad Kamath <[email protected]>
1 parent 0f188fe commit afaa270

File tree

17 files changed

+828
-551
lines changed

17 files changed

+828
-551
lines changed

‎.github/workflows/test.yaml‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,35 @@ jobs:
8888
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955
8989
with:
9090
submodules: recursive
91+
fetch-depth: 0 # Fetch full history for commit comparison
9192
- name: Setup Python
9293
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
9394
with:
9495
python-version: "3.11"
9596
- uses: ./.github/actions/setup-env
97+
- name: Get changed files and save to disk
98+
id: get-changed-files
99+
run: |
100+
if [ "${{ github.event_name }}" = "pull_request" ]; then
101+
BASE_SHA=$(git merge-base "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")
102+
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
103+
else
104+
# On push or force push to the feature branch
105+
BASE_SHA=$(git merge-base "${{ github.event.before }}" "${{ github.sha }}")
106+
HEAD_SHA="${{ github.sha }}"
107+
fi
108+
109+
echo "Diffing commits: $BASE_SHA..$HEAD_SHA"
110+
111+
# Get changed files and save to disk
112+
FILE_LIST="changed_files.txt"
113+
git diff --name-only "$BASE_SHA" "$HEAD_SHA" > "$FILE_LIST"
114+
echo "Changed files saved to $FILE_LIST"
115+
echo "file_list=$FILE_LIST" >> $GITHUB_OUTPUT
116+
echo "List of files changed in the PR"
117+
cat $FILE_LIST
96118
- name: Run json infra tests
97-
run: tox -e json_infra
119+
run: tox -e json_infra -- --file-list="${{ steps.get-changed-files.outputs.file_list }}"
98120
- name: Upload coverage reports to Codecov
99121
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7
100122
with:

‎src/ethereum_spec_tools/evm_tools/statetest/__init__.py‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from copy import deepcopy
1010
from dataclasses import dataclass
1111
from io import StringIO
12-
from typing import Any, Dict, Iterable, List, Optional, TextIO
12+
from typing import Any, Dict, Generator, Iterable, List, Optional, TextIO
1313

1414
from ethereum.utils.hexadecimal import hex_to_bytes
1515

@@ -35,6 +35,41 @@ class TestCase:
3535
transaction: Dict
3636

3737

38+
def read_test_case(
39+
test_file_path: str, key: str, test: Dict[str, Any]
40+
) -> Generator[TestCase, None, None]:
41+
"""
42+
Given a key and a value, return a `TestCase` object.
43+
"""
44+
env = test["env"]
45+
if not isinstance(env, dict):
46+
raise TypeError("env not dict")
47+
48+
pre = test["pre"]
49+
if not isinstance(pre, dict):
50+
raise TypeError("pre not dict")
51+
52+
transaction = test["transaction"]
53+
if not isinstance(transaction, dict):
54+
raise TypeError("transaction not dict")
55+
56+
for fork_name, content in test["post"].items():
57+
for idx, post in enumerate(content):
58+
if not isinstance(post, dict):
59+
raise TypeError(f'post["{fork_name}"] not dict')
60+
61+
yield TestCase(
62+
path=test_file_path,
63+
key=key,
64+
index=idx,
65+
fork_name=fork_name,
66+
post=post,
67+
env=env,
68+
pre=pre,
69+
transaction=transaction,
70+
)
71+
72+
3873
def read_test_cases(test_file_path: str) -> Iterable[TestCase]:
3974
"""
4075
Given a path to a filled state test in JSON format, return all the
@@ -44,33 +79,7 @@ def read_test_cases(test_file_path: str) -> Iterable[TestCase]:
4479
tests = json.load(test_file)
4580

4681
for key, test in tests.items():
47-
env = test["env"]
48-
if not isinstance(env, dict):
49-
raise TypeError("env not dict")
50-
51-
pre = test["pre"]
52-
if not isinstance(pre, dict):
53-
raise TypeError("pre not dict")
54-
55-
transaction = test["transaction"]
56-
if not isinstance(transaction, dict):
57-
raise TypeError("transaction not dict")
58-
59-
for fork_name, content in test["post"].items():
60-
for idx, post in enumerate(content):
61-
if not isinstance(post, dict):
62-
raise TypeError(f'post["{fork_name}"] not dict')
63-
64-
yield TestCase(
65-
path=test_file_path,
66-
key=key,
67-
index=idx,
68-
fork_name=fork_name,
69-
post=post,
70-
env=env,
71-
pre=pre,
72-
transaction=transaction,
73-
)
82+
yield from read_test_case(test_file_path, key, test)
7483

7584

7685
def run_test_case(

‎tests/json_infra/__init__.py‎

Lines changed: 5 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
"""Tests related to json infrastructure."""
22

3-
from typing import Dict, Optional, TypedDict
3+
from typing import Dict, TypedDict
44

55
from typing_extensions import NotRequired
66

7+
from .hardfork import TestHardfork
8+
79

810
class _FixtureSource(TypedDict):
911
url: str
@@ -31,113 +33,6 @@ class _FixtureSource(TypedDict):
3133
}
3234

3335

34-
def _get_fixture_path(key: str) -> str:
35-
return TEST_FIXTURES[key]["fixture_path"]
36-
37-
38-
def _build_ethereum_test_paths(
39-
base_path: str, legacy_fork: Optional[str] = None
40-
) -> tuple:
41-
if legacy_fork:
42-
bc_path = f"{base_path}/LegacyTests/{legacy_fork}/BlockchainTests/"
43-
state_path = (
44-
f"{base_path}/LegacyTests/{legacy_fork}/GeneralStateTests/"
45-
)
46-
else:
47-
bc_path = f"{base_path}/BlockchainTests/"
48-
state_path = f"{base_path}/GeneralStateTests/"
49-
return bc_path, state_path
50-
51-
52-
def _build_eest_test_paths(base_path: str) -> tuple:
53-
bc_path = f"{base_path}/fixtures/blockchain_tests/"
54-
state_path = f"{base_path}/fixtures/state_tests/"
55-
return bc_path, state_path
56-
57-
58-
# Base paths
59-
ETHEREUM_TESTS_BASE = _get_fixture_path("ethereum_tests")
60-
EEST_TESTS_BASE = _get_fixture_path("latest_fork_tests")
61-
62-
# Ethereum test paths
63-
(
64-
PRE_CONSTANTINOPLE_BC_ETHEREUM_TESTS,
65-
PRE_CONSTANTINOPLE_STATE_ETHEREUM_TESTS,
66-
) = _build_ethereum_test_paths(ETHEREUM_TESTS_BASE, "Constantinople")
67-
(
68-
PRE_CANCUN_BC_ETHEREUM_TESTS,
69-
PRE_CANCUN_STATE_ETHEREUM_TESTS,
70-
) = _build_ethereum_test_paths(ETHEREUM_TESTS_BASE, "Cancun")
71-
BC_ETHEREUM_TESTS, STATE_ETHEREUM_TESTS = _build_ethereum_test_paths(
72-
ETHEREUM_TESTS_BASE
73-
)
74-
75-
# EEST test paths
76-
EEST_BC_TESTS, EEST_STATE_TESTS = _build_eest_test_paths(EEST_TESTS_BASE)
77-
78-
ForkConfig = TypedDict(
79-
"ForkConfig",
80-
{
81-
"eels_fork": str,
82-
"blockchain_test_dirs": list[str],
83-
"state_test_dirs": list[str],
84-
},
85-
)
86-
87-
88-
def _create_fork_config(
89-
eels_fork: str, bc_dirs: list, state_dirs: list
90-
) -> ForkConfig:
91-
return {
92-
"eels_fork": eels_fork,
93-
"blockchain_test_dirs": bc_dirs,
94-
"state_test_dirs": state_dirs,
95-
}
96-
97-
98-
PRE_CONSTANTINOPLE_DIRS = (
99-
[PRE_CONSTANTINOPLE_BC_ETHEREUM_TESTS, EEST_BC_TESTS],
100-
[PRE_CONSTANTINOPLE_STATE_ETHEREUM_TESTS, EEST_STATE_TESTS],
101-
)
102-
103-
PRE_CANCUN_DIRS = (
104-
[PRE_CANCUN_BC_ETHEREUM_TESTS, EEST_BC_TESTS],
105-
[PRE_CANCUN_STATE_ETHEREUM_TESTS, EEST_STATE_TESTS],
106-
)
107-
108-
CURRENT_DIRS = (
109-
[BC_ETHEREUM_TESTS, EEST_BC_TESTS],
110-
[STATE_ETHEREUM_TESTS, EEST_STATE_TESTS],
111-
)
112-
113-
FORKS: Dict[str, ForkConfig] = {
114-
**{
115-
json_fork: _create_fork_config(eels_fork, *PRE_CONSTANTINOPLE_DIRS)
116-
for json_fork, eels_fork in [
117-
("Frontier", "frontier"),
118-
("Homestead", "homestead"),
119-
("EIP150", "tangerine_whistle"),
120-
("EIP158", "spurious_dragon"),
121-
("Byzantium", "byzantium"),
122-
("ConstantinopleFix", "constantinople"),
123-
]
124-
},
125-
**{
126-
json_fork: _create_fork_config(eels_fork, *PRE_CANCUN_DIRS)
127-
for json_fork, eels_fork in [
128-
("Istanbul", "istanbul"),
129-
("Berlin", "berlin"),
130-
("London", "london"),
131-
("Paris", "paris"),
132-
("Shanghai", "shanghai"),
133-
]
134-
},
135-
**{
136-
json_fork: _create_fork_config(eels_fork, *CURRENT_DIRS)
137-
for json_fork, eels_fork in [
138-
("Cancun", "cancun"),
139-
("Prague", "prague"),
140-
("Osaka", "osaka"),
141-
]
142-
},
36+
FORKS: Dict[str, TestHardfork] = {
37+
fork.json_test_name: fork for fork in TestHardfork.discover()
14338
}

0 commit comments

Comments
 (0)