statetest: Add changes required to be executed in goevmlab (#658)

chfast · web-flow · commit 33c66e70f5ac · 2023-09-18T22:52:08.000+02:00
- Add `--trace-summary` CLI flag to output state root. - Make `_info` JSON section optional (goevmlab is not providing this field). - Implement convention for `Host::get_block_hash()` (done in #698).
diff --git a/test/integration/statetest/CMakeLists.txt b/test/integration/statetest/CMakeLists.txt
@@ -8,6 +8,7 @@ set(PREFIX ${PREFIX}/statetest)
 set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}/tests1)
 set(TESTS2 ${CMAKE_CURRENT_SOURCE_DIR}/tests2)
 set(TESTS_EOF ${CMAKE_CURRENT_SOURCE_DIR}/eof)
+set(TESTS_TX ${CMAKE_CURRENT_SOURCE_DIR}/tx)
 
 add_test(
     NAME ${PREFIX}/no_arguments
@@ -79,9 +80,23 @@ add_test(
     NAME ${PREFIX}/trace
     COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace
 )
+set(EXPECTED_TRACE [[
+{"pc":0,"op":96,"gas":"0x5c878","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":2,"op":96,"gas":"0x5c875","gasCost":"0x3","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":["0x1","0x1"],"depth":1,"refund":0,"opName":"ADD"}
+{"pc":5,"op":96,"gas":"0x5c86f","gasCost":"0x3","memSize":0,"stack":["0x2"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":7,"op":85,"gas":"0x5c86c","gasCost":"0x0","memSize":0,"stack":["0x2","0x0"],"depth":1,"refund":0,"opName":"SSTORE"}
+{"pc":8,"op":0,"gas":"0x57218","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"}
+{"pass":true,"gasUsed":"0xa868","stateRoot":"0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530"}
+]])
+# Escape regex special characters.
+string(REPLACE "{" "\\{" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "}" "\\}" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "[" "\\[" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "]" "\\]" EXPECTED_TRACE ${EXPECTED_TRACE})
 set_tests_properties(
     ${PREFIX}/trace PROPERTIES
-    PASS_REGULAR_EXPRESSION [=[\{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":\["0x1","0x1"\],"depth":1,"refund":0,"opName":"ADD"\}]=]
+    PASS_REGULAR_EXPRESSION ${EXPECTED_TRACE}
 )
 
 add_test(
@@ -92,3 +107,12 @@ set_tests_properties(
     ${PREFIX}/invalid_eof_in_state PROPERTIES
     PASS_REGULAR_EXPRESSION "EOF container at 0x0000000000000000000000000000000000bade0f is invalid"
 )
+
+add_test(
+    NAME ${PREFIX}/tx_invalid_nonce
+    COMMAND evmone-statetest ${TESTS_TX}/invalid_nonce.json
+)
+set_tests_properties(
+    ${PREFIX}/tx_invalid_nonce PROPERTIES
+    PASS_REGULAR_EXPRESSION "unexpected invalid transaction: nonce too high"
+)
diff --git a/test/integration/statetest/tx/invalid_nonce.json b/test/integration/statetest/tx/invalid_nonce.json
@@ -0,0 +1,42 @@
+{
+  "invalid_nonce": {
+    "env": {
+      "currentBaseFee": "0x0a",
+      "currentCoinbase": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "currentDifficulty": "0x020000",
+      "currentGasLimit": "0xff112233445566",
+      "currentNumber": "0x01",
+      "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000",
+      "currentTimestamp": "0x03e8"
+    },
+    "post": {
+      "Shanghai": [
+        {
+          "hash": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+          "indexes": {
+            "data": 0,
+            "gas": 0,
+            "value": 0
+          },
+          "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+        }
+      ]
+    },
+    "pre": {},
+    "transaction": {
+      "data": [
+        "0x"
+      ],
+      "gasLimit": [
+        "0x00"
+      ],
+      "gasPrice": "0x0a",
+      "nonce": "0x01",
+      "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "to": "0x00",
+      "value": [
+        "0x00"
+      ]
+    }
+  }
+}
diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp
@@ -14,27 +14,28 @@ class StateTest : public testing::Test
 {
     fs::path m_json_test_file;
     evmc::VM& m_vm;
+    bool m_trace = false;
 
 public:
-    explicit StateTest(fs::path json_test_file, evmc::VM& vm) noexcept
-      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}
+    explicit StateTest(fs::path json_test_file, evmc::VM& vm, bool trace) noexcept
+      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}, m_trace{trace}
     {}
 
     void TestBody() final
     {
         std::ifstream f{m_json_test_file};
-        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm);
+        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm, m_trace);
     }
 };
 
-void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm)
+void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm, bool trace)
 {
     testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
         file.string().c_str(), 0,
-        [file, &vm]() -> testing::Test* { return new StateTest(file, vm); });
+        [file, &vm, trace]() -> testing::Test* { return new StateTest(file, vm, trace); });
 }
 
-void register_test_files(const fs::path& root, evmc::VM& vm)
+void register_test_files(const fs::path& root, evmc::VM& vm, bool trace)
 {
     if (is_directory(root))
     {
@@ -46,11 +47,11 @@ void register_test_files(const fs::path& root, evmc::VM& vm)
         std::sort(test_files.begin(), test_files.end());
 
         for (const auto& p : test_files)
-            register_test(fs::relative(p, root).parent_path().string(), p, vm);
+            register_test(fs::relative(p, root).parent_path().string(), p, vm, trace);
     }
     else  // Treat as a file.
     {
-        register_test(root.parent_path().string(), root, vm);
+        register_test(root.parent_path().string(), root, vm, trace);
     }
 }
 }  // namespace
@@ -80,18 +81,21 @@ int main(int argc, char* argv[])
             ->required()
             ->check(CLI::ExistingPath);
 
-        bool trace_flag = false;
-        app.add_flag("--trace", trace_flag, "Enable EVM tracing");
+        bool trace = false;
+        bool trace_summary = false;
+        const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing");
+        app.add_flag("--trace-summary", trace_summary, "Output trace summary only")
+            ->excludes(trace_opt);
 
         CLI11_PARSE(app, argc, argv);
 
         evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}};
 
-        if (trace_flag)
+        if (trace)
             vm.set_option("trace", "1");
 
         for (const auto& p : paths)
-            register_test_files(p, vm);
+            register_test_files(p, vm, trace || trace_summary);
 
         return RUN_ALL_TESTS();
     }
diff --git a/test/statetest/statetest.hpp b/test/statetest/statetest.hpp
@@ -92,7 +92,10 @@ StateTransitionTest load_state_test(std::istream& input);
 /// Throws exception on any invalid EOF in state.
 void validate_deployed_code(const state::State& state, evmc_revision rev);
 
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm);
+/// Execute the state @p test using the @p vm.
+///
+/// @param trace_summary  Output execution summary to the default trace stream.
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary);
 
 /// Computes the hash of the RLP-encoded list of transaction logs.
 /// This method is only used in tests.
diff --git a/test/statetest/statetest_loader.cpp b/test/statetest/statetest_loader.cpp
@@ -397,10 +397,13 @@ static void from_json(const json::json& j, StateTransitionTest& o)
 
     o.block = from_json<state::BlockInfo>(j_t.at("env"));
 
-    if (const auto& info = j_t.at("_info"); info.contains("labels"))
+    if (const auto info_it = j_t.find("_info"); info_it != j_t.end())
     {
-        for (const auto& [j_id, j_label] : info.at("labels").items())
-            o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);
+        if (const auto labels_it = info_it->find("labels"); labels_it != info_it->end())
+        {
+            for (const auto& [j_id, j_label] : labels_it->items())
+                o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);
+        }
     }
 
     for (const auto& [rev_name, expectations] : j_t.at("post").items())
diff --git a/test/statetest/statetest_runner.cpp b/test/statetest/statetest_runner.cpp
@@ -9,7 +9,7 @@
 
 namespace evmone::test
 {
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary)
 {
     for (const auto& [rev, cases] : test.cases)
     {
@@ -33,12 +33,30 @@ void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
             // Finalize block with reward 0.
             state::finalize(state, rev, test.block.coinbase, 0, {}, {});
 
+            const auto state_root = state::mpt_hash(state.get_accounts());
+
+            if (trace_summary)
+            {
+                std::clog << '{';
+                if (holds_alternative<state::TransactionReceipt>(res))  // if tx valid
+                {
+                    const auto& r = get<state::TransactionReceipt>(res);
+                    if (r.status == EVMC_SUCCESS)
+                        std::clog << R"("pass":true)";
+                    else
+                        std::clog << R"("pass":false,"error":")" << r.status << '"';
+                    std::clog << R"(,"gasUsed":"0x)" << std::hex << r.gas_used << R"(",)";
+                }
+                std::clog << R"("stateRoot":"0x)" << hex(state_root) << "\"}\n";
+            }
+
             if (holds_alternative<state::TransactionReceipt>(res))
                 EXPECT_EQ(logs_hash(get<state::TransactionReceipt>(res).logs), expected.logs_hash);
             else
-                EXPECT_TRUE(expected.exception);
+                EXPECT_TRUE(expected.exception)
+                    << "unexpected invalid transaction: " << get<std::error_code>(res).message();
 
-            EXPECT_EQ(state::mpt_hash(state.get_accounts()), expected.state_hash);
+            EXPECT_EQ(state_root, expected.state_hash);
         }
     }
 }
diff --git a/test/unittests/statetest_loader_test.cpp b/test/unittests/statetest_loader_test.cpp
@@ -74,7 +74,6 @@ TEST(statetest_loader, load_minimal_test)
 {
     std::istringstream s{R"({
         "test": {
-            "_info": {},
             "pre": {},
             "transaction": {
                 "gasPrice": "",

Original file line number	Diff line number	Diff line change
`@@ -14,27 +14,28 @@ class StateTest : public testing::Test`
`14`	`14`	`{`
`15`	`15`	`fs::path m_json_test_file;`
`16`	`16`	`evmc::VM& m_vm;`
	`17`	`+ bool m_trace = false;`
`17`	`18`
`18`	`19`	`public:`
`19`		`- explicit StateTest(fs::path json_test_file, evmc::VM& vm) noexcept`
`20`		`- : m_json_test_file{std::move(json_test_file)}, m_vm{vm}`
	`20`	`+ explicit StateTest(fs::path json_test_file, evmc::VM& vm, bool trace) noexcept`
	`21`	`+ : m_json_test_file{std::move(json_test_file)}, m_vm{vm}, m_trace{trace}`
`21`	`22`	`{}`
`22`	`23`
`23`	`24`	`void TestBody() final`
`24`	`25`	`{`
`25`	`26`	`std::ifstream f{m_json_test_file};`
`26`		`- evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm);`
	`27`	`+ evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm, m_trace);`
`27`	`28`	`}`
`28`	`29`	`};`
`29`	`30`
`30`		`-void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm)`
	`31`	`+void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm, bool trace)`
`31`	`32`	`{`
`32`	`33`	`testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,`
`33`	`34`	`file.string().c_str(), 0,`
`34`		`- [file, &vm]() -> testing::Test* { return new StateTest(file, vm); });`
	`35`	`+ [file, &vm, trace]() -> testing::Test* { return new StateTest(file, vm, trace); });`
`35`	`36`	`}`
`36`	`37`
`37`		`-void register_test_files(const fs::path& root, evmc::VM& vm)`
	`38`	`+void register_test_files(const fs::path& root, evmc::VM& vm, bool trace)`
`38`	`39`	`{`
`39`	`40`	`if (is_directory(root))`
`40`	`41`	`{`
`@@ -46,11 +47,11 @@ void register_test_files(const fs::path& root, evmc::VM& vm)`
`46`	`47`	`std::sort(test_files.begin(), test_files.end());`
`47`	`48`
`48`	`49`	`for (const auto& p : test_files)`
`49`		`- register_test(fs::relative(p, root).parent_path().string(), p, vm);`
	`50`	`+ register_test(fs::relative(p, root).parent_path().string(), p, vm, trace);`
`50`	`51`	`}`
`51`	`52`	`else // Treat as a file.`
`52`	`53`	`{`
`53`		`- register_test(root.parent_path().string(), root, vm);`
	`54`	`+ register_test(root.parent_path().string(), root, vm, trace);`
`54`	`55`	`}`
`55`	`56`	`}`
`56`	`57`	`} // namespace`
`@@ -80,18 +81,21 @@ int main(int argc, char* argv[])`
`80`	`81`	`->required()`
`81`	`82`	`->check(CLI::ExistingPath);`
`82`	`83`
`83`		`- bool trace_flag = false;`
`84`		`- app.add_flag("--trace", trace_flag, "Enable EVM tracing");`
	`84`	`+ bool trace = false;`
	`85`	`+ bool trace_summary = false;`
	`86`	`+ const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing");`
	`87`	`+ app.add_flag("--trace-summary", trace_summary, "Output trace summary only")`
	`88`	`+ ->excludes(trace_opt);`
`85`	`89`
`86`	`90`	`CLI11_PARSE(app, argc, argv);`
`87`	`91`
`88`	`92`	`evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}};`
`89`	`93`
`90`		`- if (trace_flag)`
	`94`	`+ if (trace)`
`91`	`95`	`vm.set_option("trace", "1");`
`92`	`96`
`93`	`97`	`for (const auto& p : paths)`
`94`		`- register_test_files(p, vm);`
	`98`	`+ register_test_files(p, vm, trace \|\| trace_summary);`
`95`	`99`
`96`	`100`	`return RUN_ALL_TESTS();`
`97`	`101`	`}`
Original file line number	Diff line number	Diff line change
`@@ -397,10 +397,13 @@ static void from_json(const json::json& j, StateTransitionTest& o)`
`397`	`397`
`398`	`398`	`o.block = from_json<state::BlockInfo>(j_t.at("env"));`
`399`	`399`
`400`		`- if (const auto& info = j_t.at("_info"); info.contains("labels"))`
	`400`	`+ if (const auto info_it = j_t.find("_info"); info_it != j_t.end())`
`401`	`401`	`{`
`402`		`- for (const auto& [j_id, j_label] : info.at("labels").items())`
`403`		`- o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);`
	`402`	`+ if (const auto labels_it = info_it->find("labels"); labels_it != info_it->end())`
	`403`	`+ {`
	`404`	`+ for (const auto& [j_id, j_label] : labels_it->items())`
	`405`	`+ o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);`
	`406`	`+ }`
`404`	`407`	`}`
`405`	`408`
`406`	`409`	`for (const auto& [rev_name, expectations] : j_t.at("post").items())`
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,6 @@ TEST(statetest_loader, load_minimal_test)`
`74`	`74`	`{`
`75`	`75`	`std::istringstream s{R"({`
`76`	`76`	`"test": {`
`77`		`- "_info": {},`
`78`	`77`	`"pre": {},`
`79`	`78`	`"transaction": {`
`80`	`79`	`"gasPrice": "",`