statetest: --trace-summary CLI option

Add --trace-summary option that only outputs a trace summary,
including the final state root.

Co-authored-by: Paweł Bylica <pawel@ethereum.org>

Author: hugo-dc

test/integration/statetest/CMakeLists.txt

@@ -79,9 +79,23 @@ add_test(
     NAME ${PREFIX}/trace
     COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace
 )
+set(EXPECTED_TRACE [[
+{"pc":0,"op":96,"gas":"0x5c878","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":2,"op":96,"gas":"0x5c875","gasCost":"0x3","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":["0x1","0x1"],"depth":1,"refund":0,"opName":"ADD"}
+{"pc":5,"op":96,"gas":"0x5c86f","gasCost":"0x3","memSize":0,"stack":["0x2"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":7,"op":85,"gas":"0x5c86c","gasCost":"0x0","memSize":0,"stack":["0x2","0x0"],"depth":1,"refund":0,"opName":"SSTORE"}
+{"pc":8,"op":0,"gas":"0x57218","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"}
+{"pass":true,"gasUsed":"0xa868","stateRoot":"0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530"}
+]])
+# Escape regex special characters.
+string(REPLACE "{" "\\{" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "}" "\\}" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "[" "\\[" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "]" "\\]" EXPECTED_TRACE ${EXPECTED_TRACE})
 set_tests_properties(
     ${PREFIX}/trace PROPERTIES
-    PASS_REGULAR_EXPRESSION [=[\{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":\["0x1","0x1"\],"depth":1,"refund":0,"opName":"ADD"\}]=]
+    PASS_REGULAR_EXPRESSION ${EXPECTED_TRACE}
 )
 
 add_test(

test/statetest/statetest.cpp

@@ -14,27 +14,28 @@ class StateTest : public testing::Test
 {
     fs::path m_json_test_file;
     evmc::VM& m_vm;
+    bool m_trace = false;
 
 public:
-    explicit StateTest(fs::path json_test_file, evmc::VM& vm) noexcept
-      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}
+    explicit StateTest(fs::path json_test_file, evmc::VM& vm, bool trace) noexcept
+      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}, m_trace{trace}
     {}
 
     void TestBody() final
     {
         std::ifstream f{m_json_test_file};
-        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm);
+        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm, m_trace);
     }
 };
 
-void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm)
+void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm, bool trace)
 {
     testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
         file.string().c_str(), 0,
-        [file, &vm]() -> testing::Test* { return new StateTest(file, vm); });
+        [file, &vm, trace]() -> testing::Test* { return new StateTest(file, vm, trace); });
 }
 
-void register_test_files(const fs::path& root, evmc::VM& vm)
+void register_test_files(const fs::path& root, evmc::VM& vm, bool trace)
 {
     if (is_directory(root))
     {
@@ -46,11 +47,11 @@ void register_test_files(const fs::path& root, evmc::VM& vm)
         std::sort(test_files.begin(), test_files.end());
 
         for (const auto& p : test_files)
-            register_test(fs::relative(p, root).parent_path().string(), p, vm);
+            register_test(fs::relative(p, root).parent_path().string(), p, vm, trace);
     }
     else  // Treat as a file.
     {
-        register_test(root.parent_path().string(), root, vm);
+        register_test(root.parent_path().string(), root, vm, trace);
     }
 }
 }  // namespace
@@ -80,18 +81,21 @@ int main(int argc, char* argv[])
             ->required()
             ->check(CLI::ExistingPath);
 
-        bool trace_flag = false;
-        app.add_flag("--trace", trace_flag, "Enable EVM tracing");
+        bool trace = false;
+        bool trace_summary = false;
+        const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing");
+        app.add_flag("--trace-summary", trace_summary, "Output trace summary only")
+            ->excludes(trace_opt);
 
         CLI11_PARSE(app, argc, argv);
 
         evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}};
 
-        if (trace_flag)
+        if (trace)
             vm.set_option("trace", "1");
 
         for (const auto& p : paths)
-            register_test_files(p, vm);
+            register_test_files(p, vm, trace || trace_summary);
 
         return RUN_ALL_TESTS();
     }

test/statetest/statetest.hpp

@@ -92,7 +92,10 @@ StateTransitionTest load_state_test(std::istream& input);
 /// Throws exception on any invalid EOF in state.
 void validate_deployed_code(const state::State& state, evmc_revision rev);
 
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm);
+/// Execute the state @p test using the @p vm.
+///
+/// @param trace_summary  Output execution summary to the default trace stream.
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary);
 
 /// Computes the hash of the RLP-encoded list of transaction logs.
 /// This method is only used in tests.

test/statetest/statetest_runner.cpp

@@ -9,7 +9,7 @@
 
 namespace evmone::test
 {
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary)
 {
     for (const auto& [rev, cases] : test.cases)
     {
@@ -33,12 +33,29 @@ void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
             // Finalize block with reward 0.
             state::finalize(state, rev, test.block.coinbase, 0, {}, {});
 
+            const auto state_root = state::mpt_hash(state.get_accounts());
+
+            if (trace_summary)
+            {
+                std::clog << '{';
+                if (holds_alternative<state::TransactionReceipt>(res))  // if tx valid
+                {
+                    const auto& r = get<state::TransactionReceipt>(res);
+                    if (r.status == EVMC_SUCCESS)
+                        std::clog << R"("pass":true)";
+                    else
+                        std::clog << R"("pass":false,"error":")" << r.status << '"';
+                    std::clog << R"(,"gasUsed":"0x)" << std::hex << r.gas_used << R"(",)";
+                }
+                std::clog << R"("stateRoot":"0x)" << hex(state_root) << "\"}\n";
+            }
+
             if (holds_alternative<state::TransactionReceipt>(res))
                 EXPECT_EQ(logs_hash(get<state::TransactionReceipt>(res).logs), expected.logs_hash);
             else
                 EXPECT_TRUE(expected.exception);
 
-            EXPECT_EQ(state::mpt_hash(state.get_accounts()), expected.state_hash);
+            EXPECT_EQ(state_root, expected.state_hash);
         }
     }
 }