Add virtualization host functions

.gitlab/pipeline/test.yml

@@ -17,6 +17,7 @@ test-linux-stable:
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+    RUST_LOG: "polkavm=trace"
   parallel: 3
   script:
     # Build all but only execute 'runtime' tests.
@@ -30,7 +31,9 @@ test-linux-stable:
         --release \
         --no-fail-fast \
         --features try-runtime,experimental,riscv,ci-only-tests \
-        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+        --retries 0\
+        -j 1\
+        test_virtualization
     # Upload tests results to Elasticsearch
     - echo "Upload test results to Elasticsearch"
     - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json

Cargo.toml

@@ -480,6 +480,8 @@ members = [
 	"substrate/primitives/trie",
 	"substrate/primitives/version",
 	"substrate/primitives/version/proc-macro",
+	"substrate/primitives/virtualization",
+	"substrate/primitives/virtualization/test-fixture",
 	"substrate/primitives/wasm-interface",
 	"substrate/primitives/weights",
 	"substrate/scripts/ci/node-template-release",
@@ -537,8 +539,9 @@ extra-unused-type-parameters = { level = "allow", priority = 2 }     # stylistic
 default_constructed_unit_structs = { level = "allow", priority = 2 } # stylistic
 
 [workspace.dependencies]
-polkavm-linker = "0.8.2"
-polkavm-derive = "0.8.0"
+polkavm = "0.9.0"
+polkavm-linker = "0.9.0"
+polkavm-derive = "0.9.1"
 log = { version = "0.4.20", default-features = false }
 quote = { version = "1.0.33" }
 serde = { version = "1.0.197", default-features = false }

prdoc/pr_3520.prdoc

@@ -0,0 +1,12 @@
+title: Add virtualization host functions
+
+doc:
+  - audience: Node Dev
+  - audience: Runtime Dev
+    description: |
+      Add experimental support for virtualization host functions. Those allow the runtime
+      to spawn and run PolkaVM instances. Not for production use, yet.
+
+crates:
+  - name: sp-virtualization
+  - name: sc-executor-wasmtime

substrate/bin/node/cli/Cargo.toml

@@ -67,6 +67,7 @@ sp-io = { path = "../../../primitives/io" }
 sp-mixnet = { path = "../../../primitives/mixnet" }
 sp-mmr-primitives = { path = "../../../primitives/merkle-mountain-range" }
 sp-statement-store = { path = "../../../primitives/statement-store" }
+sp-virtualization = { path = "../../../primitives/virtualization" }
 
 # client dependencies
 sc-client-api = { path = "../../../client/api" }

substrate/bin/node/cli/src/service.rs

@@ -42,14 +42,20 @@ use std::{path::Path, sync::Arc};
 
 /// Host functions required for kitchensink runtime and Substrate node.
 #[cfg(not(feature = "runtime-benchmarks"))]
-pub type HostFunctions =
-	(sp_io::SubstrateHostFunctions, sp_statement_store::runtime_api::HostFunctions);
+pub type HostFunctions = (
+	sp_io::SubstrateHostFunctions,
+	sp_statement_store::runtime_api::HostFunctions,
+	// Experimental: Don't activate in your production runtime.
+	sp_virtualization::host_fn::HostFunctions,
+);
 
 /// Host functions required for kitchensink runtime and Substrate node.
 #[cfg(feature = "runtime-benchmarks")]
 pub type HostFunctions = (
 	sp_io::SubstrateHostFunctions,
 	sp_statement_store::runtime_api::HostFunctions,
+	// Experimental: Don't activate in your production runtime.
+	sp_virtualization::host_fn::HostFunctions,
 	frame_benchmarking::benchmarking::HostFunctions,
 );
 

substrate/client/executor/Cargo.toml

@@ -46,6 +46,8 @@ sp-runtime = { path = "../../primitives/runtime" }
 sp-maybe-compressed-blob = { path = "../../primitives/maybe-compressed-blob" }
 sc-tracing = { path = "../tracing" }
 sp-tracing = { path = "../../primitives/tracing" }
+sp-virtualization = { path = "../../primitives/virtualization" }
+sp-virtualization-test-fixture = { path = "../../primitives/virtualization/test-fixture" }
 tracing-subscriber = "0.2.19"
 paste = "1.0"
 regex = "1.6.0"

substrate/client/executor/runtime-test/Cargo.toml

@@ -21,6 +21,7 @@ sp-io = { path = "../../../primitives/io", default-features = false, features =
 sp-runtime = { path = "../../../primitives/runtime", default-features = false }
 sp-runtime-interface = { path = "../../../primitives/runtime-interface", default-features = false }
 sp-std = { path = "../../../primitives/std", default-features = false }
+sp-virtualization = { path = "../../../primitives/virtualization", default-features = false }
 
 [build-dependencies]
 substrate-wasm-builder = { path = "../../../utils/wasm-builder", optional = true }
@@ -33,5 +34,6 @@ std = [
 	"sp-runtime-interface/std",
 	"sp-runtime/std",
 	"sp-std/std",
+	"sp-virtualization/std",
 	"substrate-wasm-builder",
 ]

substrate/client/executor/runtime-test/src/lib.rs

@@ -367,6 +367,10 @@ sp_core::wasm_export_functions! {
 		// Mainly a test that the macro is working when we have a return statement here.
 		return 1234;
 	}
+
+	fn test_virtualization(test_fixture: Vec<u8>) {
+		sp_virtualization::run_tests(test_fixture.as_ref());
+	}
 }
 
 // Tests that check output validity. We explicitly return the ptr and len, so we avoid using the

substrate/client/executor/src/integration_tests/mod.rs

@@ -41,7 +41,7 @@ use tracing_subscriber::layer::SubscriberExt;
 use crate::WasmExecutionMethod;
 
 pub type TestExternalities = CoreTestExternalities<BlakeTwo256>;
-type HostFunctions = sp_io::SubstrateHostFunctions;
+type HostFunctions = (sp_io::SubstrateHostFunctions, sp_virtualization::host_fn::HostFunctions);
 
 /// Simple macro that runs a given method as test with the available wasm execution methods.
 #[macro_export]
@@ -799,3 +799,12 @@ fn return_overflow(wasm_method: WasmExecutionMethod) {
 		error => panic!("unexpected error: {:?}", error),
 	}
 }
+
+test_wasm_execution!(test_virtualization);
+fn test_virtualization(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let fixture = sp_virtualization_test_fixture::binary().encode();
+
+	call_in_wasm("test_virtualization", fixture.as_ref(), wasm_method, &mut ext).unwrap();
+}

substrate/client/executor/wasmtime/Cargo.toml

@@ -34,6 +34,7 @@ anyhow = "1.0.68"
 sc-allocator = { path = "../../allocator" }
 sc-executor-common = { path = "../common" }
 sp-runtime-interface = { path = "../../../primitives/runtime-interface" }
+sp-virtualization = { path = "../../../primitives/virtualization" }
 sp-wasm-interface = { path = "../../../primitives/wasm-interface", features = ["wasmtime"] }
 
 # Here we include the rustix crate in the exactly same semver-compatible version as used by