Wasmtime: Add (optional) bottom-up function inlining to Wasm compilation (bytecodealliance#11283)

* Wasmtime: Add (optional) bottom-up function inlining to Wasm compilation

This commit plumbs together two pieces of recently-added infrastructure:

1. function inlining in Cranelift, and
2. the parallel bottom-up inlining scheduler in Wasmtime.

Sprinkle some very simple inlining heuristics on top, and this gives us function
inlining in Wasm compilation.

The default Wasmtime configuration does not enable inlining, and when we do
enable it, we only enable it for cross-component calls by default (since
presumably the toolchain that produced a particular core Wasm module, like LLVM,
already performed any inlining that was beneficial within that module, but that
toolchain couldn't know how that Wasm module would be getting linked together
with other modules via component composition, and so it could not have done any
cross-component inlining). For what it is worth, there is a config knob to
enable intra-module function inlining, but this is primarily for use by our
fuzzers, so that they can easily excercise and explore this new inlining
functionality.

All this plumbing required some changes to the `wasmtime_environ::Compiler`
trait, since Winch cannot do inlining but Cranelift can. This is mostly
encapsulated in the new `wasmtime_environ::InliningCompiler` trait, for the most
part. Additionally, we take care not to construct the call graph, or any other
data structures required only by the inliner and not regular compilation, both
when using Winch and when using Cranelift with inlining disabled.

Finally, we add a `disas` test to verify that we successfully inline a series of
calls from a function in one component, to a cross-component adapter function,
to a function in another component. Most test coverage is expected to come from
our fuzzing, however.

* Fix dead code warning when not `cfg(feature = "component-model")`

* fix winch trampoline compilation

* Move CLI options to codegen

* Move parameters into struct

* Use an index set for call-graph construction

* Smuggle inlining heuristic options through cranelift flags

* Remove old CLI flags

* set tunables before settings

* Only configure inlining options for cranelift in fuzzing

Author: fitzgen

cranelift/codegen/src/inline.rs

@@ -73,7 +73,7 @@ pub trait Inline {
     /// Failure to uphold these invariants may result in panics during
     /// compilation or incorrect runtime behavior in the generated code.
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         call_inst: ir::Inst,
         call_opcode: ir::Opcode,
@@ -82,12 +82,12 @@ pub trait Inline {
     ) -> InlineCommand<'_>;
 }
 
-impl<'a, T> Inline for &'a T
+impl<'a, T> Inline for &'a mut T
 where
     T: Inline,
 {
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         inst: ir::Inst,
         opcode: ir::Opcode,
@@ -102,7 +102,12 @@ where
 /// instruction, and inline the callee when directed to do so.
 ///
 /// Returns whether any call was inlined.
-pub(crate) fn do_inlining(func: &mut ir::Function, inliner: impl Inline) -> CodegenResult<bool> {
+pub(crate) fn do_inlining(
+    func: &mut ir::Function,
+    mut inliner: impl Inline,
+) -> CodegenResult<bool> {
+    trace!("function {} before inlining: {}", func.name, func);
+
     let mut inlined_any = false;
     let mut allocs = InliningAllocs::default();
 
@@ -175,6 +180,12 @@ pub(crate) fn do_inlining(func: &mut ir::Function, inliner: impl Inline) -> Code
         }
     }
 
+    if inlined_any {
+        trace!("function {} after inlining: {}", func.name, func);
+    } else {
+        trace!("function {} did not have any callees inlined", func.name);
+    }
+
     Ok(inlined_any)
 }
 

cranelift/codegen/src/ir/dfg.rs

@@ -333,6 +333,16 @@ impl<'a> Iterator for Values<'a> {
             .find(|kv| valid_valuedata(*kv.1))
             .map(|kv| kv.0)
     }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+impl ExactSizeIterator for Values<'_> {
+    fn len(&self) -> usize {
+        self.inner.len()
+    }
 }
 
 /// Handling values.

cranelift/entity/src/primary.rs

@@ -306,6 +306,15 @@ where
     }
 }
 
+impl<K, V> From<PrimaryMap<K, V>> for Vec<V>
+where
+    K: EntityRef,
+{
+    fn from(map: PrimaryMap<K, V>) -> Self {
+        map.elems
+    }
+}
+
 impl<K: EntityRef + fmt::Debug, V: fmt::Debug> fmt::Debug for PrimaryMap<K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut struct_ = f.debug_struct("PrimaryMap");

cranelift/filetests/src/test_inline.rs

@@ -121,7 +121,7 @@ struct Inliner<'a>(Ref<'a, HashMap<ir::UserFuncName, ir::Function>>);
 
 impl<'a> Inline for Inliner<'a> {
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         _inst: ir::Inst,
         _opcode: ir::Opcode,

crates/cli-flags/src/lib.rs

@@ -237,6 +237,9 @@ wasmtime_option_group! {
         /// object files.
         pub native_unwind_info: Option<bool>,
 
+        /// Whether to perform function inlining during compilation.
+        pub inlining: Option<bool>,
+
         #[prefixed = "cranelift"]
         #[serde(default)]
         /// Set a cranelift-specific option. Use `wasmtime settings` to see
@@ -822,6 +825,9 @@ impl CommonOptions {
         if let Some(enable) = self.codegen.native_unwind_info {
             config.native_unwind_info(enable);
         }
+        if let Some(enable) = self.codegen.inlining {
+            config.compiler_inlining(enable);
+        }
 
         // async_stack_size enabled by either async or stack-switching, so
         // cannot directly use match_feature!

crates/cli-flags/src/opt.rs

@@ -23,6 +23,7 @@ macro_rules! wasmtime_option_group {
         pub struct $opts:ident {
             $(
                 $(#[doc = $doc:tt])*
+                $(#[doc($doc_attr:meta)])?
                 $(#[serde($serde_attr:meta)])*
                 pub $opt:ident: $container:ident<$payload:ty>,
             )+
@@ -31,6 +32,7 @@ macro_rules! wasmtime_option_group {
                 #[prefixed = $prefix:tt]
                 $(#[serde($serde_attr2:meta)])*
                 $(#[doc = $prefixed_doc:tt])*
+                $(#[doc($prefixed_doc_attr:meta)])?
                 pub $prefixed:ident: Vec<(String, Option<String>)>,
             )?
         }
@@ -43,6 +45,7 @@ macro_rules! wasmtime_option_group {
         pub struct $opts {
             $(
                 $(#[serde($serde_attr)])*
+                $(#[doc($doc_attr)])?
                 pub $opt: $container<$payload>,
             )+
             $(

crates/cranelift/src/builder.rs

@@ -65,16 +65,27 @@ impl CompilerBuilder for Builder {
 
     fn set(&mut self, name: &str, value: &str) -> Result<()> {
         // Special wasmtime-cranelift-only settings first
-        if name == "wasmtime_linkopt_padding_between_functions" {
-            self.linkopts.padding_between_functions = value.parse()?;
-            return Ok(());
+        match name {
+            "wasmtime_linkopt_padding_between_functions" => {
+                self.linkopts.padding_between_functions = value.parse()?;
+            }
+            "wasmtime_linkopt_force_jump_veneer" => {
+                self.linkopts.force_jump_veneers = value.parse()?;
+            }
+            "wasmtime_inlining_intra_module" => {
+                self.tunables.as_mut().unwrap().inlining_intra_module = value.parse()?;
+            }
+            "wasmtime_inlining_small_callee_size" => {
+                self.tunables.as_mut().unwrap().inlining_small_callee_size = value.parse()?;
+            }
+            "wasmtime_inlining_sum_size_threshold" => {
+                self.tunables.as_mut().unwrap().inlining_sum_size_threshold = value.parse()?;
+            }
+            _ => {
+                self.inner.set(name, value)?;
+            }
         }
-        if name == "wasmtime_linkopt_force_jump_veneer" {
-            self.linkopts.force_jump_veneers = value.parse()?;
-            return Ok(());
-        }
-
-        self.inner.set(name, value)
+        Ok(())
     }
 
     fn enable(&mut self, name: &str) -> Result<()> {