Wasmtime: Add (optional) bottom-up function inlining to Wasm compilation

cranelift/codegen/src/inline.rs

@@ -73,7 +73,7 @@ pub trait Inline {
     /// Failure to uphold these invariants may result in panics during
     /// compilation or incorrect runtime behavior in the generated code.
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         call_inst: ir::Inst,
         call_opcode: ir::Opcode,
@@ -82,12 +82,12 @@ pub trait Inline {
     ) -> InlineCommand<'_>;
 }
 
-impl<'a, T> Inline for &'a T
+impl<'a, T> Inline for &'a mut T
 where
     T: Inline,
 {
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         inst: ir::Inst,
         opcode: ir::Opcode,
@@ -102,7 +102,12 @@ where
 /// instruction, and inline the callee when directed to do so.
 ///
 /// Returns whether any call was inlined.
-pub(crate) fn do_inlining(func: &mut ir::Function, inliner: impl Inline) -> CodegenResult<bool> {
+pub(crate) fn do_inlining(
+    func: &mut ir::Function,
+    mut inliner: impl Inline,
+) -> CodegenResult<bool> {
+    trace!("function {} before inlining: {}", func.name, func);
+
     let mut inlined_any = false;
     let mut allocs = InliningAllocs::default();
 
@@ -175,6 +180,12 @@ pub(crate) fn do_inlining(func: &mut ir::Function, inliner: impl Inline) -> Code
         }
     }
 
+    if inlined_any {
+        trace!("function {} after inlining: {}", func.name, func);
+    } else {
+        trace!("function {} did not have any callees inlined", func.name);
+    }
+
     Ok(inlined_any)
 }
 

cranelift/codegen/src/ir/dfg.rs

@@ -333,6 +333,16 @@ impl<'a> Iterator for Values<'a> {
             .find(|kv| valid_valuedata(*kv.1))
             .map(|kv| kv.0)
     }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+impl ExactSizeIterator for Values<'_> {
+    fn len(&self) -> usize {
+        self.inner.len()
+    }
 }
 
 /// Handling values.

cranelift/entity/src/primary.rs

@@ -306,6 +306,15 @@ where
     }
 }
 
+impl<K, V> From<PrimaryMap<K, V>> for Vec<V>
+where
+    K: EntityRef,
+{
+    fn from(map: PrimaryMap<K, V>) -> Self {
+        map.elems
+    }
+}
+
 impl<K: EntityRef + fmt::Debug, V: fmt::Debug> fmt::Debug for PrimaryMap<K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut struct_ = f.debug_struct("PrimaryMap");

cranelift/filetests/src/test_inline.rs

@@ -121,7 +121,7 @@ struct Inliner<'a>(Ref<'a, HashMap<ir::UserFuncName, ir::Function>>);
 
 impl<'a> Inline for Inliner<'a> {
     fn inline(
-        &self,
+        &mut self,
         caller: &ir::Function,
         _inst: ir::Inst,
         _opcode: ir::Opcode,

crates/cli-flags/src/lib.rs

@@ -237,6 +237,19 @@ wasmtime_option_group! {
         /// object files.
         pub native_unwind_info: Option<bool>,
 
+        /// Whether to perform function inlining during compilation.
+        pub inlining: Option<bool>,
+
+        /// Configure the "small-callee" size for function inlining
+        /// heuristics. Only exposed for fuzzing.
+        #[doc(hidden)]
+        pub inlining_small_callee_size: Option<u32>,
+
+        /// Configure the "sum size threshold" for function inlining
+        /// heuristics. Only exposed for fuzzing.
+        #[doc(hidden)]
+        pub inlining_sum_size_threshold: Option<u32>,
+
         #[prefixed = "cranelift"]
         #[serde(default)]
         /// Set a cranelift-specific option. Use `wasmtime settings` to see
@@ -822,6 +835,9 @@ impl CommonOptions {
         if let Some(enable) = self.codegen.native_unwind_info {
             config.native_unwind_info(enable);
         }
+        if let Some(enable) = self.codegen.inlining {
+            config.compiler_inlining(enable);
+        }
 
         // async_stack_size enabled by either async or stack-switching, so
         // cannot directly use match_feature!

crates/cli-flags/src/opt.rs

@@ -23,6 +23,7 @@ macro_rules! wasmtime_option_group {
         pub struct $opts:ident {
             $(
                 $(#[doc = $doc:tt])*
+                $(#[doc($doc_attr:meta)])?
                 $(#[serde($serde_attr:meta)])*
                 pub $opt:ident: $container:ident<$payload:ty>,
             )+
@@ -31,6 +32,7 @@ macro_rules! wasmtime_option_group {
                 #[prefixed = $prefix:tt]
                 $(#[serde($serde_attr2:meta)])*
                 $(#[doc = $prefixed_doc:tt])*
+                $(#[doc($prefixed_doc_attr:meta)])?
                 pub $prefixed:ident: Vec<(String, Option<String>)>,
             )?
         }
@@ -43,6 +45,7 @@ macro_rules! wasmtime_option_group {
         pub struct $opts {
             $(
                 $(#[serde($serde_attr)])*
+                $(#[doc($doc_attr)])?
                 pub $opt: $container<$payload>,
             )+
             $(

crates/cranelift/src/builder.rs

@@ -65,16 +65,27 @@ impl CompilerBuilder for Builder {
 
     fn set(&mut self, name: &str, value: &str) -> Result<()> {
         // Special wasmtime-cranelift-only settings first
-        if name == "wasmtime_linkopt_padding_between_functions" {
-            self.linkopts.padding_between_functions = value.parse()?;
-            return Ok(());
+        match name {
+            "wasmtime_linkopt_padding_between_functions" => {
+                self.linkopts.padding_between_functions = value.parse()?;
+            }
+            "wasmtime_linkopt_force_jump_veneer" => {
+                self.linkopts.force_jump_veneers = value.parse()?;
+            }
+            "wasmtime_inlining_intra_module" => {
+                self.tunables.as_mut().unwrap().inlining_intra_module = value.parse()?;
+            }
+            "wasmtime_inlining_small_callee_size" => {
+                self.tunables.as_mut().unwrap().inlining_small_callee_size = value.parse()?;
+            }
+            "wasmtime_inlining_sum_size_threshold" => {
+                self.tunables.as_mut().unwrap().inlining_sum_size_threshold = value.parse()?;
+            }
+            _ => {
+                self.inner.set(name, value)?;
+            }
         }
-        if name == "wasmtime_linkopt_force_jump_veneer" {
-            self.linkopts.force_jump_veneers = value.parse()?;
-            return Ok(());
-        }
-
-        self.inner.set(name, value)
+        Ok(())
     }
 
     fn enable(&mut self, name: &str) -> Result<()> {