Change how Instance stores instantiated memories in the runtime.

This commit changes `Instance` such that memories can be stored statically,
with just a base pointer, size, maximum, and a callback to make memory
accessible.

Previously the memories were being stored as boxed trait objects, which would
require the pooling allocator to do some unpleasant things to avoid
allocations.

With this change, the pooling allocator can simply define a memory for the
instance without using a trait object.

Author: peterhuene

crates/runtime/src/instance.rs

@@ -4,7 +4,7 @@
 
 use crate::export::Export;
 use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
-use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator};
+use crate::memory::{Memory, RuntimeMemoryCreator};
 use crate::table::{Table, TableElement};
 use crate::traphandlers::Trap;
 use crate::vmcontext::{
@@ -45,7 +45,7 @@ pub(crate) struct Instance {
     offsets: VMOffsets,
 
     /// WebAssembly linear memory data.
-    memories: PrimaryMap<DefinedMemoryIndex, Box<dyn RuntimeLinearMemory>>,
+    memories: PrimaryMap<DefinedMemoryIndex, Memory>,
 
     /// WebAssembly table data.
     tables: PrimaryMap<DefinedTableIndex, Table>,

crates/runtime/src/instance/allocator.rs

@@ -1,7 +1,7 @@
 use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
 use crate::imports::Imports;
 use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator};
-use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory};
+use crate::memory::{DefaultMemoryCreator, Memory};
 use crate::table::{Table, TableElement};
 use crate::traphandlers::Trap;
 use crate::vmcontext::{
@@ -296,8 +296,7 @@ impl OnDemandInstanceAllocator {
     fn create_memories(
         &self,
         module: &Module,
-    ) -> Result<PrimaryMap<DefinedMemoryIndex, Box<dyn RuntimeLinearMemory>>, InstantiationError>
-    {
+    ) -> Result<PrimaryMap<DefinedMemoryIndex, Memory>, InstantiationError> {
         let creator = self
             .mem_creator
             .as_deref()
@@ -306,11 +305,8 @@ impl OnDemandInstanceAllocator {
         let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
             PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
         for plan in &module.memory_plans.values().as_slice()[num_imports..] {
-            memories.push(
-                creator
-                    .new_memory(plan)
-                    .map_err(InstantiationError::Resource)?,
-            );
+            memories
+                .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?);
         }
         Ok(memories)
     }

crates/runtime/src/lib.rs

@@ -42,7 +42,7 @@ pub use crate::instance::{
     OnDemandInstanceAllocator,
 };
 pub use crate::jit_int::GdbJitImageRegistration;
-pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator};
+pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator};
 pub use crate::mmap::Mmap;
 pub use crate::table::{Table, TableElement};
 pub use crate::traphandlers::{

crates/runtime/src/memory.rs

@@ -5,7 +5,7 @@
 use crate::mmap::Mmap;
 use crate::vmcontext::VMMemoryDefinition;
 use more_asserts::{assert_ge, assert_le};
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
 use std::convert::TryFrom;
 use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE};
 
@@ -170,3 +170,116 @@ impl RuntimeLinearMemory for MmapMemory {
         }
     }
 }
+
+enum MemoryStorage {
+    Static {
+        base: *mut u8,
+        size: Cell<u32>,
+        maximum: u32,
+        make_accessible: Option<fn(usize, usize) -> bool>,
+    },
+    Dynamic(Box<dyn RuntimeLinearMemory>),
+}
+
+/// Represents an instantiation of a WebAssembly memory.
+pub struct Memory {
+    storage: MemoryStorage,
+}
+
+impl Memory {
+    /// Create a new dynamic (movable) memory instance for the specified plan.
+    pub fn new_dynamic(
+        plan: &MemoryPlan,
+        creator: &dyn RuntimeMemoryCreator,
+    ) -> Result<Self, String> {
+        Ok(Self {
+            storage: MemoryStorage::Dynamic(creator.new_memory(plan)?),
+        })
+    }
+
+    /// Create a new static (immovable) memory instance for the specified plan.
+    pub fn new_static(
+        plan: &MemoryPlan,
+        base: *mut u8,
+        maximum: u32,
+        make_accessible: Option<fn(usize, usize) -> bool>,
+    ) -> Result<Self, String> {
+        debug_assert!(plan.memory.maximum.unwrap_or(maximum) <= maximum);
+
+        if plan.memory.minimum > 0 {
+            if let Some(make_accessible) = &make_accessible {
+                if !make_accessible(0, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize) {
+                    return Err("memory cannot be made accessible".into());
+                }
+            }
+        }
+
+        Ok(Self {
+            storage: MemoryStorage::Static {
+                base,
+                size: Cell::new(plan.memory.minimum),
+                maximum,
+                make_accessible,
+            },
+        })
+    }
+
+    /// Returns the number of allocated wasm pages.
+    pub fn size(&self) -> u32 {
+        match &self.storage {
+            MemoryStorage::Static { size, .. } => size.get(),
+            MemoryStorage::Dynamic(mem) => mem.size(),
+        }
+    }
+
+    /// Grow memory by the specified amount of wasm pages.
+    ///
+    /// Returns `None` if memory can't be grown by the specified amount
+    /// of wasm pages.
+    pub fn grow(&self, delta: u32) -> Option<u32> {
+        match &self.storage {
+            MemoryStorage::Static {
+                size,
+                maximum,
+                make_accessible,
+                ..
+            } => {
+                let old_size = size.get();
+                if delta == 0 {
+                    return Some(old_size);
+                }
+
+                let new_size = old_size.checked_add(delta)?;
+
+                if new_size > *maximum || new_size >= WASM_MAX_PAGES {
+                    return None;
+                }
+
+                let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize;
+                let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize;
+
+                if let Some(make_accessible) = make_accessible {
+                    if !make_accessible(start, len) {
+                        return None;
+                    }
+                }
+
+                size.set(new_size);
+
+                Some(old_size)
+            }
+            MemoryStorage::Dynamic(mem) => mem.grow(delta),
+        }
+    }
+
+    /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code.
+    pub fn vmmemory(&self) -> VMMemoryDefinition {
+        match &self.storage {
+            MemoryStorage::Static { base, size, .. } => VMMemoryDefinition {
+                base: *base,
+                current_length: size.get() as usize,
+            },
+            MemoryStorage::Dynamic(mem) => mem.vmmemory(),
+        }
+    }
+}