Struct wasmtime_runtime::Memory

pub struct Memory(_);

Representation of a runtime wasm linear memory.

Implementations

impl Memory

pub fn new_dynamic(
    plan: &MemoryPlan,
    creator: &dyn RuntimeMemoryCreator,
    store: &mut dyn Store,
    memory_image: Option<&Arc<MemoryImage>>
) -> Result<Self>

Create a new dynamic (movable) memory instance for the specified plan.

Examples found in repository

src/instance/allocator.rs (lines 422-431)

    fn create_memories(
        &self,
        store: &mut StorePtr,
        runtime_info: &Arc<dyn ModuleRuntimeInfo>,
    ) -> Result<PrimaryMap<DefinedMemoryIndex, Memory>> {
        let module = runtime_info.module();
        let creator = self
            .mem_creator
            .as_deref()
            .unwrap_or_else(|| &DefaultMemoryCreator);
        let num_imports = module.num_imported_memories;
        let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
            PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
        for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) {
            let defined_memory_idx = module
                .defined_memory_index(memory_idx)
                .expect("Skipped imports, should never be None");
            let image = runtime_info.memory_image(defined_memory_idx)?;

            memories.push(Memory::new_dynamic(
                plan,
                creator,
                unsafe {
                    store
                        .get()
                        .expect("if module has memory plans, store is not empty")
                },
                image,
            )?);
        }
        Ok(memories)
    }

pub fn new_static(
    plan: &MemoryPlan,
    base: &'static mut [u8],
    memory_image: MemoryImageSlot,
    store: &mut dyn Store
) -> Result<Self>

Create a new static (immovable) memory instance for the specified plan.

pub fn byte_size(&self) -> usize

Returns the number of allocated wasm pages.

pub fn maximum_byte_size(&self) -> Option<usize>

Returns the maximum number of pages the memory can grow to at runtime.

Returns None if the memory is unbounded.

The runtime maximum may not be equal to the maximum from the linear memory’s Wasm type when it is being constrained by an instance allocator.

pub unsafe fn grow(
    &mut self,
    delta_pages: u64,
    store: Option<&mut dyn Store>
) -> Result<Option<usize>, Error>

Grow memory by the specified amount of wasm pages.

Returns None if memory can’t be grown by the specified amount of wasm pages. Returns Some with the old size of memory, in bytes, on successful growth.

Safety

Resizing the memory can reallocate the memory buffer for dynamic memories. An instance’s VMContext may have pointers to the memory’s base and will need to be fixed up after growing the memory.

Generally, prefer using InstanceHandle::memory_grow, which encapsulates this unsafety.

Ensure that the provided Store is not used to get access any Memory which lives inside it.

Examples found in repository

src/instance.rs (line 425)

    pub(crate) fn memory_grow(
        &mut self,
        index: MemoryIndex,
        delta: u64,
    ) -> Result<Option<usize>, Error> {
        let (idx, instance) = if let Some(idx) = self.module().defined_memory_index(index) {
            (idx, self)
        } else {
            let import = self.imported_memory(index);
            unsafe {
                let foreign_instance = (*import.vmctx).instance_mut();
                (import.index, foreign_instance)
            }
        };
        let store = unsafe { &mut *instance.store() };
        let memory = &mut instance.memories[idx];

        let result = unsafe { memory.grow(delta, Some(store)) };

        // Update the state used by a non-shared Wasm memory in case the base
        // pointer and/or the length changed.
        if memory.as_shared_memory().is_none() {
            let vmmemory = memory.vmmemory();
            instance.set_memory(idx, vmmemory);
        }

        result
    }

pub fn vmmemory(&mut self) -> VMMemoryDefinition

Return a VMMemoryDefinition for exposing the memory to compiled wasm code.

Examples found in repository

src/memory.rs (line 840)

    pub fn atomic_notify(&mut self, addr: u64, count: u32) -> Result<u32, Trap> {
        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
            Some(m) => m.atomic_notify(addr, count),
            None => {
                validate_atomic_addr(&self.vmmemory(), addr, 4, 4)?;
                Ok(0)
            }
        }
    }

    /// Implementation of `memory.atomic.wait32` for all memories.
    pub fn atomic_wait32(
        &mut self,
        addr: u64,
        expected: u32,
        deadline: Option<Instant>,
    ) -> Result<WaitResult, Trap> {
        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
            Some(m) => m.atomic_wait32(addr, expected, deadline),
            None => {
                validate_atomic_addr(&self.vmmemory(), addr, 4, 4)?;
                Err(Trap::AtomicWaitNonSharedMemory)
            }
        }
    }

    /// Implementation of `memory.atomic.wait64` for all memories.
    pub fn atomic_wait64(
        &mut self,
        addr: u64,
        expected: u64,
        deadline: Option<Instant>,
    ) -> Result<WaitResult, Trap> {
        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
            Some(m) => m.atomic_wait64(addr, expected, deadline),
            None => {
                validate_atomic_addr(&self.vmmemory(), addr, 8, 8)?;
                Err(Trap::AtomicWaitNonSharedMemory)
            }
        }
    }

src/instance.rs (line 430)

    pub(crate) fn memory_grow(
        &mut self,
        index: MemoryIndex,
        delta: u64,
    ) -> Result<Option<usize>, Error> {
        let (idx, instance) = if let Some(idx) = self.module().defined_memory_index(index) {
            (idx, self)
        } else {
            let import = self.imported_memory(index);
            unsafe {
                let foreign_instance = (*import.vmctx).instance_mut();
                (import.index, foreign_instance)
            }
        };
        let store = unsafe { &mut *instance.store() };
        let memory = &mut instance.memories[idx];

        let result = unsafe { memory.grow(delta, Some(store)) };

        // Update the state used by a non-shared Wasm memory in case the base
        // pointer and/or the length changed.
        if memory.as_shared_memory().is_none() {
            let vmmemory = memory.vmmemory();
            instance.set_memory(idx, vmmemory);
        }

        result
    }

    pub(crate) fn table_element_type(&mut self, table_index: TableIndex) -> TableElementType {
        unsafe { (*self.get_table(table_index)).element_type() }
    }

    /// Grow table by the specified amount of elements, filling them with
    /// `init_value`.
    ///
    /// Returns `None` if table can't be grown by the specified amount of
    /// elements, or if `init_value` is the wrong type of table element.
    pub(crate) fn table_grow(
        &mut self,
        table_index: TableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Result<Option<u32>, Error> {
        let (defined_table_index, instance) =
            self.get_defined_table_index_and_instance(table_index);
        instance.defined_table_grow(defined_table_index, delta, init_value)
    }

    fn defined_table_grow(
        &mut self,
        table_index: DefinedTableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Result<Option<u32>, Error> {
        let store = unsafe { &mut *self.store() };
        let table = self
            .tables
            .get_mut(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));

        let result = unsafe { table.grow(delta, init_value, store) };

        // Keep the `VMContext` pointers used by compiled Wasm code up to
        // date.
        let element = self.tables[table_index].vmtable();
        self.set_table(table_index, element);

        result
    }

    fn alloc_layout(offsets: &VMOffsets<HostPtr>) -> Layout {
        let size = mem::size_of::<Self>()
            .checked_add(usize::try_from(offsets.size_of_vmctx()).unwrap())
            .unwrap();
        let align = mem::align_of::<Self>();
        Layout::from_size_align(size, align).unwrap()
    }

    /// Construct a new VMCallerCheckedAnyfunc for the given function
    /// (imported or defined in this module) and store into the given
    /// location. Used during lazy initialization.
    ///
    /// Note that our current lazy-init scheme actually calls this every
    /// time the anyfunc pointer is fetched; this turns out to be better
    /// than tracking state related to whether it's been initialized
    /// before, because resetting that state on (re)instantiation is
    /// very expensive if there are many anyfuncs.
    fn construct_anyfunc(
        &mut self,
        index: FuncIndex,
        sig: SignatureIndex,
        into: *mut VMCallerCheckedAnyfunc,
    ) {
        let type_index = unsafe {
            let base: *const VMSharedSignatureIndex =
                *self.vmctx_plus_offset(self.offsets().vmctx_signature_ids_array());
            *base.add(sig.index())
        };

        let (func_ptr, vmctx) = if let Some(def_index) = self.module().defined_func_index(index) {
            (
                self.runtime_info.function(def_index),
                VMOpaqueContext::from_vmcontext(self.vmctx_ptr()),
            )
        } else {
            let import = self.imported_function(index);
            (import.body.as_ptr(), import.vmctx)
        };

        // Safety: we have a `&mut self`, so we have exclusive access
        // to this Instance.
        unsafe {
            *into = VMCallerCheckedAnyfunc {
                vmctx,
                type_index,
                func_ptr: NonNull::new(func_ptr).expect("Non-null function pointer"),
            };
        }
    }

    /// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`.
    ///
    /// Returns `None` if the index is the reserved index value.
    ///
    /// The returned reference is a stable reference that won't be moved and can
    /// be passed into JIT code.
    pub(crate) fn get_caller_checked_anyfunc(
        &mut self,
        index: FuncIndex,
    ) -> Option<*mut VMCallerCheckedAnyfunc> {
        if index == FuncIndex::reserved_value() {
            return None;
        }

        // Safety: we have a `&mut self`, so we have exclusive access
        // to this Instance.
        unsafe {
            // For now, we eagerly initialize an anyfunc struct in-place
            // whenever asked for a reference to it. This is mostly
            // fine, because in practice each anyfunc is unlikely to be
            // requested more than a few times: once-ish for funcref
            // tables used for call_indirect (the usual compilation
            // strategy places each function in the table at most once),
            // and once or a few times when fetching exports via API.
            // Note that for any case driven by table accesses, the lazy
            // table init behaves like a higher-level cache layer that
            // protects this initialization from happening multiple
            // times, via that particular table at least.
            //
            // When `ref.func` becomes more commonly used or if we
            // otherwise see a use-case where this becomes a hotpath,
            // we can reconsider by using some state to track
            // "uninitialized" explicitly, for example by zeroing the
            // anyfuncs (perhaps together with other
            // zeroed-at-instantiate-time state) or using a separate
            // is-initialized bitmap.
            //
            // We arrived at this design because zeroing memory is
            // expensive, so it's better for instantiation performance
            // if we don't have to track "is-initialized" state at
            // all!
            let func = &self.module().functions[index];
            let sig = func.signature;
            let anyfunc: *mut VMCallerCheckedAnyfunc = self
                .vmctx_plus_offset::<VMCallerCheckedAnyfunc>(
                    self.offsets().vmctx_anyfunc(func.anyfunc),
                );
            self.construct_anyfunc(index, sig, anyfunc);

            Some(anyfunc)
        }
    }

    /// The `table.init` operation: initializes a portion of a table with a
    /// passive element.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the range within the table is out of bounds
    /// or the range within the passive element is out of bounds.
    pub(crate) fn table_init(
        &mut self,
        table_index: TableIndex,
        elem_index: ElemIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // TODO: this `clone()` shouldn't be necessary but is used for now to
        // inform `rustc` that the lifetime of the elements here are
        // disconnected from the lifetime of `self`.
        let module = self.module().clone();

        let elements = match module.passive_elements_map.get(&elem_index) {
            Some(index) if !self.dropped_elements.contains(elem_index) => {
                module.passive_elements[*index].as_ref()
            }
            _ => &[],
        };
        self.table_init_segment(table_index, elements, dst, src, len)
    }

    pub(crate) fn table_init_segment(
        &mut self,
        table_index: TableIndex,
        elements: &[FuncIndex],
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init

        let table = unsafe { &mut *self.get_table(table_index) };

        let elements = match elements
            .get(usize::try_from(src).unwrap()..)
            .and_then(|s| s.get(..usize::try_from(len).unwrap()))
        {
            Some(elements) => elements,
            None => return Err(Trap::TableOutOfBounds),
        };

        match table.element_type() {
            TableElementType::Func => {
                table.init_funcs(
                    dst,
                    elements.iter().map(|idx| {
                        self.get_caller_checked_anyfunc(*idx)
                            .unwrap_or(std::ptr::null_mut())
                    }),
                )?;
            }

            TableElementType::Extern => {
                debug_assert!(elements.iter().all(|e| *e == FuncIndex::reserved_value()));
                table.fill(dst, TableElement::ExternRef(None), len)?;
            }
        }
        Ok(())
    }

    /// Drop an element.
    pub(crate) fn elem_drop(&mut self, elem_index: ElemIndex) {
        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop

        self.dropped_elements.insert(elem_index);

        // Note that we don't check that we actually removed a segment because
        // dropping a non-passive segment is a no-op (not a trap).
    }

    /// Get a locally-defined memory.
    pub(crate) fn get_defined_memory(&mut self, index: DefinedMemoryIndex) -> *mut Memory {
        ptr::addr_of_mut!(self.memories[index])
    }

    /// Do a `memory.copy`
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the source or destination ranges are out of
    /// bounds.
    pub(crate) fn memory_copy(
        &mut self,
        dst_index: MemoryIndex,
        dst: u64,
        src_index: MemoryIndex,
        src: u64,
        len: u64,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy

        let src_mem = self.get_memory(src_index);
        let dst_mem = self.get_memory(dst_index);

        let src = self.validate_inbounds(src_mem.current_length(), src, len)?;
        let dst = self.validate_inbounds(dst_mem.current_length(), dst, len)?;

        // Bounds and casts are checked above, by this point we know that
        // everything is safe.
        unsafe {
            let dst = dst_mem.base.add(dst);
            let src = src_mem.base.add(src);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::copy(src, dst, len as usize);
        }

        Ok(())
    }

    fn validate_inbounds(&self, max: usize, ptr: u64, len: u64) -> Result<usize, Trap> {
        let oob = || Trap::MemoryOutOfBounds;
        let end = ptr
            .checked_add(len)
            .and_then(|i| usize::try_from(i).ok())
            .ok_or_else(oob)?;
        if end > max {
            Err(oob())
        } else {
            Ok(ptr as usize)
        }
    }

    /// Perform the `memory.fill` operation on a locally defined memory.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the memory range is out of bounds.
    pub(crate) fn memory_fill(
        &mut self,
        memory_index: MemoryIndex,
        dst: u64,
        val: u8,
        len: u64,
    ) -> Result<(), Trap> {
        let memory = self.get_memory(memory_index);
        let dst = self.validate_inbounds(memory.current_length(), dst, len)?;

        // Bounds and casts are checked above, by this point we know that
        // everything is safe.
        unsafe {
            let dst = memory.base.add(dst);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::write_bytes(dst, val, len as usize);
        }

        Ok(())
    }

    /// Performs the `memory.init` operation.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the destination range is out of this module's
    /// memory's bounds or if the source range is outside the data segment's
    /// bounds.
    pub(crate) fn memory_init(
        &mut self,
        memory_index: MemoryIndex,
        data_index: DataIndex,
        dst: u64,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let range = match self.module().passive_data_map.get(&data_index).cloned() {
            Some(range) if !self.dropped_data.contains(data_index) => range,
            _ => 0..0,
        };
        self.memory_init_segment(memory_index, range, dst, src, len)
    }

    pub(crate) fn wasm_data(&self, range: Range<u32>) -> &[u8] {
        &self.runtime_info.wasm_data()[range.start as usize..range.end as usize]
    }

    pub(crate) fn memory_init_segment(
        &mut self,
        memory_index: MemoryIndex,
        range: Range<u32>,
        dst: u64,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init

        let memory = self.get_memory(memory_index);
        let data = self.wasm_data(range);
        let dst = self.validate_inbounds(memory.current_length(), dst, len.into())?;
        let src = self.validate_inbounds(data.len(), src.into(), len.into())?;
        let len = len as usize;

        unsafe {
            let src_start = data.as_ptr().add(src);
            let dst_start = memory.base.add(dst);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::copy_nonoverlapping(src_start, dst_start, len);
        }

        Ok(())
    }

    /// Drop the given data segment, truncating its length to zero.
    pub(crate) fn data_drop(&mut self, data_index: DataIndex) {
        self.dropped_data.insert(data_index);

        // Note that we don't check that we actually removed a segment because
        // dropping a non-passive segment is a no-op (not a trap).
    }

    /// Get a table by index regardless of whether it is locally-defined
    /// or an imported, foreign table. Ensure that the given range of
    /// elements in the table is lazily initialized.  We define this
    /// operation all-in-one for safety, to ensure the lazy-init
    /// happens.
    ///
    /// Takes an `Iterator` for the index-range to lazy-initialize,
    /// for flexibility. This can be a range, single item, or empty
    /// sequence, for example. The iterator should return indices in
    /// increasing order, so that the break-at-out-of-bounds behavior
    /// works correctly.
    pub(crate) fn get_table_with_lazy_init(
        &mut self,
        table_index: TableIndex,
        range: impl Iterator<Item = u32>,
    ) -> *mut Table {
        let (idx, instance) = self.get_defined_table_index_and_instance(table_index);
        let elt_ty = instance.tables[idx].element_type();

        if elt_ty == TableElementType::Func {
            for i in range {
                let value = match instance.tables[idx].get(i) {
                    Some(value) => value,
                    None => {
                        // Out-of-bounds; caller will handle by likely
                        // throwing a trap. No work to do to lazy-init
                        // beyond the end.
                        break;
                    }
                };
                if value.is_uninit() {
                    let table_init = match &instance.module().table_initialization {
                        // We unfortunately can't borrow `tables`
                        // outside the loop because we need to call
                        // `get_caller_checked_anyfunc` (a `&mut`
                        // method) below; so unwrap it dynamically
                        // here.
                        TableInitialization::FuncTable { tables, .. } => tables,
                        _ => break,
                    }
                    .get(table_index);

                    // The TableInitialization::FuncTable elements table may
                    // be smaller than the current size of the table: it
                    // always matches the initial table size, if present. We
                    // want to iterate up through the end of the accessed
                    // index range so that we set an "initialized null" even
                    // if there is no initializer. We do a checked `get()` on
                    // the initializer table below and unwrap to a null if
                    // we're past its end.
                    let func_index =
                        table_init.and_then(|indices| indices.get(i as usize).cloned());
                    let anyfunc = func_index
                        .and_then(|func_index| instance.get_caller_checked_anyfunc(func_index))
                        .unwrap_or(std::ptr::null_mut());

                    let value = TableElement::FuncRef(anyfunc);

                    instance.tables[idx]
                        .set(i, value)
                        .expect("Table type should match and index should be in-bounds");
                }
            }
        }

        ptr::addr_of_mut!(instance.tables[idx])
    }

    /// Get a table by index regardless of whether it is locally-defined or an
    /// imported, foreign table.
    pub(crate) fn get_table(&mut self, table_index: TableIndex) -> *mut Table {
        let (idx, instance) = self.get_defined_table_index_and_instance(table_index);
        ptr::addr_of_mut!(instance.tables[idx])
    }

    /// Get a locally-defined table.
    pub(crate) fn get_defined_table(&mut self, index: DefinedTableIndex) -> *mut Table {
        ptr::addr_of_mut!(self.tables[index])
    }

    pub(crate) fn get_defined_table_index_and_instance(
        &mut self,
        index: TableIndex,
    ) -> (DefinedTableIndex, &mut Instance) {
        if let Some(defined_table_index) = self.module().defined_table_index(index) {
            (defined_table_index, self)
        } else {
            let import = self.imported_table(index);
            unsafe {
                let foreign_instance = (*import.vmctx).instance_mut();
                let foreign_table_def = &*import.from;
                let foreign_table_index = foreign_instance.table_index(foreign_table_def);
                (foreign_table_index, foreign_instance)
            }
        }
    }

    /// Initialize the VMContext data associated with this Instance.
    ///
    /// The `VMContext` memory is assumed to be uninitialized; any field
    /// that we need in a certain state will be explicitly written by this
    /// function.
    unsafe fn initialize_vmctx(
        &mut self,
        module: &Module,
        offsets: &VMOffsets<HostPtr>,
        store: StorePtr,
        imports: Imports,
    ) {
        assert!(std::ptr::eq(module, self.module().as_ref()));

        *self.vmctx_plus_offset(offsets.vmctx_magic()) = VMCONTEXT_MAGIC;
        self.set_callee(None);
        self.set_store(store.as_raw());

        // Initialize shared signatures
        let signatures = self.runtime_info.signature_ids();
        *self.vmctx_plus_offset(offsets.vmctx_signature_ids_array()) = signatures.as_ptr();

        // Initialize the built-in functions
        *self.vmctx_plus_offset(offsets.vmctx_builtin_functions()) = &VMBuiltinFunctionsArray::INIT;

        // Initialize the imports
        debug_assert_eq!(imports.functions.len(), module.num_imported_funcs);
        ptr::copy_nonoverlapping(
            imports.functions.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_functions_begin()),
            imports.functions.len(),
        );
        debug_assert_eq!(imports.tables.len(), module.num_imported_tables);
        ptr::copy_nonoverlapping(
            imports.tables.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_tables_begin()),
            imports.tables.len(),
        );
        debug_assert_eq!(imports.memories.len(), module.num_imported_memories);
        ptr::copy_nonoverlapping(
            imports.memories.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_memories_begin()),
            imports.memories.len(),
        );
        debug_assert_eq!(imports.globals.len(), module.num_imported_globals);
        ptr::copy_nonoverlapping(
            imports.globals.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_globals_begin()),
            imports.globals.len(),
        );

        // N.B.: there is no need to initialize the anyfuncs array because
        // we eagerly construct each element in it whenever asked for a
        // reference to that element. In other words, there is no state
        // needed to track the lazy-init, so we don't need to initialize
        // any state now.

        // Initialize the defined tables
        let mut ptr = self.vmctx_plus_offset(offsets.vmctx_tables_begin());
        for i in 0..module.table_plans.len() - module.num_imported_tables {
            ptr::write(ptr, self.tables[DefinedTableIndex::new(i)].vmtable());
            ptr = ptr.add(1);
        }

        // Initialize the defined memories. This fills in both the
        // `defined_memories` table and the `owned_memories` table at the same
        // time. Entries in `defined_memories` hold a pointer to a definition
        // (all memories) whereas the `owned_memories` hold the actual
        // definitions of memories owned (not shared) in the module.
        let mut ptr = self.vmctx_plus_offset(offsets.vmctx_memories_begin());
        let mut owned_ptr = self.vmctx_plus_offset(offsets.vmctx_owned_memories_begin());
        for i in 0..module.memory_plans.len() - module.num_imported_memories {
            let defined_memory_index = DefinedMemoryIndex::new(i);
            let memory_index = module.memory_index(defined_memory_index);
            if module.memory_plans[memory_index].memory.shared {
                let def_ptr = self.memories[defined_memory_index]
                    .as_shared_memory()
                    .unwrap()
                    .vmmemory_ptr();
                ptr::write(ptr, def_ptr.cast_mut());
            } else {
                ptr::write(owned_ptr, self.memories[defined_memory_index].vmmemory());
                ptr::write(ptr, owned_ptr);
                owned_ptr = owned_ptr.add(1);
            }
            ptr = ptr.add(1);
        }

        // Initialize the defined globals
        self.initialize_vmctx_globals(module);
    }

pub fn as_shared_memory(&mut self) -> Option<&mut SharedMemory>

If the Memory is a SharedMemory, unwrap it and return a clone to that shared memory.

Examples found in repository

src/instance.rs (line 429)

    pub(crate) fn memory_grow(
        &mut self,
        index: MemoryIndex,
        delta: u64,
    ) -> Result<Option<usize>, Error> {
        let (idx, instance) = if let Some(idx) = self.module().defined_memory_index(index) {
            (idx, self)
        } else {
            let import = self.imported_memory(index);
            unsafe {
                let foreign_instance = (*import.vmctx).instance_mut();
                (import.index, foreign_instance)
            }
        };
        let store = unsafe { &mut *instance.store() };
        let memory = &mut instance.memories[idx];

        let result = unsafe { memory.grow(delta, Some(store)) };

        // Update the state used by a non-shared Wasm memory in case the base
        // pointer and/or the length changed.
        if memory.as_shared_memory().is_none() {
            let vmmemory = memory.vmmemory();
            instance.set_memory(idx, vmmemory);
        }

        result
    }

    pub(crate) fn table_element_type(&mut self, table_index: TableIndex) -> TableElementType {
        unsafe { (*self.get_table(table_index)).element_type() }
    }

    /// Grow table by the specified amount of elements, filling them with
    /// `init_value`.
    ///
    /// Returns `None` if table can't be grown by the specified amount of
    /// elements, or if `init_value` is the wrong type of table element.
    pub(crate) fn table_grow(
        &mut self,
        table_index: TableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Result<Option<u32>, Error> {
        let (defined_table_index, instance) =
            self.get_defined_table_index_and_instance(table_index);
        instance.defined_table_grow(defined_table_index, delta, init_value)
    }

    fn defined_table_grow(
        &mut self,
        table_index: DefinedTableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Result<Option<u32>, Error> {
        let store = unsafe { &mut *self.store() };
        let table = self
            .tables
            .get_mut(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));

        let result = unsafe { table.grow(delta, init_value, store) };

        // Keep the `VMContext` pointers used by compiled Wasm code up to
        // date.
        let element = self.tables[table_index].vmtable();
        self.set_table(table_index, element);

        result
    }

    fn alloc_layout(offsets: &VMOffsets<HostPtr>) -> Layout {
        let size = mem::size_of::<Self>()
            .checked_add(usize::try_from(offsets.size_of_vmctx()).unwrap())
            .unwrap();
        let align = mem::align_of::<Self>();
        Layout::from_size_align(size, align).unwrap()
    }

    /// Construct a new VMCallerCheckedAnyfunc for the given function
    /// (imported or defined in this module) and store into the given
    /// location. Used during lazy initialization.
    ///
    /// Note that our current lazy-init scheme actually calls this every
    /// time the anyfunc pointer is fetched; this turns out to be better
    /// than tracking state related to whether it's been initialized
    /// before, because resetting that state on (re)instantiation is
    /// very expensive if there are many anyfuncs.
    fn construct_anyfunc(
        &mut self,
        index: FuncIndex,
        sig: SignatureIndex,
        into: *mut VMCallerCheckedAnyfunc,
    ) {
        let type_index = unsafe {
            let base: *const VMSharedSignatureIndex =
                *self.vmctx_plus_offset(self.offsets().vmctx_signature_ids_array());
            *base.add(sig.index())
        };

        let (func_ptr, vmctx) = if let Some(def_index) = self.module().defined_func_index(index) {
            (
                self.runtime_info.function(def_index),
                VMOpaqueContext::from_vmcontext(self.vmctx_ptr()),
            )
        } else {
            let import = self.imported_function(index);
            (import.body.as_ptr(), import.vmctx)
        };

        // Safety: we have a `&mut self`, so we have exclusive access
        // to this Instance.
        unsafe {
            *into = VMCallerCheckedAnyfunc {
                vmctx,
                type_index,
                func_ptr: NonNull::new(func_ptr).expect("Non-null function pointer"),
            };
        }
    }

    /// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`.
    ///
    /// Returns `None` if the index is the reserved index value.
    ///
    /// The returned reference is a stable reference that won't be moved and can
    /// be passed into JIT code.
    pub(crate) fn get_caller_checked_anyfunc(
        &mut self,
        index: FuncIndex,
    ) -> Option<*mut VMCallerCheckedAnyfunc> {
        if index == FuncIndex::reserved_value() {
            return None;
        }

        // Safety: we have a `&mut self`, so we have exclusive access
        // to this Instance.
        unsafe {
            // For now, we eagerly initialize an anyfunc struct in-place
            // whenever asked for a reference to it. This is mostly
            // fine, because in practice each anyfunc is unlikely to be
            // requested more than a few times: once-ish for funcref
            // tables used for call_indirect (the usual compilation
            // strategy places each function in the table at most once),
            // and once or a few times when fetching exports via API.
            // Note that for any case driven by table accesses, the lazy
            // table init behaves like a higher-level cache layer that
            // protects this initialization from happening multiple
            // times, via that particular table at least.
            //
            // When `ref.func` becomes more commonly used or if we
            // otherwise see a use-case where this becomes a hotpath,
            // we can reconsider by using some state to track
            // "uninitialized" explicitly, for example by zeroing the
            // anyfuncs (perhaps together with other
            // zeroed-at-instantiate-time state) or using a separate
            // is-initialized bitmap.
            //
            // We arrived at this design because zeroing memory is
            // expensive, so it's better for instantiation performance
            // if we don't have to track "is-initialized" state at
            // all!
            let func = &self.module().functions[index];
            let sig = func.signature;
            let anyfunc: *mut VMCallerCheckedAnyfunc = self
                .vmctx_plus_offset::<VMCallerCheckedAnyfunc>(
                    self.offsets().vmctx_anyfunc(func.anyfunc),
                );
            self.construct_anyfunc(index, sig, anyfunc);

            Some(anyfunc)
        }
    }

    /// The `table.init` operation: initializes a portion of a table with a
    /// passive element.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the range within the table is out of bounds
    /// or the range within the passive element is out of bounds.
    pub(crate) fn table_init(
        &mut self,
        table_index: TableIndex,
        elem_index: ElemIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // TODO: this `clone()` shouldn't be necessary but is used for now to
        // inform `rustc` that the lifetime of the elements here are
        // disconnected from the lifetime of `self`.
        let module = self.module().clone();

        let elements = match module.passive_elements_map.get(&elem_index) {
            Some(index) if !self.dropped_elements.contains(elem_index) => {
                module.passive_elements[*index].as_ref()
            }
            _ => &[],
        };
        self.table_init_segment(table_index, elements, dst, src, len)
    }

    pub(crate) fn table_init_segment(
        &mut self,
        table_index: TableIndex,
        elements: &[FuncIndex],
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init

        let table = unsafe { &mut *self.get_table(table_index) };

        let elements = match elements
            .get(usize::try_from(src).unwrap()..)
            .and_then(|s| s.get(..usize::try_from(len).unwrap()))
        {
            Some(elements) => elements,
            None => return Err(Trap::TableOutOfBounds),
        };

        match table.element_type() {
            TableElementType::Func => {
                table.init_funcs(
                    dst,
                    elements.iter().map(|idx| {
                        self.get_caller_checked_anyfunc(*idx)
                            .unwrap_or(std::ptr::null_mut())
                    }),
                )?;
            }

            TableElementType::Extern => {
                debug_assert!(elements.iter().all(|e| *e == FuncIndex::reserved_value()));
                table.fill(dst, TableElement::ExternRef(None), len)?;
            }
        }
        Ok(())
    }

    /// Drop an element.
    pub(crate) fn elem_drop(&mut self, elem_index: ElemIndex) {
        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop

        self.dropped_elements.insert(elem_index);

        // Note that we don't check that we actually removed a segment because
        // dropping a non-passive segment is a no-op (not a trap).
    }

    /// Get a locally-defined memory.
    pub(crate) fn get_defined_memory(&mut self, index: DefinedMemoryIndex) -> *mut Memory {
        ptr::addr_of_mut!(self.memories[index])
    }

    /// Do a `memory.copy`
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the source or destination ranges are out of
    /// bounds.
    pub(crate) fn memory_copy(
        &mut self,
        dst_index: MemoryIndex,
        dst: u64,
        src_index: MemoryIndex,
        src: u64,
        len: u64,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy

        let src_mem = self.get_memory(src_index);
        let dst_mem = self.get_memory(dst_index);

        let src = self.validate_inbounds(src_mem.current_length(), src, len)?;
        let dst = self.validate_inbounds(dst_mem.current_length(), dst, len)?;

        // Bounds and casts are checked above, by this point we know that
        // everything is safe.
        unsafe {
            let dst = dst_mem.base.add(dst);
            let src = src_mem.base.add(src);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::copy(src, dst, len as usize);
        }

        Ok(())
    }

    fn validate_inbounds(&self, max: usize, ptr: u64, len: u64) -> Result<usize, Trap> {
        let oob = || Trap::MemoryOutOfBounds;
        let end = ptr
            .checked_add(len)
            .and_then(|i| usize::try_from(i).ok())
            .ok_or_else(oob)?;
        if end > max {
            Err(oob())
        } else {
            Ok(ptr as usize)
        }
    }

    /// Perform the `memory.fill` operation on a locally defined memory.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the memory range is out of bounds.
    pub(crate) fn memory_fill(
        &mut self,
        memory_index: MemoryIndex,
        dst: u64,
        val: u8,
        len: u64,
    ) -> Result<(), Trap> {
        let memory = self.get_memory(memory_index);
        let dst = self.validate_inbounds(memory.current_length(), dst, len)?;

        // Bounds and casts are checked above, by this point we know that
        // everything is safe.
        unsafe {
            let dst = memory.base.add(dst);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::write_bytes(dst, val, len as usize);
        }

        Ok(())
    }

    /// Performs the `memory.init` operation.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the destination range is out of this module's
    /// memory's bounds or if the source range is outside the data segment's
    /// bounds.
    pub(crate) fn memory_init(
        &mut self,
        memory_index: MemoryIndex,
        data_index: DataIndex,
        dst: u64,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let range = match self.module().passive_data_map.get(&data_index).cloned() {
            Some(range) if !self.dropped_data.contains(data_index) => range,
            _ => 0..0,
        };
        self.memory_init_segment(memory_index, range, dst, src, len)
    }

    pub(crate) fn wasm_data(&self, range: Range<u32>) -> &[u8] {
        &self.runtime_info.wasm_data()[range.start as usize..range.end as usize]
    }

    pub(crate) fn memory_init_segment(
        &mut self,
        memory_index: MemoryIndex,
        range: Range<u32>,
        dst: u64,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init

        let memory = self.get_memory(memory_index);
        let data = self.wasm_data(range);
        let dst = self.validate_inbounds(memory.current_length(), dst, len.into())?;
        let src = self.validate_inbounds(data.len(), src.into(), len.into())?;
        let len = len as usize;

        unsafe {
            let src_start = data.as_ptr().add(src);
            let dst_start = memory.base.add(dst);
            // FIXME audit whether this is safe in the presence of shared memory
            // (https://github.com/bytecodealliance/wasmtime/issues/4203).
            ptr::copy_nonoverlapping(src_start, dst_start, len);
        }

        Ok(())
    }

    /// Drop the given data segment, truncating its length to zero.
    pub(crate) fn data_drop(&mut self, data_index: DataIndex) {
        self.dropped_data.insert(data_index);

        // Note that we don't check that we actually removed a segment because
        // dropping a non-passive segment is a no-op (not a trap).
    }

    /// Get a table by index regardless of whether it is locally-defined
    /// or an imported, foreign table. Ensure that the given range of
    /// elements in the table is lazily initialized.  We define this
    /// operation all-in-one for safety, to ensure the lazy-init
    /// happens.
    ///
    /// Takes an `Iterator` for the index-range to lazy-initialize,
    /// for flexibility. This can be a range, single item, or empty
    /// sequence, for example. The iterator should return indices in
    /// increasing order, so that the break-at-out-of-bounds behavior
    /// works correctly.
    pub(crate) fn get_table_with_lazy_init(
        &mut self,
        table_index: TableIndex,
        range: impl Iterator<Item = u32>,
    ) -> *mut Table {
        let (idx, instance) = self.get_defined_table_index_and_instance(table_index);
        let elt_ty = instance.tables[idx].element_type();

        if elt_ty == TableElementType::Func {
            for i in range {
                let value = match instance.tables[idx].get(i) {
                    Some(value) => value,
                    None => {
                        // Out-of-bounds; caller will handle by likely
                        // throwing a trap. No work to do to lazy-init
                        // beyond the end.
                        break;
                    }
                };
                if value.is_uninit() {
                    let table_init = match &instance.module().table_initialization {
                        // We unfortunately can't borrow `tables`
                        // outside the loop because we need to call
                        // `get_caller_checked_anyfunc` (a `&mut`
                        // method) below; so unwrap it dynamically
                        // here.
                        TableInitialization::FuncTable { tables, .. } => tables,
                        _ => break,
                    }
                    .get(table_index);

                    // The TableInitialization::FuncTable elements table may
                    // be smaller than the current size of the table: it
                    // always matches the initial table size, if present. We
                    // want to iterate up through the end of the accessed
                    // index range so that we set an "initialized null" even
                    // if there is no initializer. We do a checked `get()` on
                    // the initializer table below and unwrap to a null if
                    // we're past its end.
                    let func_index =
                        table_init.and_then(|indices| indices.get(i as usize).cloned());
                    let anyfunc = func_index
                        .and_then(|func_index| instance.get_caller_checked_anyfunc(func_index))
                        .unwrap_or(std::ptr::null_mut());

                    let value = TableElement::FuncRef(anyfunc);

                    instance.tables[idx]
                        .set(i, value)
                        .expect("Table type should match and index should be in-bounds");
                }
            }
        }

        ptr::addr_of_mut!(instance.tables[idx])
    }

    /// Get a table by index regardless of whether it is locally-defined or an
    /// imported, foreign table.
    pub(crate) fn get_table(&mut self, table_index: TableIndex) -> *mut Table {
        let (idx, instance) = self.get_defined_table_index_and_instance(table_index);
        ptr::addr_of_mut!(instance.tables[idx])
    }

    /// Get a locally-defined table.
    pub(crate) fn get_defined_table(&mut self, index: DefinedTableIndex) -> *mut Table {
        ptr::addr_of_mut!(self.tables[index])
    }

    pub(crate) fn get_defined_table_index_and_instance(
        &mut self,
        index: TableIndex,
    ) -> (DefinedTableIndex, &mut Instance) {
        if let Some(defined_table_index) = self.module().defined_table_index(index) {
            (defined_table_index, self)
        } else {
            let import = self.imported_table(index);
            unsafe {
                let foreign_instance = (*import.vmctx).instance_mut();
                let foreign_table_def = &*import.from;
                let foreign_table_index = foreign_instance.table_index(foreign_table_def);
                (foreign_table_index, foreign_instance)
            }
        }
    }

    /// Initialize the VMContext data associated with this Instance.
    ///
    /// The `VMContext` memory is assumed to be uninitialized; any field
    /// that we need in a certain state will be explicitly written by this
    /// function.
    unsafe fn initialize_vmctx(
        &mut self,
        module: &Module,
        offsets: &VMOffsets<HostPtr>,
        store: StorePtr,
        imports: Imports,
    ) {
        assert!(std::ptr::eq(module, self.module().as_ref()));

        *self.vmctx_plus_offset(offsets.vmctx_magic()) = VMCONTEXT_MAGIC;
        self.set_callee(None);
        self.set_store(store.as_raw());

        // Initialize shared signatures
        let signatures = self.runtime_info.signature_ids();
        *self.vmctx_plus_offset(offsets.vmctx_signature_ids_array()) = signatures.as_ptr();

        // Initialize the built-in functions
        *self.vmctx_plus_offset(offsets.vmctx_builtin_functions()) = &VMBuiltinFunctionsArray::INIT;

        // Initialize the imports
        debug_assert_eq!(imports.functions.len(), module.num_imported_funcs);
        ptr::copy_nonoverlapping(
            imports.functions.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_functions_begin()),
            imports.functions.len(),
        );
        debug_assert_eq!(imports.tables.len(), module.num_imported_tables);
        ptr::copy_nonoverlapping(
            imports.tables.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_tables_begin()),
            imports.tables.len(),
        );
        debug_assert_eq!(imports.memories.len(), module.num_imported_memories);
        ptr::copy_nonoverlapping(
            imports.memories.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_memories_begin()),
            imports.memories.len(),
        );
        debug_assert_eq!(imports.globals.len(), module.num_imported_globals);
        ptr::copy_nonoverlapping(
            imports.globals.as_ptr(),
            self.vmctx_plus_offset(offsets.vmctx_imported_globals_begin()),
            imports.globals.len(),
        );

        // N.B.: there is no need to initialize the anyfuncs array because
        // we eagerly construct each element in it whenever asked for a
        // reference to that element. In other words, there is no state
        // needed to track the lazy-init, so we don't need to initialize
        // any state now.

        // Initialize the defined tables
        let mut ptr = self.vmctx_plus_offset(offsets.vmctx_tables_begin());
        for i in 0..module.table_plans.len() - module.num_imported_tables {
            ptr::write(ptr, self.tables[DefinedTableIndex::new(i)].vmtable());
            ptr = ptr.add(1);
        }

        // Initialize the defined memories. This fills in both the
        // `defined_memories` table and the `owned_memories` table at the same
        // time. Entries in `defined_memories` hold a pointer to a definition
        // (all memories) whereas the `owned_memories` hold the actual
        // definitions of memories owned (not shared) in the module.
        let mut ptr = self.vmctx_plus_offset(offsets.vmctx_memories_begin());
        let mut owned_ptr = self.vmctx_plus_offset(offsets.vmctx_owned_memories_begin());
        for i in 0..module.memory_plans.len() - module.num_imported_memories {
            let defined_memory_index = DefinedMemoryIndex::new(i);
            let memory_index = module.memory_index(defined_memory_index);
            if module.memory_plans[memory_index].memory.shared {
                let def_ptr = self.memories[defined_memory_index]
                    .as_shared_memory()
                    .unwrap()
                    .vmmemory_ptr();
                ptr::write(ptr, def_ptr.cast_mut());
            } else {
                ptr::write(owned_ptr, self.memories[defined_memory_index].vmmemory());
                ptr::write(ptr, owned_ptr);
                owned_ptr = owned_ptr.add(1);
            }
            ptr = ptr.add(1);
        }

        // Initialize the defined globals
        self.initialize_vmctx_globals(module);
    }

pub fn atomic_notify(&mut self, addr: u64, count: u32) -> Result<u32, Trap>

Implementation of memory.atomic.notify for all memories.

Examples found in repository

src/libcalls.rs (line 444)

unsafe fn memory_atomic_notify(
    vmctx: *mut VMContext,
    memory_index: u32,
    addr_index: u64,
    count: u32,
) -> Result<u32, Trap> {
    let memory = MemoryIndex::from_u32(memory_index);
    let instance = (*vmctx).instance_mut();
    instance
        .get_runtime_memory(memory)
        .atomic_notify(addr_index, count)
}

pub fn atomic_wait32(
    &mut self,
    addr: u64,
    expected: u32,
    deadline: Option<Instant>
) -> Result<WaitResult, Trap>

Implementation of memory.atomic.wait32 for all memories.

Examples found in repository

src/libcalls.rs (line 461)

unsafe fn memory_atomic_wait32(
    vmctx: *mut VMContext,
    memory_index: u32,
    addr_index: u64,
    expected: u32,
    timeout: u64,
) -> Result<u32, Trap> {
    // convert timeout to Instant, before any wait happens on locking
    let timeout = (timeout as i64 >= 0).then(|| Instant::now() + Duration::from_nanos(timeout));
    let memory = MemoryIndex::from_u32(memory_index);
    let instance = (*vmctx).instance_mut();
    Ok(instance
        .get_runtime_memory(memory)
        .atomic_wait32(addr_index, expected, timeout)? as u32)
}

pub fn atomic_wait64(
    &mut self,
    addr: u64,
    expected: u64,
    deadline: Option<Instant>
) -> Result<WaitResult, Trap>

Implementation of memory.atomic.wait64 for all memories.

Examples found in repository

src/libcalls.rs (line 478)

unsafe fn memory_atomic_wait64(
    vmctx: *mut VMContext,
    memory_index: u32,
    addr_index: u64,
    expected: u64,
    timeout: u64,
) -> Result<u32, Trap> {
    // convert timeout to Instant, before any wait happens on locking
    let timeout = (timeout as i64 >= 0).then(|| Instant::now() + Duration::from_nanos(timeout));
    let memory = MemoryIndex::from_u32(memory_index);
    let instance = (*vmctx).instance_mut();
    Ok(instance
        .get_runtime_memory(memory)
        .atomic_wait64(addr_index, expected, timeout)? as u32)
}