ic_memory/runtime/
backing.rs

1use ic_stable_structures::{Memory, memory_manager::VirtualMemory};
2use std::{cell::RefCell, rc::Rc};
3
4// Backing, immutable geometry and one live bucket count for the sole manager,
5// shared by the runtime and all handles, including the ledger. The count is
6// seeded from validated metadata on reopen and updated only after grow.
7pub(super) struct GrowthState<M: Memory> {
8    pub backing: Rc<M>,
9    pub bucket_size_pages: u16,
10    pub allocated_buckets: RefCell<u16>,
11}
12
13///
14/// RuntimeMemory
15///
16/// Cloneable virtual memory opened through a runtime's committed authority.
17/// Implements [`Memory`] for stable structures without exposing the backing
18/// memory or an alternate manager. Cloning does not require `M: Clone`.
19/// IO checks the virtual extent before upstream bucket translation. Empty IO
20/// is valid at the end of memory, but not beyond it. Reads preserve the upstream
21/// optimized support for uninitialized destinations through `Memory::read_unsafe`.
22/// Growth reserves physical capacity before assigning manager buckets. Ordinary
23/// backing refusal, arithmetic overflow and bucket exhaustion return typed errors
24/// without changing virtual extents or manager metadata. Backing implementations
25/// must obey the `Memory` contract; traps and partial writes are not transactions
26/// on native memory. Typed collections retain their own failure behavior.
27///
28
29pub struct RuntimeMemory<M: Memory> {
30    pub(super) memory: VirtualMemory<Rc<M>>,
31    pub(super) growth: Rc<GrowthState<M>>,
32}
33
34impl<M: Memory> Clone for RuntimeMemory<M> {
35    fn clone(&self) -> Self {
36        Self {
37            memory: self.memory.clone(),
38            growth: Rc::clone(&self.growth),
39        }
40    }
41}
42
43impl<M: Memory> RuntimeMemory<M> {
44    // The manager bounds page counts by 32768 buckets of at most u16::MAX
45    // pages, so converting its virtual extent to bytes cannot overflow u64.
46    // Check before the upstream cache: its buckets can include virtual slack,
47    // and its span/translation arithmetic is unchecked in release builds.
48    #[expect(
49        clippy::inline_always,
50        reason = "matched PocketIC measurements reduce IO instructions with bounded Wasm growth"
51    )]
52    #[inline(always)]
53    fn check_io_bounds(&self, offset: u64, count: usize) {
54        let extent = self.memory.size() * crate::constants::WASM_PAGE_SIZE_BYTES;
55        assert!(
56            offset <= extent && count as u64 <= extent - offset,
57            "virtual memory access out of bounds",
58        );
59    }
60
61    /// Grow by the requested pages, returning the previous virtual page count.
62    ///
63    /// Capacity is reserved before assigning manager buckets. Ordinary refusal
64    /// preserves virtual extents and manager metadata and permits retry. The
65    /// upstream [`Memory::grow`] adapter translates errors into its required
66    /// `-1` sentinel; direct runtime callers receive [`super::RuntimeGrowError`].
67    /// A zero-page request returns the current extent without backing IO or
68    /// manager mutation, after checking for reentrant growth.
69    ///
70    /// # Panics
71    ///
72    /// Panics if a private growth-accounting invariant is broken or backing
73    /// memory panics.
74    pub fn grow(&self, pages: u64) -> Result<u64, super::RuntimeGrowError> {
75        use super::RuntimeGrowError;
76        let mut allocated = self
77            .growth
78            .allocated_buckets
79            .try_borrow_mut()
80            .map_err(|_| RuntimeGrowError::ReentrantAccess)?;
81        let old_pages = self.memory.size();
82        if pages == 0 {
83            return Ok(old_pages);
84        }
85        let new_pages = old_pages
86            .checked_add(pages)
87            .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
88        let bucket_pages = u64::from(self.growth.bucket_size_pages);
89        let extra = new_pages.div_ceil(bucket_pages) - old_pages.div_ceil(bucket_pages);
90        let total = u64::from(*allocated)
91            .checked_add(extra)
92            .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
93        if total > u64::from(super::layout::BUCKET_CAPACITY) {
94            return Err(RuntimeGrowError::BucketExhausted {
95                required_buckets: total,
96                capacity: super::layout::BUCKET_CAPACITY,
97            });
98        }
99        #[expect(
100            clippy::cast_possible_truncation,
101            reason = "admission bounds total by the u16 bucket capacity"
102        )]
103        let total_buckets = total as u16;
104        let required_pages = 1 + total * bucket_pages;
105        let physical_pages = self.growth.backing.size();
106        if required_pages > physical_pages
107            && self.growth.backing.grow(required_pages - physical_pages) < 0
108        {
109            return Err(RuntimeGrowError::BackingRefused {
110                additional_pages: required_pages - physical_pages,
111            });
112        }
113        // The pinned manager's only refusal is bucket exhaustion, already
114        // checked above while all handles share this exclusive reservation.
115        // Physical capacity is reserved before it assigns any buckets.
116        assert_eq!(
117            self.memory.grow(pages),
118            old_pages.cast_signed(),
119            "preflighted manager growth returns the previous virtual extent"
120        );
121        *allocated = total_buckets;
122        Ok(old_pages)
123    }
124}
125
126impl<M: Memory> Memory for RuntimeMemory<M> {
127    fn size(&self) -> u64 {
128        self.memory.size()
129    }
130    fn grow(&self, pages: u64) -> i64 {
131        // The upstream trait fixes the sentinel contract. Direct calls use the
132        // inherent typed method; virtual extents fit in i64 by bucket capacity.
133        Self::grow(self, pages).map_or(-1, u64::cast_signed)
134    }
135    fn read(&self, offset: u64, dst: &mut [u8]) {
136        self.check_io_bounds(offset, dst.len());
137        self.memory.read(offset, dst);
138    }
139    #[expect(
140        unsafe_code,
141        reason = "delegate the upstream raw-read contract unchanged"
142    )]
143    unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) {
144        self.check_io_bounds(offset, count);
145        // SAFETY: The caller supplies a valid destination disjoint from this
146        // memory and its backing. Forwarding preserves the pointer and count;
147        // VirtualMemory owns bucket translation and initializes the destination
148        // on success. After a panic, initialization must not be assumed.
149        unsafe { self.memory.read_unsafe(offset, dst, count) }
150    }
151    fn write(&self, offset: u64, src: &[u8]) {
152        self.check_io_bounds(offset, src.len());
153        self.memory.write(offset, src);
154    }
155}
ic_memory/runtime/backing.rs

ic_memory/runtime/
backing.rs
