ic_memory/runtime/
backing.rs

1use ic_stable_structures::{Memory, memory_manager::VirtualMemory};
2use std::{
3    cell::{Cell, RefCell},
4    rc::Rc,
5};
6
7// Backing, immutable geometry and one live bucket count for the sole manager,
8// shared by the runtime and all handles, including the ledger. The count is
9// seeded from validated metadata on reopen and updated only after grow.
10pub(super) struct GrowthState<M: Memory> {
11    pub backing: Rc<M>,
12    pub bucket_size_pages: u16,
13    pub allocated_buckets: RefCell<u16>,
14    pub open_slots: [Cell<u64>; 4],
15}
16
17///
18/// RuntimeMemory
19///
20/// Exclusive virtual memory leased through a runtime's committed authority.
21/// Implements [`Memory`] for stable structures without exposing the backing
22/// memory or an alternate manager. The lease cannot be cloned; dropping it
23/// permits another open of the same allocation. Moving it into a collection
24/// transfers the lease until that collection releases the memory.
25/// The handle can outlive its runtime. Hosts must not construct another runtime
26/// or access aliased backing while any runtime or lease for that backing lives.
27/// IO checks the virtual extent before upstream bucket translation. Empty IO
28/// is valid at the end of memory, but not beyond it. Reads preserve the upstream
29/// optimized support for uninitialized destinations through `Memory::read_unsafe`.
30/// Growth reserves physical capacity before assigning manager buckets. Ordinary
31/// backing refusal, arithmetic overflow and bucket exhaustion return typed errors
32/// without changing virtual extents or manager metadata. Backing implementations
33/// must obey the `Memory` contract; traps and partial writes are not transactions
34/// on native memory. Typed collections retain their own failure behavior.
35///
36
37pub struct RuntimeMemory<M: Memory> {
38    pub(super) memory: VirtualMemory<Rc<M>>,
39    pub(super) growth: Rc<GrowthState<M>>,
40    pub(super) id: u8,
41}
42
43impl<M: Memory> Drop for RuntimeMemory<M> {
44    fn drop(&mut self) {
45        let word = &self.growth.open_slots[usize::from(self.id / 64)];
46        word.set(word.get() & !(1_u64 << (self.id % 64)));
47    }
48}
49
50impl<M: Memory> RuntimeMemory<M> {
51    // The manager bounds page counts by 32768 buckets of at most u16::MAX
52    // pages, so converting its virtual extent to bytes cannot overflow u64.
53    // Check before the upstream cache: its buckets can include virtual slack,
54    // and its span/translation arithmetic is unchecked in release builds.
55    #[expect(
56        clippy::inline_always,
57        reason = "matched PocketIC measurements reduce IO instructions with bounded Wasm growth"
58    )]
59    #[inline(always)]
60    fn check_io_bounds(&self, offset: u64, count: usize) {
61        let extent = self.memory.size() * crate::constants::WASM_PAGE_SIZE_BYTES;
62        assert!(
63            offset <= extent && count as u64 <= extent - offset,
64            "virtual memory access out of bounds",
65        );
66    }
67
68    /// Grow by the requested pages, returning the previous virtual page count.
69    ///
70    /// Capacity is reserved before assigning manager buckets. Ordinary refusal
71    /// preserves virtual extents and manager metadata and permits retry. The
72    /// upstream [`Memory::grow`] adapter translates errors into its required
73    /// `-1` sentinel; direct runtime callers receive [`super::RuntimeGrowError`].
74    /// A zero-page request returns the current extent without backing IO or
75    /// manager mutation, after checking for reentrant growth.
76    ///
77    /// # Panics
78    ///
79    /// Panics if a private growth-accounting invariant is broken or backing
80    /// memory panics.
81    pub fn grow(&self, pages: u64) -> Result<u64, super::RuntimeGrowError> {
82        use super::RuntimeGrowError;
83        let mut allocated = self
84            .growth
85            .allocated_buckets
86            .try_borrow_mut()
87            .map_err(|_| RuntimeGrowError::ReentrantAccess)?;
88        let old_pages = self.memory.size();
89        if pages == 0 {
90            return Ok(old_pages);
91        }
92        let new_pages = old_pages
93            .checked_add(pages)
94            .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
95        let bucket_pages = u64::from(self.growth.bucket_size_pages);
96        let extra = new_pages.div_ceil(bucket_pages) - old_pages.div_ceil(bucket_pages);
97        let total = u64::from(*allocated)
98            .checked_add(extra)
99            .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
100        if total > u64::from(super::layout::BUCKET_CAPACITY) {
101            return Err(RuntimeGrowError::BucketExhausted {
102                required_buckets: total,
103                capacity: super::layout::BUCKET_CAPACITY,
104            });
105        }
106        #[expect(
107            clippy::cast_possible_truncation,
108            reason = "admission bounds total by the u16 bucket capacity"
109        )]
110        let total_buckets = total as u16;
111        let required_pages = 1 + total * bucket_pages;
112        let physical_pages = self.growth.backing.size();
113        if required_pages > physical_pages
114            && self.growth.backing.grow(required_pages - physical_pages) < 0
115        {
116            return Err(RuntimeGrowError::BackingRefused {
117                additional_pages: required_pages - physical_pages,
118            });
119        }
120        // The pinned manager's only refusal is bucket exhaustion, already
121        // checked above while all handles share this exclusive reservation.
122        // Physical capacity is reserved before it assigns any buckets.
123        assert_eq!(
124            self.memory.grow(pages),
125            old_pages.cast_signed(),
126            "preflighted manager growth returns the previous virtual extent"
127        );
128        *allocated = total_buckets;
129        Ok(old_pages)
130    }
131}
132
133impl<M: Memory> Memory for RuntimeMemory<M> {
134    fn size(&self) -> u64 {
135        self.memory.size()
136    }
137    fn grow(&self, pages: u64) -> i64 {
138        // The upstream trait fixes the sentinel contract. Direct calls use the
139        // inherent typed method; virtual extents fit in i64 by bucket capacity.
140        Self::grow(self, pages).map_or(-1, u64::cast_signed)
141    }
142    fn read(&self, offset: u64, dst: &mut [u8]) {
143        self.check_io_bounds(offset, dst.len());
144        self.memory.read(offset, dst);
145    }
146    #[expect(
147        unsafe_code,
148        reason = "delegate the upstream raw-read contract unchanged"
149    )]
150    unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) {
151        self.check_io_bounds(offset, count);
152        // SAFETY: The caller supplies a valid destination disjoint from this
153        // memory and its backing. Forwarding preserves the pointer and count;
154        // VirtualMemory owns bucket translation and initializes the destination
155        // on success. After a panic, initialization must not be assumed.
156        unsafe { self.memory.read_unsafe(offset, dst, count) }
157    }
158    fn write(&self, offset: u64, src: &[u8]) {
159        self.check_io_bounds(offset, src.len());
160        self.memory.write(offset, src);
161    }
162}
ic_memory/runtime/backing.rs

ic_memory/runtime/
backing.rs
