ic_memory/runtime/backing.rs
1use ic_platform::stable_structures::{Memory, memory_manager::VirtualMemory};
2use std::{
3 cell::{Cell, RefCell},
4 rc::Rc,
5};
6
7// Physical custody, immutable geometry and exact manager metadata authority are
8// shared by the runtime and every handle, including detached ledger handles.
9pub(super) struct GrowthState<M: Memory> {
10 pub backing: Rc<M>,
11 pub bucket_size_pages: u16,
12 pub authority: RefCell<super::layout::ManagerAuthority>,
13 pub _claim: super::custody::BackingClaim,
14 pub open_slots: [Cell<u64>; 4],
15}
16
17impl<M: Memory> GrowthState<M> {
18 pub fn verify(&self) -> Result<(), super::MemoryManagerLayoutError> {
19 self.authority.borrow().verify(self.backing.as_ref())
20 }
21}
22
23///
24/// RuntimeMemory
25///
26/// Exclusive virtual memory leased through a runtime's committed authority.
27/// Implements [`Memory`] for stable structures without exposing the backing
28/// memory or an alternate manager. The lease cannot be cloned; dropping it
29/// permits another open of the same allocation. Moving it into a collection
30/// transfers the lease until that collection releases the memory.
31/// The handle can outlive its runtime and retains the backing claim. A replacement
32/// runtime is refused until every handle releases it. Custom backing identities
33/// must follow [`super::MemoryBacking`]; raw aliased access remains prohibited.
34/// IO checks the virtual extent before upstream bucket translation. Empty IO
35/// is valid at the end of memory, but not beyond it. Reads preserve the upstream
36/// optimized support for uninitialized destinations through `Memory::read_unsafe`.
37/// Growth reserves physical capacity before assigning manager buckets. Ordinary
38/// backing refusal, arithmetic overflow and bucket exhaustion return typed errors
39/// without changing virtual extents or manager metadata. Backing implementations
40/// must obey the `Memory` contract; traps and partial writes are not transactions
41/// on native memory. Typed collections retain their own failure behavior.
42/// Size and IO trust exclusive backing custody between full metadata checks at
43/// construction, initial bootstrap, lease opening and growth. Raw changes during
44/// a live lease are prohibited; ordinary IO does not scan for them. Detected
45/// divergence permanently poisons this owner, including its existing handles.
46/// Invalid IO bounds still refuse before backing access.
47///
48
49pub struct RuntimeMemory<M: Memory> {
50 pub(super) memory: VirtualMemory<Rc<M>>,
51 pub(super) growth: Rc<GrowthState<M>>,
52 pub(super) id: u8,
53}
54
55impl<M: Memory> Drop for RuntimeMemory<M> {
56 fn drop(&mut self) {
57 let word = &self.growth.open_slots[usize::from(self.id / 64)];
58 word.set(word.get() & !(1_u64 << (self.id % 64)));
59 }
60}
61
62impl<M: Memory> RuntimeMemory<M> {
63 // The manager bounds page counts by 32768 buckets of at most u16::MAX
64 // pages, so converting its virtual extent to bytes cannot overflow u64.
65 // Check before the upstream cache: its buckets can include virtual slack,
66 // and its span/translation arithmetic is unchecked in release builds.
67 #[expect(
68 clippy::inline_always,
69 reason = "matched PocketIC measurements reduce IO instructions with bounded Wasm growth"
70 )]
71 #[inline(always)]
72 fn check_io_bounds(&self, offset: u64, count: usize) {
73 let extent = self.memory.size() * crate::constants::WASM_PAGE_SIZE_BYTES;
74 assert!(
75 offset <= extent && count as u64 <= extent - offset,
76 "virtual memory access out of bounds",
77 );
78 self.growth
79 .authority
80 .borrow()
81 .check_healthy()
82 .expect("persisted manager metadata differs from runtime authority");
83 }
84
85 /// Grow by the requested pages, returning the previous virtual page count.
86 ///
87 /// Capacity is reserved before assigning manager buckets. Ordinary refusal
88 /// preserves virtual extents and manager metadata and permits retry. The
89 /// upstream [`Memory::grow`] adapter translates errors into its required
90 /// `-1` sentinel; direct runtime callers receive [`super::RuntimeGrowError`].
91 /// A zero-page request verifies metadata and returns the current extent
92 /// without growth or writes, after checking for reentrant growth.
93 ///
94 /// # Panics
95 ///
96 /// Panics if a private growth-accounting invariant is broken or backing
97 /// memory panics.
98 pub fn grow(&self, pages: u64) -> Result<u64, super::RuntimeGrowError> {
99 use super::RuntimeGrowError;
100 let mut authority = self
101 .growth
102 .authority
103 .try_borrow_mut()
104 .map_err(|_| RuntimeGrowError::ReentrantAccess)?;
105 authority.verify(self.growth.backing.as_ref())?;
106 let old_pages = self.memory.size();
107 if pages == 0 {
108 return Ok(old_pages);
109 }
110 let new_pages = old_pages
111 .checked_add(pages)
112 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
113 let bucket_pages = u64::from(self.growth.bucket_size_pages);
114 let extra = new_pages.div_ceil(bucket_pages) - old_pages.div_ceil(bucket_pages);
115 let total = u64::from(authority.allocated_buckets())
116 .checked_add(extra)
117 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
118 if total > u64::from(super::layout::BUCKET_CAPACITY) {
119 return Err(RuntimeGrowError::BucketExhausted {
120 required_buckets: total,
121 capacity: super::layout::BUCKET_CAPACITY,
122 });
123 }
124 #[expect(
125 clippy::cast_possible_truncation,
126 reason = "admission bounds total by the u16 bucket capacity"
127 )]
128 let total_buckets = total as u16;
129 let required_pages = 1 + total * bucket_pages;
130 let physical_pages = self.growth.backing.size();
131 if required_pages > physical_pages
132 && self.growth.backing.grow(required_pages - physical_pages) < 0
133 {
134 return Err(RuntimeGrowError::BackingRefused {
135 additional_pages: required_pages - physical_pages,
136 });
137 }
138 // The pinned manager's only refusal is bucket exhaustion, already
139 // checked above while all handles share this exclusive reservation.
140 // Physical capacity is reserved before it assigns any buckets.
141 assert_eq!(
142 self.memory.grow(pages),
143 old_pages.cast_signed(),
144 "preflighted manager growth returns the previous virtual extent"
145 );
146 authority.record_growth(self.id, new_pages, total_buckets);
147 authority.verify(self.growth.backing.as_ref())?;
148 Ok(old_pages)
149 }
150}
151
152impl<M: Memory> Memory for RuntimeMemory<M> {
153 fn size(&self) -> u64 {
154 self.growth
155 .authority
156 .borrow()
157 .check_healthy()
158 .expect("persisted manager metadata differs from runtime authority");
159 self.memory.size()
160 }
161 fn grow(&self, pages: u64) -> i64 {
162 // The upstream trait fixes the sentinel contract. Direct calls use the
163 // inherent typed method; virtual extents fit in i64 by bucket capacity.
164 Self::grow(self, pages).map_or(-1, u64::cast_signed)
165 }
166 fn read(&self, offset: u64, dst: &mut [u8]) {
167 self.check_io_bounds(offset, dst.len());
168 self.memory.read(offset, dst);
169 }
170 #[expect(
171 unsafe_code,
172 reason = "delegate the upstream raw-read contract unchanged"
173 )]
174 unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) {
175 self.check_io_bounds(offset, count);
176 // SAFETY: The caller supplies a valid destination disjoint from this
177 // memory and its backing. Forwarding preserves the pointer and count;
178 // VirtualMemory owns bucket translation and initializes the destination
179 // on success. After a panic, initialization must not be assumed.
180 unsafe { self.memory.read_unsafe(offset, dst, count) }
181 }
182 fn write(&self, offset: u64, src: &[u8]) {
183 self.check_io_bounds(offset, src.len());
184 self.memory.write(offset, src);
185 }
186}