ic_memory/runtime/backing.rs
1use ic_platform::stable_structures::{Memory, memory_manager::VirtualMemory};
2use std::{
3 cell::{Cell, RefCell},
4 rc::Rc,
5};
6
7// Physical custody, immutable geometry and exact manager metadata authority are
8// shared by the runtime and every handle, including detached ledger handles.
9pub(super) struct GrowthState<M: Memory> {
10 // Fields drop in declaration order: retire the claim while its backing's
11 // identity allocation is still alive, before another thread can reuse it.
12 pub _claim: super::custody::BackingClaim,
13 pub backing: Rc<M>,
14 pub bucket_size_pages: u16,
15 pub authority: RefCell<super::layout::ManagerAuthority>,
16 pub open_slots: [Cell<u64>; 4],
17}
18
19impl<M: Memory> GrowthState<M> {
20 pub fn lease_bit(&self, id: u8) -> (&Cell<u64>, u64) {
21 const WORD_BITS: u8 = 64;
22 (
23 &self.open_slots[usize::from(id / WORD_BITS)],
24 1_u64 << (id % WORD_BITS),
25 )
26 }
27
28 pub fn verify(&self) -> Result<(), super::MemoryManagerLayoutError> {
29 self.authority.borrow().verify(self.backing.as_ref())
30 }
31}
32
33///
34/// RuntimeMemory
35///
36/// Exclusive virtual memory leased through a runtime's committed authority.
37/// Implements [`Memory`] for stable structures without exposing the backing
38/// memory or an alternate manager. The lease cannot be cloned; dropping it
39/// permits another open of the same allocation. Moving it into a collection
40/// transfers the lease until that collection releases the memory.
41/// The handle can outlive its runtime and retains the backing claim. A replacement
42/// runtime is refused until every handle releases it. Custom backing identities
43/// must follow [`super::MemoryBacking`]; raw aliased access remains prohibited.
44/// IO checks the virtual extent before upstream bucket translation. Empty IO
45/// is valid at the end of memory, but not beyond it. Reads preserve the upstream
46/// optimized support for uninitialized destinations through `Memory::read_unsafe`.
47/// Growth reserves physical capacity before assigning manager buckets. Ordinary
48/// backing refusal, arithmetic overflow and bucket exhaustion return typed errors
49/// without changing virtual extents or manager metadata. Backing implementations
50/// must obey the `Memory` contract; traps and partial writes are not transactions
51/// on native memory. Typed collections retain their own failure behavior.
52/// Size and IO trust exclusive backing custody between full metadata checks at
53/// construction, initial bootstrap, lease opening and growth. Raw changes during
54/// a live lease are prohibited; ordinary IO does not scan for them. Detected
55/// divergence permanently poisons this owner, including its existing handles.
56/// Invalid IO bounds still refuse before backing access.
57///
58
59pub struct RuntimeMemory<M: Memory> {
60 pub(super) memory: VirtualMemory<Rc<M>>,
61 pub(super) growth: Rc<GrowthState<M>>,
62 pub(super) id: u8,
63}
64
65impl<M: Memory> Drop for RuntimeMemory<M> {
66 fn drop(&mut self) {
67 let (word, mask) = self.growth.lease_bit(self.id);
68 word.set(word.get() & !mask);
69 }
70}
71
72impl<M: Memory> RuntimeMemory<M> {
73 // The manager bounds page counts by 32768 buckets of at most u16::MAX
74 // pages, so converting its virtual extent to bytes cannot overflow u64.
75 // Check before the upstream cache: its buckets can include virtual slack,
76 // and its span/translation arithmetic is unchecked in release builds.
77 #[expect(
78 clippy::inline_always,
79 reason = "matched PocketIC measurements reduce IO instructions with bounded Wasm growth"
80 )]
81 #[inline(always)]
82 fn check_io_bounds(&self, offset: u64, count: usize) {
83 let extent = self.memory.size() * crate::constants::WASM_PAGE_SIZE_BYTES;
84 assert!(
85 offset <= extent && count as u64 <= extent - offset,
86 "virtual memory access out of bounds",
87 );
88 self.growth
89 .authority
90 .borrow()
91 .check_healthy()
92 .expect("persisted manager metadata differs from runtime authority");
93 }
94
95 /// Grow by the requested pages, returning the previous virtual page count.
96 ///
97 /// Capacity is reserved before assigning manager buckets. Ordinary refusal
98 /// preserves virtual extents and manager metadata and permits retry. The
99 /// upstream [`Memory::grow`] adapter translates errors into its required
100 /// `-1` sentinel; direct runtime callers receive [`super::RuntimeGrowError`].
101 /// A zero-page request verifies metadata and returns the current extent
102 /// without growth or writes, after checking for reentrant growth.
103 ///
104 /// # Panics
105 ///
106 /// Panics if a private growth-accounting invariant is broken or backing
107 /// memory panics.
108 pub fn grow(&self, pages: u64) -> Result<u64, super::RuntimeGrowError> {
109 use super::RuntimeGrowError;
110 let mut authority = self
111 .growth
112 .authority
113 .try_borrow_mut()
114 .map_err(|_| RuntimeGrowError::ReentrantAccess)?;
115 authority.verify(self.growth.backing.as_ref())?;
116 let old_pages = self.memory.size();
117 if pages == 0 {
118 return Ok(old_pages);
119 }
120 let plan = GrowthPlan::new(
121 old_pages,
122 pages,
123 self.growth.bucket_size_pages,
124 authority.allocated_buckets(),
125 )?;
126 let physical_pages = self.growth.backing.size();
127 if plan.required_pages > physical_pages
128 && self
129 .growth
130 .backing
131 .grow(plan.required_pages - physical_pages)
132 < 0
133 {
134 return Err(RuntimeGrowError::BackingRefused {
135 additional_pages: plan.required_pages - physical_pages,
136 });
137 }
138 // The pinned manager's only refusal is bucket exhaustion, already
139 // checked above while all handles share this exclusive reservation.
140 // Physical capacity is reserved before it assigns any buckets.
141 assert_eq!(
142 self.memory.grow(pages),
143 old_pages.cast_signed(),
144 "preflighted manager growth returns the previous virtual extent"
145 );
146 authority.record_growth(self.id, plan.new_pages, plan.total_buckets);
147 authority.verify(self.growth.backing.as_ref())?;
148 Ok(old_pages)
149 }
150}
151
152// Pure capacity admission; reservation and manager mutation remain in grow().
153struct GrowthPlan {
154 new_pages: u64,
155 total_buckets: u16,
156 required_pages: u64,
157}
158
159impl GrowthPlan {
160 fn new(
161 old_pages: u64,
162 pages: u64,
163 bucket_size_pages: u16,
164 allocated_buckets: u16,
165 ) -> Result<Self, super::RuntimeGrowError> {
166 use super::RuntimeGrowError;
167 let new_pages = old_pages
168 .checked_add(pages)
169 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
170 let bucket_pages = u64::from(bucket_size_pages);
171 let extra = new_pages.div_ceil(bucket_pages) - old_pages.div_ceil(bucket_pages);
172 let total = u64::from(allocated_buckets)
173 .checked_add(extra)
174 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
175 if total > u64::from(super::layout::BUCKET_CAPACITY) {
176 return Err(RuntimeGrowError::BucketExhausted {
177 required_buckets: total,
178 capacity: super::layout::BUCKET_CAPACITY,
179 });
180 }
181 #[expect(
182 clippy::cast_possible_truncation,
183 reason = "admission bounds total by the u16 bucket capacity"
184 )]
185 let total_buckets = total as u16;
186 let required_pages = 1 + total * bucket_pages;
187 Ok(Self {
188 new_pages,
189 total_buckets,
190 required_pages,
191 })
192 }
193}
194
195impl<M: Memory> Memory for RuntimeMemory<M> {
196 fn size(&self) -> u64 {
197 self.growth
198 .authority
199 .borrow()
200 .check_healthy()
201 .expect("persisted manager metadata differs from runtime authority");
202 self.memory.size()
203 }
204 fn grow(&self, pages: u64) -> i64 {
205 // The upstream trait fixes the sentinel contract. Direct calls use the
206 // inherent typed method; virtual extents fit in i64 by bucket capacity.
207 Self::grow(self, pages).map_or(-1, u64::cast_signed)
208 }
209 fn read(&self, offset: u64, dst: &mut [u8]) {
210 self.check_io_bounds(offset, dst.len());
211 self.memory.read(offset, dst);
212 }
213 #[expect(
214 unsafe_code,
215 reason = "delegate the upstream raw-read contract unchanged"
216 )]
217 unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) {
218 self.check_io_bounds(offset, count);
219 // SAFETY: The caller supplies a valid destination disjoint from this
220 // memory and its backing. Forwarding preserves the pointer and count;
221 // VirtualMemory owns bucket translation and initializes the destination
222 // on success. After a panic, initialization must not be assumed.
223 unsafe { self.memory.read_unsafe(offset, dst, count) }
224 }
225 fn write(&self, offset: u64, src: &[u8]) {
226 self.check_io_bounds(offset, src.len());
227 self.memory.write(offset, src);
228 }
229}